将Spider.processRequest()抛出异常改回原来的逻辑
parent
97592d6720
commit
eeb607fd0e
|
@ -403,7 +403,9 @@ public class Spider implements Runnable, Task {
|
|||
protected void processRequest(Request request) {
|
||||
Page page = downloader.download(request, this);
|
||||
if (page == null) {
|
||||
throw new RuntimeException("unaccpetable response status");
|
||||
sleep(site.getSleepTime());
|
||||
onError(request);
|
||||
return;
|
||||
}
|
||||
// for cycle retry
|
||||
if (page.isNeedCycleRetry()) {
|
||||
|
|
Loading…
Reference in New Issue