将Spider.processRequest()抛出异常改回原来的逻辑

master
yihua.huang 2016-12-18 11:04:58 +08:00
parent 97592d6720
commit eeb607fd0e
1 changed files with 3 additions and 1 deletions

View File

@ -403,7 +403,9 @@ public class Spider implements Runnable, Task {
protected void processRequest(Request request) {
Page page = downloader.download(request, this);
if (page == null) {
throw new RuntimeException("unaccpetable response status");
sleep(site.getSleepTime());
onError(request);
return;
}
// for cycle retry
if (page.isNeedCycleRetry()) {