diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java index 050eec0..bb7cb3f 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java @@ -403,7 +403,9 @@ public class Spider implements Runnable, Task { protected void processRequest(Request request) { Page page = downloader.download(request, this); if (page == null) { - throw new RuntimeException("unaccpetable response status"); + sleep(site.getSleepTime()); + onError(request); + return; } // for cycle retry if (page.isNeedCycleRetry()) {