diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java index 7f075b1..6560a1b 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java @@ -402,6 +402,7 @@ public class Spider implements Runnable, Task { if (page == null) { sleep(site.getSleepTime()); onError(request); + return; } // for cycle retry if (page.isNeedCycleRetry()) { diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/example/PatternProcessorExample.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/example/PatternProcessorExample.java index f6b2e9b..8ecb08f 100644 --- a/webmagic-extension/src/main/java/us/codecraft/webmagic/example/PatternProcessorExample.java +++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/example/PatternProcessorExample.java @@ -54,7 +54,7 @@ public class PatternProcessorExample { } }; - CompositePageProcessor pageProcessor = new CompositePageProcessor(Site.me().setDomain("github.com").setRetryTimes(0).setSleepTime(0)); + CompositePageProcessor pageProcessor = new CompositePageProcessor(Site.me().setDomain("github.com").setRetryTimes(3)); CompositePipeline pipeline = new CompositePipeline(); pageProcessor.setSubPageProcessors(githubRepoProcessor, githubUserProcessor);