Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

return when page is null

master
yihua.huang 2014-04-25 16:07:41 +08:00
parent 0336f4cdb4
commit 179baa7a22
2 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
View File

@ -402,6 +402,7 @@ public class Spider implements Runnable, Task {
if (page == null) {
sleep(site.getSleepTime());
onError(request);
return;
}
// for cycle retry
if (page.isNeedCycleRetry()) {

2
webmagic-extension/src/main/java/us/codecraft/webmagic/example/PatternProcessorExample.java
View File

@ -54,7 +54,7 @@ public class PatternProcessorExample {
}
};
CompositePageProcessor pageProcessor = new CompositePageProcessor(Site.me().setDomain("github.com").setRetryTimes(0).setSleepTime(0));
CompositePageProcessor pageProcessor = new CompositePageProcessor(Site.me().setDomain("github.com").setRetryTimes(3));
CompositePipeline pipeline = new CompositePipeline();
pageProcessor.setSubPageProcessors(githubRepoProcessor, githubUserProcessor);