Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add isExtractLinks to OOSpider #575

master
yihua.huang 2017-05-27 16:20:06 +08:00
parent 8999ea9320
commit 7ffc6998ef
2 changed files with 19 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
webmagic-extension/src/main/java/us/codecraft/webmagic/model/ModelPageProcessor.java
View File

@ -23,6 +23,8 @@ class ModelPageProcessor implements PageProcessor {
private Site site; private Site site;
private boolean extractLinks = true;
public static ModelPageProcessor create(Site site, Class... clazzs) { public static ModelPageProcessor create(Site site, Class... clazzs) {
ModelPageProcessor modelPageProcessor = new ModelPageProcessor(site); ModelPageProcessor modelPageProcessor = new ModelPageProcessor(site);
for (Class clazz : clazzs) { for (Class clazz : clazzs) {
@ -45,8 +47,10 @@ class ModelPageProcessor implements PageProcessor {
@Override @Override
public void process(Page page) { public void process(Page page) {
for (PageModelExtractor pageModelExtractor : pageModelExtractorList) { for (PageModelExtractor pageModelExtractor : pageModelExtractorList) {
extractLinks(page, pageModelExtractor.getHelpUrlRegionSelector(), pageModelExtractor.getHelpUrlPatterns()); if (extractLinks) {
extractLinks(page, pageModelExtractor.getTargetUrlRegionSelector(), pageModelExtractor.getTargetUrlPatterns()); extractLinks(page, pageModelExtractor.getHelpUrlRegionSelector(), pageModelExtractor.getHelpUrlPatterns());
extractLinks(page, pageModelExtractor.getTargetUrlRegionSelector(), pageModelExtractor.getTargetUrlPatterns());
}
Object process = pageModelExtractor.process(page); Object process = pageModelExtractor.process(page);
if (process == null || (process instanceof List && ((List) process).size() == 0)) { if (process == null || (process instanceof List && ((List) process).size() == 0)) {
continue; continue;
@ -83,4 +87,12 @@ class ModelPageProcessor implements PageProcessor {
public Site getSite() { public Site getSite() {
return site; return site;
} }
public boolean isExtractLinks() {
return extractLinks;
}
public void setExtractLinks(boolean extractLinks) {
this.extractLinks = extractLinks;
}
} }

5
webmagic-extension/src/main/java/us/codecraft/webmagic/model/OOSpider.java
View File

@ -97,4 +97,9 @@ public class OOSpider<T> extends Spider {
return this; return this;
} }
public OOSpider setIsExtractLinks(boolean isExtractLinks){
modelPageProcessor.setExtractLinks(isExtractLinks);
return this;
}
} }