Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix jsonpath in AngularJSProcessor

master
yihua.huang 2014-04-13 12:54:44 +08:00
parent 84b897f83b
commit b14f0ee479
1 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
webmagic-samples/src/main/java/us/codecraft/webmagic/samples/AngularJSProcessor.java
View File

@ -3,6 +3,7 @@ package us.codecraft.webmagic.samples;
import org.apache.commons.collections.CollectionUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.JsonPathSelector;
@ -23,15 +24,15 @@ public class AngularJSProcessor implements PageProcessor {
@Override
public void process(Page page) {
if (page.getUrl().regex(LIST_URL).match()) {
List<String> ids = new JsonPathSelector("$.data._id").selectList(page.getRawText());
List<String> ids = new JsonPathSelector("$.data[*]._id").selectList(page.getRawText());
if (CollectionUtils.isNotEmpty(ids)) {
for (String id : ids) {
page.addTargetRequest("http://angularjs\\.cn/api/article/" + id);
page.addTargetRequest("http://angularjs.cn/api/article/" + id);
}
}
} else {
page.putField("title", new JsonPathSelector("$.title").select(page.getRawText()));
page.putField("content", new JsonPathSelector("$.content").select(page.getRawText()));
page.putField("title", new JsonPathSelector("$.data.title").select(page.getRawText()));
page.putField("content", new JsonPathSelector("$.data.content").select(page.getRawText()));
}
}
@ -40,4 +41,8 @@ public class AngularJSProcessor implements PageProcessor {
public Site getSite() {
return site;
}
public static void main(String[] args) {
Spider.create(new AngularJSProcessor()).addUrl("http://angularjs.cn/api/article/latest?p=1&s=20").run();
}
}