webMagic-Mobai
/
magic-Dependency
mirror of https://gitea.qinmian.online/Mobai/magic-Dependency.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

comments on sinablogprocessor sample

master
yihua.huang 2014-04-12 20:10:24 +08:00
parent be37d8b216
commit 843e928c2c
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
webmagic-samples/src/main/java/us/codecraft/webmagic/samples/SinaBlogProcessor.java
View File

@ -23,13 +23,14 @@ public class SinaBlogProcessor implements PageProcessor {
@Override @Override
public void process(Page page) { public void process(Page page) {
//列表页
if (page.getUrl().regex(URL_LIST).match()) { if (page.getUrl().regex(URL_LIST).match()) {
page.addTargetRequests(page.getHtml().xpath("//div[@class=\"articleList\"]").links().regex(URL_POST).all()); page.addTargetRequests(page.getHtml().xpath("//div[@class=\"articleList\"]").links().regex(URL_POST).all());
page.addTargetRequests(page.getHtml().links().regex(URL_LIST).all()); page.addTargetRequests(page.getHtml().links().regex(URL_LIST).all());
//文章页
} else { } else {
page.putField("title", page.getHtml().xpath("//div[@class='articalTitle']/h2")); page.putField("title", page.getHtml().xpath("//div[@class='articalTitle']/h2"));
page.putField("content", page.getHtml().xpath("//div[@id='articlebody']//div[@class='articalContent']")); page.putField("content", page.getHtml().xpath("//div[@id='articlebody']//div[@class='articalContent']"));
page.putField("id", page.getUrl().regex("http://blog\\.sina\\.com\\.cn/s/blog_(\\w+)"));
page.putField("date", page.putField("date",
page.getHtml().xpath("//div[@id='articlebody']//span[@class='time SG_txtc']").regex("\\((.*)\\)")); page.getHtml().xpath("//div[@id='articlebody']//span[@class='time SG_txtc']").regex("\\((.*)\\)"));
} }