Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix ZipCodePageProcessor

master
yihua.huang 2014-03-13 22:55:50 +08:00
parent 2801daf185
commit 9606a173cd
3 changed files with 33 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
webmagic-core/src/main/java/us/codecraft/webmagic/selector/PlainText.java
View File

@ -109,7 +109,7 @@ public class PlainText implements Selectable {
}
@Override
public String toString() {
public String get() {
if (CollectionUtils.isNotEmpty(all())) {
return all().get(0);
} else {
@ -117,6 +117,11 @@ public class PlainText implements Selectable {
}
}
@Override
public String toString() {
return get();
}
@Override
public boolean match() {
return strings != null && strings.size() > 0;

7
webmagic-core/src/main/java/us/codecraft/webmagic/selector/Selectable.java
View File

@ -99,6 +99,13 @@ public interface Selectable {
*/
public String toString();
/**
* single string result
*
* @return single string result
*/
public String get();
/**
* if result exist for select
*

44
webmagic-samples/src/main/java/us/codecraft/webmagic/samples/scheduler/ZipCodePageProcessor.java
View File

@ -9,8 +9,9 @@ import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.PriorityScheduler;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static us.codecraft.webmagic.selector.Selectors.regex;
import static us.codecraft.webmagic.selector.Selectors.xpath;
/**
@ -19,16 +20,16 @@ import static us.codecraft.webmagic.selector.Selectors.xpath;
public class ZipCodePageProcessor implements PageProcessor {
private Site site = Site.me().setCharset("gb2312")
.setSleepTime(100).addStartUrl("http://www.ip138.com/post/");
.setSleepTime(100);
@Override
public void process(Page page) {
if (page.getUrl().toString().equals("http://www.ip138.com/post/")) {
processCountry(page);
} else if (page.getUrl().regex("http://www\\.ip138\\.com/post/\\w+[/]?$").toString() != null) {
processProvince(page);
} else {
} else if (page.getUrl().regex("http://www\\.ip138\\.com/\\d{6}[/]?$").toString() != null) {
processDistrict(page);
} else {
processProvince(page);
}
}
@ -45,28 +46,26 @@ public class ZipCodePageProcessor implements PageProcessor {
private void processProvince(Page page) {
//这里仅靠xpath没法精准定位，所以使用正则作为筛选，不符合正则的会被过滤掉
List<String> districts = page.getHtml().xpath("//body/table/tbody/tr/td").regex(".*http://www\\.ip138\\.com/post/\\w+/\\w+.*").all();
List<String> districts = page.getHtml().xpath("//body/table/tbody/tr[@bgcolor=\"#ffffff\"]").all();
Pattern pattern = Pattern.compile("<td>([^<>]+)</td>.*?href=\"(.*?)\"",Pattern.DOTALL);
for (String district : districts) {
String link = xpath("//@href").select(district);
String title = xpath("/text()").select(district);
Request request = new Request(link).setPriority(1).putExtra("province", page.getRequest().getExtra("province")).putExtra("district", title);
page.addTargetRequest(request);
Matcher matcher = pattern.matcher(district);
while (matcher.find()) {
String title = matcher.group(1);
String link = matcher.group(2);
Request request = new Request(link).setPriority(1).putExtra("province", page.getRequest().getExtra("province")).putExtra("district", title);
page.addTargetRequest(request);
}
}
}
private void processDistrict(Page page) {
String province = page.getRequest().getExtra("province").toString();
String district = page.getRequest().getExtra("district").toString();
List<String> counties = page.getHtml().xpath("//body/table/tbody/tr").regex(".*<td>\\d+</td>.*").all();
String regex = "<td[^<>]*>([^<>]+)</td><td[^<>]*>([^<>]+)</td><td[^<>]*>([^<>]+)</td><td[^<>]*>([^<>]+)</td>";
for (String county : counties) {
String county0 = regex(regex, 1).select(county);
String county1 = regex(regex, 2).select(county);
String zipCode = regex(regex, 3).select(county);
page.putField("result", StringUtils.join(new String[]{province, district,
county0, county1, zipCode}, "\t"));
}
List<String> links = page.getHtml().links().regex("http://www\\.ip138\\.com/post/\\w+/\\w+").all();
String zipCode = page.getHtml().regex("<h2>邮编：(\\d+)</h2>").toString();
page.putField("result", StringUtils.join(new String[]{province, district,
zipCode}, "\t"));
List<String> links = page.getHtml().links().regex("http://www\\.ip138\\.com/\\d{6}[/]?$").all();
for (String link : links) {
page.addTargetRequest(new Request(link).setPriority(2).putExtra("province", province).putExtra("district", district));
}
@ -79,11 +78,8 @@ public class ZipCodePageProcessor implements PageProcessor {
}
public static void main(String[] args) {
Spider.create(new ZipCodePageProcessor()).scheduler(new PriorityScheduler()).run();
Spider spider = Spider.create(new ZipCodePageProcessor()).scheduler(new PriorityScheduler()).addUrl("http://www.ip138.com/post/");
PriorityScheduler scheduler = new PriorityScheduler();
Spider spider = Spider.create(new ZipCodePageProcessor()).scheduler(scheduler);
scheduler.push(new Request("http://www.baidu.com/s?wd=webmagic&f=12&rsp=0&oq=webmagix&tn=baiduhome_pg&ie=utf-8"),spider);
spider.run();
}
}