Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

new sample

master
yihua.huang 2014-06-10 17:38:54 +08:00
parent b3a282e58d
commit eae37c868b
2 changed files with 66 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
View File

@ -325,7 +325,7 @@ public class Spider implements Runnable, Task {
onError(requestFinal);
logger.error("process request " + requestFinal + " error", e);
} finally {
if (site.getHttpProxyPool().isEnable()) {
if (site.getHttpProxyPool()!=null && site.getHttpProxyPool().isEnable()) {
site.returnHttpProxyToPool((HttpHost) requestFinal.getExtra(Request.PROXY), (Integer) requestFinal
.getExtra(Request.STATUS_CODE));
}

65
webmagic-samples/src/main/java/us/codecraft/webmagic/samples/AlexanderMcqueenGoodsProcessor.java 100644
View File

@ -0,0 +1,65 @@
package us.codecraft.webmagic.samples;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.PriorityScheduler;
/**
* @author code4crafer@gmail.com
*/
public class AlexanderMcqueenGoodsProcessor implements PageProcessor {
private Site site = Site.me().setRetryTimes(3).setSleepTime(0);
public static final String URL_LIST = "http://www\\.alexandermcqueen\\.cn/.*";
public static final String URL_POST = "http://www\\.alexandermcqueen\\.cn/cn/\\w+/.*\\.html";
@Override
public void process(Page page) {
if (page.getUrl().regex(URL_POST).match()) {
page.putField("goodsName", page.getHtml().xpath("//div[@id='description']/h1/tidyText()"));
if (page.getResultItems().get("goodsName") == null) {
page.setSkip(true);
}
page.putField("currency", page.getHtml().xpath("//div[@id='description']//div[@class='itemBoxPrice']/span//span[@class='currency']/tidyText()"));
page.putField("goodsPrice", page.getHtml().xpath("//div[@id='description']//div[@class='itemBoxPrice']/span//span[@class='priceValue']/tidyText()"));
page.putField("description", page.getHtml()
.xpath("//div[@id='tabbedDescription']//div[@class='tabbedDescription']//ul[@id='tabs']//li[@id='tab_description']/div[@id='description_pane']/tidyText()"));
page.putField("material", page.getHtml()
.xpath("//div[@id='tabbedDescription']" +
"//div[@class='tabbedDescription']" +
"//ul[@id='tabs']" +
"//li[@id='tab_description']" +
"//div[@class='productProperty']" +
"//div[@class='productPropertyRow']/span[2]/tidyText()"));
page.putField("goodsCode", page.getHtml()
.xpath("//div[@id='tabbedDescription']" +
"//div[@class='tabbedDescription']" +
"//ul[@id='tabs']" +
"//li[@id='tab_description']" +
"//div[@class='productProperty']" +
"//div[@class='productPropertyRow']//span[@id='modelFabricColorContainer']/tidyText()"));
page.putField("goodsSize", page.getHtml()
.xpath("//div[@id='sizesContainer']//div[@id='sizes']//ul[@class='SizeW']"));
page.putField("goodsColors", page.getHtml()
.xpath("//div[@id='colors']/ul/html()"));
} else {
page.addTargetRequests(page.getHtml().links().regex(URL_POST).all(), 1000);
page.addTargetRequests(page.getHtml().links().regex(URL_LIST).all(), 1);
}
}
@Override
public Site getSite() {
return site;
}
public static void main(String[] args) {
Spider.create(new AlexanderMcqueenGoodsProcessor()).setScheduler(new PriorityScheduler())
.addUrl("http://www.alexandermcqueen.cn/sitemap.asp?tskay=E2F1A848").thread(5).run();
}
}