diff --git a/lib/commons-codec-1.6.jar b/lib/commons-codec-1.6.jar new file mode 100644 index 0000000..ee1bc49 Binary files /dev/null and b/lib/commons-codec-1.6.jar differ diff --git a/lib/commons-collections-3.2.1.jar b/lib/commons-collections-3.2.1.jar new file mode 100644 index 0000000..c35fa1f Binary files /dev/null and b/lib/commons-collections-3.2.1.jar differ diff --git a/lib/commons-io-1.3.2.jar b/lib/commons-io-1.3.2.jar new file mode 100644 index 0000000..865c9e4 Binary files /dev/null and b/lib/commons-io-1.3.2.jar differ diff --git a/lib/commons-lang-2.6.jar b/lib/commons-lang-2.6.jar new file mode 100644 index 0000000..98467d3 Binary files /dev/null and b/lib/commons-lang-2.6.jar differ diff --git a/lib/commons-lang3-3.1.jar b/lib/commons-lang3-3.1.jar new file mode 100644 index 0000000..a85e539 Binary files /dev/null and b/lib/commons-lang3-3.1.jar differ diff --git a/lib/commons-logging-1.1.3.jar b/lib/commons-logging-1.1.3.jar new file mode 100644 index 0000000..ab51254 Binary files /dev/null and b/lib/commons-logging-1.1.3.jar differ diff --git a/lib/commons-pool-1.5.5.jar b/lib/commons-pool-1.5.5.jar new file mode 100644 index 0000000..7a96587 Binary files /dev/null and b/lib/commons-pool-1.5.5.jar differ diff --git a/lib/fastjson-1.1.35.jar b/lib/fastjson-1.1.35.jar new file mode 100644 index 0000000..e2876e2 Binary files /dev/null and b/lib/fastjson-1.1.35.jar differ diff --git a/lib/guava-15.0.jar b/lib/guava-15.0.jar new file mode 100644 index 0000000..eb9ef8a Binary files /dev/null and b/lib/guava-15.0.jar differ diff --git a/lib/htmlcleaner-2.5.jar b/lib/htmlcleaner-2.5.jar new file mode 100644 index 0000000..4b65546 Binary files /dev/null and b/lib/htmlcleaner-2.5.jar differ diff --git a/lib/httpclient-4.3.1.jar b/lib/httpclient-4.3.1.jar new file mode 100644 index 0000000..cdd1e38 Binary files /dev/null and b/lib/httpclient-4.3.1.jar differ diff --git a/lib/httpcore-4.3.jar b/lib/httpcore-4.3.jar new file mode 100644 index 0000000..ddfe6da Binary files /dev/null and b/lib/httpcore-4.3.jar differ diff --git a/lib/jdom2-2.0.4.jar b/lib/jdom2-2.0.4.jar new file mode 100644 index 0000000..19cafb8 Binary files /dev/null and b/lib/jdom2-2.0.4.jar differ diff --git a/lib/jedis-2.0.0.jar b/lib/jedis-2.0.0.jar new file mode 100644 index 0000000..aaf61e7 Binary files /dev/null and b/lib/jedis-2.0.0.jar differ diff --git a/lib/json-path-0.8.1.jar b/lib/json-path-0.8.1.jar new file mode 100644 index 0000000..e43dbd2 Binary files /dev/null and b/lib/json-path-0.8.1.jar differ diff --git a/lib/json-smart-1.1.1.jar b/lib/json-smart-1.1.1.jar new file mode 100644 index 0000000..0c640aa Binary files /dev/null and b/lib/json-smart-1.1.1.jar differ diff --git a/lib/jsoup-1.7.2.jar b/lib/jsoup-1.7.2.jar new file mode 100644 index 0000000..1d9879f Binary files /dev/null and b/lib/jsoup-1.7.2.jar differ diff --git a/lib/junit-4.7.jar b/lib/junit-4.7.jar new file mode 100644 index 0000000..700ad69 Binary files /dev/null and b/lib/junit-4.7.jar differ diff --git a/lib/log4j-1.2.17.jar b/lib/log4j-1.2.17.jar new file mode 100644 index 0000000..1d425cf Binary files /dev/null and b/lib/log4j-1.2.17.jar differ diff --git a/lib/webmagic-core-0.4.2.jar b/lib/webmagic-core-0.4.2.jar new file mode 100644 index 0000000..0f07572 Binary files /dev/null and b/lib/webmagic-core-0.4.2.jar differ diff --git a/lib/webmagic-extension-0.4.2.jar b/lib/webmagic-extension-0.4.2.jar new file mode 100644 index 0000000..a19fa63 Binary files /dev/null and b/lib/webmagic-extension-0.4.2.jar differ diff --git a/lib/xsoup-0.1.0.jar b/lib/xsoup-0.1.0.jar new file mode 100644 index 0000000..926665c Binary files /dev/null and b/lib/xsoup-0.1.0.jar differ diff --git a/make.sh b/make.sh new file mode 100644 index 0000000..0a72237 --- /dev/null +++ b/make.sh @@ -0,0 +1,3 @@ +#!/bin/sh +mvn clean package +rsync -avz --delete ./webmagic-samples/target/lib/ ./lib/ diff --git a/webmagic-core/module_webmagic-core.xml b/webmagic-core/module_webmagic-core.xml new file mode 100644 index 0000000..88c4cfa --- /dev/null +++ b/webmagic-core/module_webmagic-core.xml @@ -0,0 +1,156 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/DianpingFtlDataScanner.java b/webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/DianpingFtlDataScanner.java new file mode 100644 index 0000000..7239e36 --- /dev/null +++ b/webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/DianpingFtlDataScanner.java @@ -0,0 +1,37 @@ +package us.codecraft.webmagic.model.samples; + +import us.codecraft.webmagic.Page; +import us.codecraft.webmagic.Site; +import us.codecraft.webmagic.model.AfterExtractor; +import us.codecraft.webmagic.model.OOSpider; +import us.codecraft.webmagic.model.annotation.ExtractBy; +import us.codecraft.webmagic.model.annotation.TargetUrl; + +import java.util.List; + +/** + * @author yihua.huang@dianping.com
+ * Date: 13-8-13
+ * Time: 上午10:13
+ */ +@TargetUrl("http://*.alpha.dp/*") +public class DianpingFtlDataScanner implements AfterExtractor { + + @ExtractBy(value = "(DP\\.data\\(\\{.*\\}\\));", type = ExtractBy.Type.Regex, notNull = true, multi = true) + private List data; + + public static void main(String[] args) { + OOSpider.create(Site.me().addStartUrl("http://w.alpha.dp/").setSleepTime(0), DianpingFtlDataScanner.class) + .thread(5).run(); + } + + @Override + public void afterProcess(Page page) { + if (data.size() > 1) { + System.err.println(page.getUrl()); + } + if (data.size() > 0 && data.get(0).length() > 100) { + System.err.println(page.getUrl()); + } + } +} diff --git a/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/DiaoyuwengProcessor.java b/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/DiaoyuwengProcessor.java new file mode 100644 index 0000000..3ceba0a --- /dev/null +++ b/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/DiaoyuwengProcessor.java @@ -0,0 +1,46 @@ +package us.codecraft.webmagic.samples; + +import us.codecraft.webmagic.Page; +import us.codecraft.webmagic.Site; +import us.codecraft.webmagic.Spider; +import us.codecraft.webmagic.processor.PageProcessor; +import us.codecraft.webmagic.selector.PlainText; + +import java.util.List; + +/** + * @author code4crafter@gmail.com
+ * Date: 13-4-21 + * Time: 下午8:08 + */ +public class DiaoyuwengProcessor implements PageProcessor { + + private Site site; + + @Override + public void process(Page page) { + List requests = page.getHtml().links().regex("(http://www\\.diaoyuweng\\.com/home\\.php\\?mod=space&uid=88304&do=thread&view=me&type=thread&order=dateline&from=space&page=\\d+)").all(); + page.addTargetRequests(requests); + requests = page.getHtml().links().regex("(http://www\\.diaoyuweng\\.com/thread-\\d+-1-1.html)").all(); + page.addTargetRequests(requests); + if (page.getUrl().toString().contains("thread")){ + page.putField("title", page.getHtml().xpath("//a[@id='thread_subject']")); + page.putField("content", page.getHtml().xpath("//div[@class='pcb']//tbody/tidyText()")); + page.putField("date",page.getHtml().regex("发表于 (\\d{4}-\\d+-\\d+ \\d+:\\d+:\\d+)")); + page.putField("id",new PlainText("1000"+page.getUrl().regex("http://www\\.diaoyuweng\\.com/thread-(\\d+)-1-1.html").toString())); + } + } + + @Override + public Site getSite() { + if (site==null){ + site= Site.me().setDomain("www.diaoyuweng.com").addStartUrl("http://www.diaoyuweng.com/home.php?mod=space&uid=88304&do=thread&view=me&type=thread&from=space"). + setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31").setCharset("GBK").setSleepTime(500); + } + return site; + } + + public static void main(String[] args) { + Spider.create(new DiaoyuwengProcessor()).run(); + } +} diff --git a/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/F58PageProcesser.java b/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/F58PageProcesser.java new file mode 100644 index 0000000..3d27be8 --- /dev/null +++ b/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/F58PageProcesser.java @@ -0,0 +1,34 @@ +package us.codecraft.webmagic.samples; + +import us.codecraft.webmagic.Site; +import us.codecraft.webmagic.Page; +import us.codecraft.webmagic.Spider; +import us.codecraft.webmagic.processor.PageProcessor; +import us.codecraft.webmagic.scheduler.RedisScheduler; + +import java.util.List; + +/** + * @author code4crafter@gmail.com
+ * Date: 13-4-21 + * Time: 下午1:48 + */ +public class F58PageProcesser implements PageProcessor { + + @Override + public void process(Page page) { + List strings = page.getHtml().links().regex(".*/yewu/.*").all(); + page.addTargetRequests(strings); + page.putField("title",page.getHtml().regex("(.*)")); + page.putField("body",page.getHtml().xpath("//dd")); + } + + @Override + public Site getSite() { + return Site.me().setDomain("sh.58.com").addStartUrl("http://sh1.51a8.com/").setCycleRetryTimes(2); //To change body of implemented methods use File | Settings | File Templates. + } + + public static void main(String[] args) { + Spider.create(new F58PageProcesser()).setScheduler(new RedisScheduler("localhost")).run(); + } +} diff --git a/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/HuxiuProcessor.java b/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/HuxiuProcessor.java index 7cb7be2..000cb99 100644 --- a/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/HuxiuProcessor.java +++ b/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/HuxiuProcessor.java @@ -27,4 +27,5 @@ public class HuxiuProcessor implements PageProcessor { public static void main(String[] args) { Spider.create(new HuxiuProcessor()).run(); } + } diff --git a/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/KaichibaProcessor.java b/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/KaichibaProcessor.java new file mode 100644 index 0000000..0ab6c64 --- /dev/null +++ b/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/KaichibaProcessor.java @@ -0,0 +1,32 @@ +package us.codecraft.webmagic.samples; + +import us.codecraft.webmagic.Page; +import us.codecraft.webmagic.Site; +import us.codecraft.webmagic.Spider; +import us.codecraft.webmagic.processor.PageProcessor; + +/** + * @author code4crafter@gmail.com
+ * Date: 13-5-20 + * Time: 下午5:31 + */ +public class KaichibaProcessor implements PageProcessor { + @Override + public void process(Page page) { + //http://progressdaily.diandian.com/post/2013-01-24/40046867275 + int i = Integer.valueOf(page.getUrl().regex("shop/(\\d+)").toString()) + 1; + page.addTargetRequest("http://kaichiba.com/shop/" + i); + page.putField("title",page.getHtml().xpath("//Title")); + page.putField("items", page.getHtml().xpath("//li[@class=\"foodTitle\"]").replace("^\\s+", "").replace("\\s+$", "").replace(".*?", "")); + } + + @Override + public Site getSite() { + return Site.me().setDomain("kaichiba.com").addStartUrl("http://kaichiba.com/shop/41725781").setCharset("utf-8"). + setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31"); + } + + public static void main(String[] args) { + Spider.create(new KaichibaProcessor()).run(); + } +} diff --git a/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/MeicanProcessor.java b/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/MeicanProcessor.java new file mode 100644 index 0000000..bfa347d --- /dev/null +++ b/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/MeicanProcessor.java @@ -0,0 +1,38 @@ +package us.codecraft.webmagic.samples; + +import us.codecraft.webmagic.Page; +import us.codecraft.webmagic.Site; +import us.codecraft.webmagic.Spider; +import us.codecraft.webmagic.processor.PageProcessor; + +import java.util.List; + +/** + * @author code4crafter@gmail.com
+ * Date: 13-5-20 + * Time: 下午5:31 + */ +public class MeicanProcessor implements PageProcessor { + @Override + public void process(Page page) { + //http://progressdaily.diandian.com/post/2013-01-24/40046867275 + List requests = page.getHtml().xpath("//a[@class=\"area_link flat_btn\"]/@href").all(); + if (requests.size() > 2) { + requests = requests.subList(0, 2); + } + page.addTargetRequests(requests); + page.addTargetRequests(page.getHtml().links().regex("(.*/restaurant/[^#]+)").all()); + page.putField("items", page.getHtml().xpath("//ul[@class=\"dishes menu_dishes\"]/li/span[@class=\"name\"]/text()")); + page.putField("prices", page.getHtml().xpath("//ul[@class=\"dishes menu_dishes\"]/li/span[@class=\"price_outer\"]/span[@class=\"price\"]/text()")); + } + + @Override + public Site getSite() { + return Site.me().setDomain("meican.com").addStartUrl("http://www.meican.com/shanghai/districts").setCharset("utf-8"). + setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31"); + } + + public static void main(String[] args) { + Spider.create(new MeicanProcessor()).run(); + } +}