![](\"https://2.gravatar.com/avatar/4ce9123a05ae222d71d2857316cbe699?d=https%3A%2F%2Fidenticons.github.com%2F19ef9dc10e8399f81a8944a399812d77.png&s=140\")
code4craft\n" +
+ " ![\"Octocat-spinner-32\"](\"https://github.global.ssl.fastly.net/images/spinners/octocat-spinner-32.gif\")
\n" +
+ " ![\"Octocat-spinner-32-eaf2f5\"](\"https://github.global.ssl.fastly.net/images/spinners/octocat-spinner-32-EAF2F5.gif\")
\n" + + "webmagic
\n" + + "\n" + + "\n" + + "\n" + + "![\"Build](\"https://travis-ci.org/code4craft/webmagic.png?branch=master\")
\n" + + "\n" + + "\n" + + "A scalable crawler framework. It covers the whole lifecycle of crawler: downloading, url management, content extraction and persistent. It can simplify the development of a specific crawler.
\n" + + "
\n" + + "Features:
\n" + + "\n" + + "- \n" +
+ "
- Simple core with high flexibility. \n" + + "
- Simple API for html extracting. \n" + + "
- Annotation with POJO to customize a crawler, no configuration. \n" + + "
- Multi-thread and Distribution support. \n" + + "
- Easy to be integrated. \n" + + "
\n" + + "Install:
\n" + + "\n" + + "Add dependencies to your pom.xml:
\n" + + "\n" + + " <dependency>\n" +
+ " <groupId>us.codecraft</groupId>\n" +
+ " <artifactId>webmagic-core</artifactId>\n" +
+ " <version>0.3.0</version>\n" +
+ " </dependency>\n" +
+ " <dependency>\n" +
+ " <groupId>us.codecraft</groupId>\n" +
+ " <artifactId>webmagic-extension</artifactId>\n" +
+ " <version>0.3.0</version>\n" +
+ " </dependency>\n" +
+ "\n" + + "Get Started:
\n" + + "\n" + + "\n" + + "First crawler:
\n" + + "\n" + + "Write a class implements PageProcessor：
\n" + + "\n" + + " public class OschinaBlogPageProcesser implements PageProcessor {\n" +
+ "\n" +
+ " private Site site = Site.me().setDomain(\"my.oschina.net\")\n" +
+ " .addStartUrl(\"http://my.oschina.net/flashsword/blog\");\n" +
+ "\n" +
+ " @Override\n" +
+ " public void process(Page page) {\n" +
+ " List<String> links = page.getHtml().links().regex(\"http://my\\\\.oschina\\\\.net/flashsword/blog/\\\\d+\").all();\n" +
+ " page.addTargetRequests(links);\n" +
+ " page.putField(\"title\", page.getHtml().xpath(\"//div[@class='BlogEntity']/div[@class='BlogTitle']/h1\").toString());\n" +
+ " page.putField(\"content\", page.getHtml().$(\"div.content\").toString());\n" +
+ " page.putField(\"tags\",page.getHtml().xpath(\"//div[@class='BlogTags']/a/text()\").all());\n" +
+ " }\n" +
+ "\n" +
+ " @Override\n" +
+ " public Site getSite() {\n" +
+ " return site;\n" +
+ "\n" +
+ " }\n" +
+ "\n" +
+ " public static void main(String[] args) {\n" +
+ " Spider.create(new OschinaBlogPageProcesser())\n" +
+ " .pipeline(new ConsolePipeline()).run();\n" +
+ " }\n" +
+ " }\n" +
+ "- \n" +
+ "
- \n" +
+ "
\n" + + "\n" + + "page.addTargetRequests(links)Add urls for crawling.
\n" + + " \n" +
+ "
You can also use annotation way:
\n" + + "\n" + + " @TargetUrl(\"http://my.oschina.net/flashsword/blog/\\\\d+\")\n" +
+ " public class OschinaBlog {\n" +
+ "\n" +
+ " @ExtractBy(\"//title\")\n" +
+ " private String title;\n" +
+ "\n" +
+ " @ExtractBy(value = \"div.BlogContent\",type = ExtractBy.Type.Css)\n" +
+ " private String content;\n" +
+ "\n" +
+ " @ExtractBy(value = \"//div[@class='BlogTags']/a/text()\", multi = true)\n" +
+ " private List<String> tags;\n" +
+ "\n" +
+ " public static void main(String[] args) {\n" +
+ " OOSpider.create(\n" +
+ " Site.me().addStartUrl(\"http://my.oschina.net/flashsword/blog\"),\n" +
+ " new ConsolePageModelPipeline(), OschinaBlog.class).run();\n" +
+ " }\n" +
+ " }\n" +
+ "\n" + + "Docs and samples:
\n" + + "\n" + + "The architecture of webmagic (refered to Scrapy)
\n" + + "\n" + + "![\"image\"](\"https://github-camo.global.ssl.fastly.net/06cb8227231a6adf6d2a57b14b60a25389a25fe9/687474703a2f2f636f64653463726166742e6769746875622e696f2f696d616765732f706f7374732f7765626d616769632e706e67\")
Javadocs: http://code4craft.github.io/webmagic/docs/en/
\n" + + "\n" + + "There are some samples in webmagic-samples package.
\n" + + "Lisence:
\n" + + "\n" + + "Lisenced under Apache 2.0 lisence
\n" + + "\n" + + "\n" + + "Thanks:
\n" + + "\n" + + "To write webmagic, I refered to the projects below :
\n" + + "\n" + + "- \n" +
+ "
- \n" +
+ "
Scrapy
\n" + + "\n" + + "A crawler framework in Python.
\n" + + "\n" + + "\n" + + " \n" +
+ " - \n" +
+ "
Spiderman
\n" + + "\n" + + "Another crawler framework in Java.
\n" + + "\n" + + "\n" + + " \n" +
+ "