diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/BaiduBaikePageProcesser.java b/webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/BaiduBaikePageProcessor.java similarity index 93% rename from webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/BaiduBaikePageProcesser.java rename to webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/BaiduBaikePageProcessor.java index 866d090..1827e06 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/BaiduBaikePageProcesser.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/BaiduBaikePageProcessor.java @@ -13,7 +13,7 @@ import java.util.List; * @author code4crafter@gmail.com
* @since 0.4.0 */ -public class BaiduBaikePageProcesser implements PageProcessor { +public class BaiduBaikePageProcessor implements PageProcessor { private Site site = Site.me()//.setHttpProxy(new HttpHost("127.0.0.1",8888)) .setRetryTimes(3).setSleepTime(1000).setUseGzip(true); @@ -31,7 +31,7 @@ public class BaiduBaikePageProcesser implements PageProcessor { public static void main(String[] args) { //single download - Spider spider = Spider.create(new BaiduBaikePageProcesser()).thread(2); + Spider spider = Spider.create(new BaiduBaikePageProcessor()).thread(2); String urlTemplate = "http://baike.baidu.com/search/word?word=%s&pic=1&sug=1&enc=utf8"; ResultItems resultItems = spider.get(String.format(urlTemplate, "水力发电")); System.out.println(resultItems); diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/GithubRepoPageProcesser.java b/webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/GithubRepoPageProcessor.java similarity index 89% rename from webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/GithubRepoPageProcesser.java rename to webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/GithubRepoPageProcessor.java index 47f904f..179bad4 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/GithubRepoPageProcesser.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/GithubRepoPageProcessor.java @@ -9,7 +9,7 @@ import us.codecraft.webmagic.processor.PageProcessor; * @author code4crafter@gmail.com
* @since 0.3.2 */ -public class GithubRepoPageProcesser implements PageProcessor { +public class GithubRepoPageProcessor implements PageProcessor { private Site site = Site.me().setRetryTimes(3).setSleepTime(100); @@ -31,6 +31,6 @@ public class GithubRepoPageProcesser implements PageProcessor { } public static void main(String[] args) { - Spider.create(new GithubRepoPageProcesser()).addUrl("https://github.com/code4craft").thread(5).run(); + Spider.create(new GithubRepoPageProcessor()).addUrl("https://github.com/code4craft").thread(5).run(); } } diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/OschinaBlogPageProcesser.java b/webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/OschinaBlogPageProcessor.java similarity index 90% rename from webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/OschinaBlogPageProcesser.java rename to webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/OschinaBlogPageProcessor.java index a59f1e9..aac0ac1 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/OschinaBlogPageProcesser.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/OschinaBlogPageProcessor.java @@ -10,7 +10,7 @@ import java.util.List; /** * @author code4crafter@gmail.com
*/ -public class OschinaBlogPageProcesser implements PageProcessor { +public class OschinaBlogPageProcessor implements PageProcessor { private Site site = Site.me().setDomain("my.oschina.net"); @@ -34,6 +34,6 @@ public class OschinaBlogPageProcesser implements PageProcessor { } public static void main(String[] args) { - Spider.create(new OschinaBlogPageProcesser()).addUrl("http://my.oschina.net/flashsword/blog").thread(2).run(); + Spider.create(new OschinaBlogPageProcessor()).addUrl("http://my.oschina.net/flashsword/blog").thread(2).run(); } } diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/configurable/PropertyLoader.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/configurable/PropertyLoader.java new file mode 100644 index 0000000..28521b8 --- /dev/null +++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/configurable/PropertyLoader.java @@ -0,0 +1,15 @@ +package us.codecraft.webmagic.configurable; + +import us.codecraft.webmagic.processor.PageProcessor; + +import java.util.Map; + +/** + * @author yihua.huang@dianping.com + */ +public interface PropertyLoader { + + PropertyLoader clazz(Class clazz); + + T load(Map properties); +} diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/example/ConfigurableBlogPageProcesser.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/example/ConfigurableBlogPageProcessor.java similarity index 91% rename from webmagic-extension/src/main/java/us/codecraft/webmagic/example/ConfigurableBlogPageProcesser.java rename to webmagic-extension/src/main/java/us/codecraft/webmagic/example/ConfigurableBlogPageProcessor.java index f5992a4..28d3ab0 100644 --- a/webmagic-extension/src/main/java/us/codecraft/webmagic/example/ConfigurableBlogPageProcesser.java +++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/example/ConfigurableBlogPageProcessor.java @@ -10,7 +10,7 @@ import us.codecraft.webmagic.processor.PageProcessor; /** * @author code4crafter@gmail.com
*/ -public class ConfigurableBlogPageProcesser implements PageProcessor { +public class ConfigurableBlogPageProcessor implements PageProcessor { private Site site = Site.me().setDomain("my.oschina.net"); @@ -46,6 +46,6 @@ public class ConfigurableBlogPageProcesser implements PageProcessor { } public static void main(String[] args) { - Spider.create(new ConfigurableBlogPageProcesser()).addUrl("http://my.oschina.net/flashsword/blog").thread(2).run(); + Spider.create(new ConfigurableBlogPageProcessor()).addUrl("http://my.oschina.net/flashsword/blog").thread(2).run(); } } diff --git a/webmagic-extension/src/test/java/us/codecraft/webmagic/downloader/MockGithubDownloader.java b/webmagic-extension/src/test/java/us/codecraft/webmagic/downloader/MockGithubDownloader.java index 49774f1..6baee72 100644 --- a/webmagic-extension/src/test/java/us/codecraft/webmagic/downloader/MockGithubDownloader.java +++ b/webmagic-extension/src/test/java/us/codecraft/webmagic/downloader/MockGithubDownloader.java @@ -769,7 +769,7 @@ public class MockGithubDownloader implements Downloader{ "\n" + "

Write a class implements PageProcessor：

\n" + "\n" + - "
    public class OschinaBlogPageProcesser implements PageProcessor {\n" +
+            "
    public class OschinaBlogPageProcessor implements PageProcessor {\n" +
             "\n" +
             "        private Site site = Site.me().setDomain(\"my.oschina.net\")\n" +
             "           .addStartUrl(\"http://my.oschina.net/flashsword/blog\");\n" +
@@ -790,7 +790,7 @@ public class MockGithubDownloader implements Downloader{
             "        }\n" +
             "\n" +
             "        public static void main(String[] args) {\n" +
-            "            Spider.create(new OschinaBlogPageProcesser())\n" +
+            "            Spider.create(new OschinaBlogPageProcessor())\n" +
             "                 .pipeline(new ConsolePipeline()).run();\n" +
             "        }\n" +
             "    }\n" +