diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/example/AppStore.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/example/AppStore.java index 504e6d2..fcc937b 100644 --- a/webmagic-extension/src/main/java/us/codecraft/webmagic/example/AppStore.java +++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/example/AppStore.java @@ -3,11 +3,15 @@ package us.codecraft.webmagic.example; import us.codecraft.webmagic.Site; import us.codecraft.webmagic.model.OOSpider; import us.codecraft.webmagic.model.annotation.ExtractBy; +import us.codecraft.webmagic.utils.Experimental; + +import java.util.List; /** * @author code4crafter@gmail.com * @since 0.4.1 */ +@Experimental public class AppStore { @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$..trackName") @@ -16,9 +20,17 @@ public class AppStore { @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$..description") private String description; + @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$..userRatingCount") + private int userRatingCount; + + @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$..screenshotUrls",multi = true) + private List screenshotUrls; + public static void main(String[] args) { AppStore appStore = OOSpider.create(Site.me(), AppStore.class).get("http://itunes.apple.com/lookup?id=653350791&country=cn&entity=software"); System.out.println(appStore.trackName); System.out.println(appStore.description); + System.out.println(appStore.userRatingCount); + System.out.println(appStore.screenshotUrls); } } diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/example/GithubRepoApi.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/example/GithubRepoApi.java new file mode 100644 index 0000000..deacde7 --- /dev/null +++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/example/GithubRepoApi.java @@ -0,0 +1,70 @@ +package us.codecraft.webmagic.example; + +import us.codecraft.webmagic.Site; +import us.codecraft.webmagic.model.ConsolePageModelPipeline; +import us.codecraft.webmagic.model.HasKey; +import us.codecraft.webmagic.model.OOSpider; +import us.codecraft.webmagic.model.annotation.ExtractBy; +import us.codecraft.webmagic.model.annotation.ExtractByUrl; + +import java.util.List; + +/** + * @author code4crafter@gmail.com
+ * @since 0.4.1 + */ +public class GithubRepoApi implements HasKey { + + @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$.name") + private String name; + + @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$..owner.login") + private String author; + + @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$.language",multi = true) + private List language; + + @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$.stargazers_count") + private int star; + + @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$.forks_count") + private int fork; + + @ExtractByUrl + private String url; + + public static void main(String[] args) { + OOSpider.create(Site.me().setSleepTime(100) + , new ConsolePageModelPipeline(), GithubRepoApi.class) + .addUrl("https://api.github.com/repos/code4craft/webmagic").run(); + } + + @Override + public String key() { + return author + ":" + name; + } + + public String getName() { + return name; + } + + public String getAuthor() { + return author; + } + + public List getLanguage() { + return language; + } + + public String getUrl() { + return url; + } + + public int getStar() { + return star; + } + + public int getFork() { + return fork; + } +} diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/selector/JsonPathSelector.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/selector/JsonPathSelector.java index 2270384..781669f 100644 --- a/webmagic-extension/src/main/java/us/codecraft/webmagic/selector/JsonPathSelector.java +++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/selector/JsonPathSelector.java @@ -1,6 +1,7 @@ package us.codecraft.webmagic.selector; import com.jayway.jsonpath.JsonPath; +import us.codecraft.webmagic.utils.Experimental; import java.util.ArrayList; import java.util.List; @@ -12,6 +13,7 @@ import java.util.List; * @author code4crafter@gmail.com
* @since 0.2.1 */ +@Experimental public class JsonPathSelector implements Selector { private String jsonPathStr;