update AngularJSProcessor

webmagic-core/src/main/java/us/codecraft/webmagic/selector/JsonPathSelector.java

@@ -1,7 +1,6 @@
 package us.codecraft.webmagic.selector;
 
 import com.jayway.jsonpath.JsonPath;
-import us.codecraft.webmagic.utils.Experimental;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -13,7 +12,6 @@ import java.util.List;
  * @author code4crafter@gmail.com <br>
  * @since 0.2.1
  */
-@Experimental
 public class JsonPathSelector implements Selector {
 
     private String jsonPathStr;

webmagic-core/src/test/java/us/codecraft/webmagic/selector/JsonTest.java

@@ -1,6 +1,7 @@
 package us.codecraft.webmagic.selector;
 
 import org.junit.Test;
+import us.codecraft.webmagic.Page;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -16,5 +17,9 @@ public class JsonTest {
     public void testRemovePadding() throws Exception {
         String name = new Json(text).removePadding("callback").jsonPath("$.name").get();
         assertThat(name).isEqualTo("json");
+        Page page = null;
+
+        page.getJson().jsonPath("$.name").get();
+        page.getJson().removePadding("callback").jsonPath("$.name").get();
     }
 }

webmagic-samples/src/main/java/us/codecraft/webmagic/samples/AngularJSProcessor.java

@@ -1,21 +1,43 @@
 package us.codecraft.webmagic.samples;
 
+import org.apache.commons.collections.CollectionUtils;
 import us.codecraft.webmagic.Page;
 import us.codecraft.webmagic.Site;
 import us.codecraft.webmagic.processor.PageProcessor;
+import us.codecraft.webmagic.selector.JsonPathSelector;
+
+import java.util.List;
 
 /**
- * @author yihua.huang@dianping.com
+ * @author code4crafter@gmail.com
+ * @since 0.5.0
  */
 public class AngularJSProcessor implements PageProcessor {
 
+    private Site site = Site.me();
+
+    private static final String ARITICALE_URL = "http://angularjs\\.cn/api/article/\\w+";
+
+    private static final String LIST_URL = "http://angularjs\\.cn/api/article/latest.*";
+
     @Override
     public void process(Page page) {
+        if (page.getUrl().regex(LIST_URL).match()) {
+            List<String> ids = new JsonPathSelector("$.data._id").selectList(page.getRawText());
+            if (CollectionUtils.isNotEmpty(ids)) {
+                for (String id : ids) {
+                    page.addTargetRequest("http://angularjs\\.cn/api/article/" + id);
+                }
+            }
+        } else {
+            page.putField("title", new JsonPathSelector("$.title").select(page.getRawText()));
+            page.putField("content", new JsonPathSelector("$.content").select(page.getRawText()));
+        }
 
     }
 
     @Override
     public Site getSite() {
-        return null;
+        return site;
     }
 }