diff --git a/pom.xml b/pom.xml
index 04f346e..692fc0d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -70,6 +70,11 @@
httpclient
4.5.2
+
+ com.jayway.jsonpath
+ json-path
+ 2.2.0
+
com.google.guava
guava
diff --git a/webmagic-core/pom.xml b/webmagic-core/pom.xml
index fa8326d..d73c9f8 100644
--- a/webmagic-core/pom.xml
+++ b/webmagic-core/pom.xml
@@ -74,6 +74,12 @@
com.jayway.jsonpath
json-path
0.8.1
+
+
+ commons-lang
+ commons-lang
+
+
diff --git a/webmagic-extension/pom.xml b/webmagic-extension/pom.xml
index 993a6f4..054342f 100644
--- a/webmagic-extension/pom.xml
+++ b/webmagic-extension/pom.xml
@@ -13,7 +13,7 @@
redis.clients
jedis
- 2.0.0
+ 2.9.0
us.codecraft
diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/example/MonitorExample.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/example/MonitorExample.java
index 7e4efee..b759c03 100644
--- a/webmagic-extension/src/main/java/us/codecraft/webmagic/example/MonitorExample.java
+++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/example/MonitorExample.java
@@ -13,14 +13,14 @@ public class MonitorExample {
public static void main(String[] args) throws Exception {
- Spider zhihuPageProcessor = Spider.create(new ZhihuPageProcessor())
+ Spider zhihuSpider = Spider.create(new ZhihuPageProcessor())
.addUrl("http://my.oschina.net/flashsword/blog");
Spider githubSpider = Spider.create(new GithubRepoPageProcessor())
.addUrl("https://github.com/code4craft");
- SpiderMonitor.instance().register(zhihuPageProcessor);
+ SpiderMonitor.instance().register(zhihuSpider);
SpiderMonitor.instance().register(githubSpider);
- zhihuPageProcessor.start();
+ zhihuSpider.start();
githubSpider.start();
}
}
diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/RedisPriorityScheduler.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/RedisPriorityScheduler.java
index e95ca92..540574a 100644
--- a/webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/RedisPriorityScheduler.java
+++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/RedisPriorityScheduler.java
@@ -1,9 +1,8 @@
package us.codecraft.webmagic.scheduler;
import com.alibaba.fastjson.JSON;
-import com.sun.org.apache.regexp.internal.RE;
import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import us.codecraft.webmagic.Request;
diff --git a/webmagic-extension/src/test/java/us/codecraft/webmagic/monitor/SpiderMonitorTest.java b/webmagic-extension/src/test/java/us/codecraft/webmagic/monitor/SpiderMonitorTest.java
index 3baa0d6..6144da7 100644
--- a/webmagic-extension/src/test/java/us/codecraft/webmagic/monitor/SpiderMonitorTest.java
+++ b/webmagic-extension/src/test/java/us/codecraft/webmagic/monitor/SpiderMonitorTest.java
@@ -3,7 +3,7 @@ package us.codecraft.webmagic.monitor;
import org.junit.Test;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.example.GithubRepoPageProcessor;
-import us.codecraft.webmagic.processor.example.OschinaBlogPageProcessor;
+import us.codecraft.webmagic.processor.example.ZhihuPageProcessor;
/**
* @author code4crafer@gmail.com
@@ -20,12 +20,12 @@ public class SpiderMonitorTest {
}
};
- Spider oschinaSpider = Spider.create(new OschinaBlogPageProcessor())
+ Spider zhihuSpider = Spider.create(new ZhihuPageProcessor())
.addUrl("http://my.oschina.net/flashsword/blog").thread(2);
Spider githubSpider = Spider.create(new GithubRepoPageProcessor())
.addUrl("https://github.com/code4craft");
- spiderMonitor.register(oschinaSpider, githubSpider);
+ spiderMonitor.register(zhihuSpider, githubSpider);
}
}