Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix compile error for example change

master
yihua.huang 2016-12-18 08:32:14 +08:00
parent a3ee9e3d08
commit a960a39c44
6 changed files with 19 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
pom.xml
View File

@ -70,6 +70,11 @@
<artifactId>httpclient</artifactId> <artifactId>httpclient</artifactId>
<version>4.5.2</version> <version>4.5.2</version>
</dependency> </dependency>
<dependency>
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId>
<version>2.2.0</version>
</dependency>
<dependency> <dependency>
<groupId>com.google.guava</groupId> <groupId>com.google.guava</groupId>
<artifactId>guava</artifactId> <artifactId>guava</artifactId>

6
webmagic-core/pom.xml
View File

@ -74,6 +74,12 @@
<groupId>com.jayway.jsonpath</groupId> <groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId> <artifactId>json-path</artifactId>
<version>0.8.1</version> <version>0.8.1</version>
<exclusions>
<exclusion>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>

2
webmagic-extension/pom.xml
View File

@ -13,7 +13,7 @@
<dependency> <dependency>
<groupId>redis.clients</groupId> <groupId>redis.clients</groupId>
<artifactId>jedis</artifactId> <artifactId>jedis</artifactId>
<version>2.0.0</version> <version>2.9.0</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>us.codecraft</groupId> <groupId>us.codecraft</groupId>

6
webmagic-extension/src/main/java/us/codecraft/webmagic/example/MonitorExample.java
View File

@ -13,14 +13,14 @@ public class MonitorExample {
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
Spider zhihuPageProcessor = Spider.create(new ZhihuPageProcessor()) Spider zhihuSpider = Spider.create(new ZhihuPageProcessor())
.addUrl("http://my.oschina.net/flashsword/blog"); .addUrl("http://my.oschina.net/flashsword/blog");
Spider githubSpider = Spider.create(new GithubRepoPageProcessor()) Spider githubSpider = Spider.create(new GithubRepoPageProcessor())
.addUrl("https://github.com/code4craft"); .addUrl("https://github.com/code4craft");
SpiderMonitor.instance().register(zhihuPageProcessor); SpiderMonitor.instance().register(zhihuSpider);
SpiderMonitor.instance().register(githubSpider); SpiderMonitor.instance().register(githubSpider);
zhihuPageProcessor.start(); zhihuSpider.start();
githubSpider.start(); githubSpider.start();
} }
} }

3
webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/RedisPriorityScheduler.java
View File

@ -1,9 +1,8 @@
package us.codecraft.webmagic.scheduler; package us.codecraft.webmagic.scheduler;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import com.sun.org.apache.regexp.internal.RE;
import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang3.StringUtils;
import redis.clients.jedis.Jedis; import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPool;
import us.codecraft.webmagic.Request; import us.codecraft.webmagic.Request;

6
webmagic-extension/src/test/java/us/codecraft/webmagic/monitor/SpiderMonitorTest.java
View File

@ -3,7 +3,7 @@ package us.codecraft.webmagic.monitor;
import org.junit.Test; import org.junit.Test;
import us.codecraft.webmagic.Spider; import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.example.GithubRepoPageProcessor; import us.codecraft.webmagic.processor.example.GithubRepoPageProcessor;
import us.codecraft.webmagic.processor.example.OschinaBlogPageProcessor; import us.codecraft.webmagic.processor.example.ZhihuPageProcessor;
/** /**
* @author code4crafer@gmail.com * @author code4crafer@gmail.com
@ -20,12 +20,12 @@ public class SpiderMonitorTest {
} }
}; };
Spider oschinaSpider = Spider.create(new OschinaBlogPageProcessor()) Spider zhihuSpider = Spider.create(new ZhihuPageProcessor())
.addUrl("http://my.oschina.net/flashsword/blog").thread(2); .addUrl("http://my.oschina.net/flashsword/blog").thread(2);
Spider githubSpider = Spider.create(new GithubRepoPageProcessor()) Spider githubSpider = Spider.create(new GithubRepoPageProcessor())
.addUrl("https://github.com/code4craft"); .addUrl("https://github.com/code4craft");
spiderMonitor.register(oschinaSpider, githubSpider); spiderMonitor.register(zhihuSpider, githubSpider);
} }
} }