Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix compile error for example change

master
yihua.huang 2016-12-18 08:32:14 +08:00
parent a3ee9e3d08
commit a960a39c44
6 changed files with 19 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
pom.xml
View File

@ -70,6 +70,11 @@
<artifactId>httpclient</artifactId>
<version>4.5.2</version>
</dependency>
<dependency>
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId>
<version>2.2.0</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>

6
webmagic-core/pom.xml
View File

@ -74,6 +74,12 @@
<groupId>com.jayway.jsonpath</groupId>
<artifactId>json-path</artifactId>
<version>0.8.1</version>
<exclusions>
<exclusion>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>

2
webmagic-extension/pom.xml
View File

@ -13,7 +13,7 @@
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>2.0.0</version>
<version>2.9.0</version>
</dependency>
<dependency>
<groupId>us.codecraft</groupId>

6
webmagic-extension/src/main/java/us/codecraft/webmagic/example/MonitorExample.java
View File

@ -13,14 +13,14 @@ public class MonitorExample {
public static void main(String[] args) throws Exception {
Spider zhihuPageProcessor = Spider.create(new ZhihuPageProcessor())
Spider zhihuSpider = Spider.create(new ZhihuPageProcessor())
.addUrl("http://my.oschina.net/flashsword/blog");
Spider githubSpider = Spider.create(new GithubRepoPageProcessor())
.addUrl("https://github.com/code4craft");
SpiderMonitor.instance().register(zhihuPageProcessor);
SpiderMonitor.instance().register(zhihuSpider);
SpiderMonitor.instance().register(githubSpider);
zhihuPageProcessor.start();
zhihuSpider.start();
githubSpider.start();
}
}

3
webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/RedisPriorityScheduler.java
View File

@ -1,9 +1,8 @@
package us.codecraft.webmagic.scheduler;
import com.alibaba.fastjson.JSON;
import com.sun.org.apache.regexp.internal.RE;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.StringUtils;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import us.codecraft.webmagic.Request;

6
webmagic-extension/src/test/java/us/codecraft/webmagic/monitor/SpiderMonitorTest.java
View File

@ -3,7 +3,7 @@ package us.codecraft.webmagic.monitor;
import org.junit.Test;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.example.GithubRepoPageProcessor;
import us.codecraft.webmagic.processor.example.OschinaBlogPageProcessor;
import us.codecraft.webmagic.processor.example.ZhihuPageProcessor;
/**
* @author code4crafer@gmail.com
@ -20,12 +20,12 @@ public class SpiderMonitorTest {
}
};
Spider oschinaSpider = Spider.create(new OschinaBlogPageProcessor())
Spider zhihuSpider = Spider.create(new ZhihuPageProcessor())
.addUrl("http://my.oschina.net/flashsword/blog").thread(2);
Spider githubSpider = Spider.create(new GithubRepoPageProcessor())
.addUrl("https://github.com/code4craft");
spiderMonitor.register(oschinaSpider, githubSpider);
spiderMonitor.register(zhihuSpider, githubSpider);
}
}