webMagic-Mobai
/
magic-Dependency
mirror of https://gitea.qinmian.online/Mobai/magic-Dependency.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

#46 Downloader thread hang up when timeout

master
yihua.huang 2013-12-03 09:59:54 +08:00
parent 486d9d276f
commit 6a828e923c
4 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
user-manual.md
View File

@ -27,12 +27,12 @@ webmagic使用maven管理依赖，在项目中添加对应的依赖即可使用w
<dependency> <dependency>
<groupId>us.codecraft</groupId> <groupId>us.codecraft</groupId>
<artifactId>webmagic-core</artifactId> <artifactId>webmagic-core</artifactId>
<version>0.4.0</version> <version>0.4.1</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>us.codecraft</groupId> <groupId>us.codecraft</groupId>
<artifactId>webmagic-extension</artifactId> <artifactId>webmagic-extension</artifactId>
<version>0.4.0</version> <version>0.4.1</version>
</dependency> </dependency>
#### 项目结构 #### 项目结构

1
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -105,6 +105,7 @@ public class HttpClientDownloader implements Downloader {
} }
RequestConfig.Builder requestConfigBuilder = RequestConfig.custom() RequestConfig.Builder requestConfigBuilder = RequestConfig.custom()
.setConnectionRequestTimeout(site.getTimeOut()) .setConnectionRequestTimeout(site.getTimeOut())
.setSocketTimeout(site.getTimeOut())
.setConnectTimeout(site.getTimeOut()) .setConnectTimeout(site.getTimeOut())
.setCookieSpec(CookieSpecs.BEST_MATCH); .setCookieSpec(CookieSpecs.BEST_MATCH);
if (site != null && site.getHttpProxy() != null) { if (site != null && site.getHttpProxy() != null) {

1
webmagic-scripts/src/main/java/us/codecraft/webmagic/scripts/ScriptConsole.java
View File

@ -93,6 +93,7 @@ public class ScriptConsole {
.language(params.getLanguage()).scriptFromFile(params.getScriptFileName()).thread(params.getThread()).build(); .language(params.getLanguage()).scriptFromFile(params.getScriptFileName()).thread(params.getThread()).build();
pageProcessor.getSite().setSleepTime(params.getSleepTime()); pageProcessor.getSite().setSleepTime(params.getSleepTime());
pageProcessor.getSite().setAcceptStatCode(Sets.<Integer>newHashSet(200, 404, 500)); pageProcessor.getSite().setAcceptStatCode(Sets.<Integer>newHashSet(200, 404, 500));
pageProcessor.getSite().setUserAgent("Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-US) AppleWebKit/532.5 (KHTML, like Gecko) Chrome/4.0.249.0 Safari/532.5");
Spider spider = Spider.create(pageProcessor).thread(params.getThread()); Spider spider = Spider.create(pageProcessor).thread(params.getThread());
spider.clearPipeline().addPipeline(new Pipeline() { spider.clearPipeline().addPipeline(new Pipeline() {
@Override @Override

1
webmagic-scripts/src/main/java/us/codecraft/webmagic/scripts/ScriptEnginePool.java
View File

@ -34,6 +34,7 @@ public class ScriptEnginePool {
public void release(ScriptEngine scriptEngine){ public void release(ScriptEngine scriptEngine){
scriptEngines.add(scriptEngine); scriptEngines.add(scriptEngine);
availableCount.incrementAndGet();
} }
} }