diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
index b6f0034..1bee564 100644
--- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
+++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
@@ -34,6 +34,8 @@ public class HttpClientDownloader implements Downloader {
private Logger logger = Logger.getLogger(getClass());
+ private HttpClientPool httpClientPool;
+
private int poolSize = 1;
/**
@@ -58,6 +60,13 @@ public class HttpClientDownloader implements Downloader {
return (Html) page.getHtml();
}
+ private HttpClientPool getHttpClientPool(){
+ if (httpClientPool==null){
+ httpClientPool = new HttpClientPool(poolSize);
+ }
+ return httpClientPool;
+ }
+
@Override
public Page download(Request request, Task task) {
Site site = null;
@@ -78,7 +87,7 @@ public class HttpClientDownloader implements Downloader {
acceptStatCode.add(200);
}
logger.info("downloading page " + request.getUrl());
- HttpClient httpClient = HttpClientPool.getInstance(poolSize).getClient(site);
+ HttpClient httpClient = getHttpClientPool().getClient(site);
try {
HttpGet httpGet = new HttpGet(request.getUrl());
if (headers!=null){
@@ -150,6 +159,7 @@ public class HttpClientDownloader implements Downloader {
@Override
public void setThread(int thread) {
poolSize = thread;
+ httpClientPool = new HttpClientPool(thread);
}
private void handleGzip(HttpResponse httpResponse) {
diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientPool.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientPool.java
index f3b72b6..c882836 100644
--- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientPool.java
+++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientPool.java
@@ -24,23 +24,19 @@ import java.util.Map;
*/
public class HttpClientPool {
- public static volatile HttpClientPool INSTANCE;
-
- public static HttpClientPool getInstance(int poolSize) {
- if (INSTANCE == null) {
- synchronized (HttpClientPool.class) {
- if (INSTANCE == null) {
- INSTANCE = new HttpClientPool(poolSize);
- }
- }
- }
- return INSTANCE;
- }
-
private int poolSize;
- private HttpClientPool(int poolSize) {
+ private PoolingClientConnectionManager connectionManager;
+
+ public HttpClientPool(int poolSize) {
this.poolSize = poolSize;
+ SchemeRegistry schemeRegistry = new SchemeRegistry();
+ schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
+ schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));
+
+ connectionManager = new PoolingClientConnectionManager(schemeRegistry);
+ connectionManager.setMaxTotal(poolSize);
+ connectionManager.setDefaultMaxPerRoute(100);
}
public HttpClient getClient(Site site) {
@@ -58,7 +54,6 @@ public class HttpClientPool {
params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 3000);
}
-
params.setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.BEST_MATCH);
HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params);
paramsBean.setVersion(HttpVersion.HTTP_1_1);
@@ -67,13 +62,6 @@ public class HttpClientPool {
}
paramsBean.setUseExpectContinue(false);
- SchemeRegistry schemeRegistry = new SchemeRegistry();
- schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
- schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));
-
- PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(schemeRegistry);
- connectionManager.setMaxTotal(poolSize);
- connectionManager.setDefaultMaxPerRoute(100);
DefaultHttpClient httpClient = new DefaultHttpClient(connectionManager, params);
if (site != null) {
generateCookie(httpClient, site);
diff --git a/webmagic-samples/pom.xml b/webmagic-samples/pom.xml
index bf97b75..79238b0 100644
--- a/webmagic-samples/pom.xml
+++ b/webmagic-samples/pom.xml
@@ -5,7 +5,7 @@
webmagic-parent
us.codecraft
- 0.3.2
+ 0.3.3-SNAPSHOT
4.0.0