diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java index b6f0034..1bee564 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java @@ -34,6 +34,8 @@ public class HttpClientDownloader implements Downloader { private Logger logger = Logger.getLogger(getClass()); + private HttpClientPool httpClientPool; + private int poolSize = 1; /** @@ -58,6 +60,13 @@ public class HttpClientDownloader implements Downloader { return (Html) page.getHtml(); } + private HttpClientPool getHttpClientPool(){ + if (httpClientPool==null){ + httpClientPool = new HttpClientPool(poolSize); + } + return httpClientPool; + } + @Override public Page download(Request request, Task task) { Site site = null; @@ -78,7 +87,7 @@ public class HttpClientDownloader implements Downloader { acceptStatCode.add(200); } logger.info("downloading page " + request.getUrl()); - HttpClient httpClient = HttpClientPool.getInstance(poolSize).getClient(site); + HttpClient httpClient = getHttpClientPool().getClient(site); try { HttpGet httpGet = new HttpGet(request.getUrl()); if (headers!=null){ @@ -150,6 +159,7 @@ public class HttpClientDownloader implements Downloader { @Override public void setThread(int thread) { poolSize = thread; + httpClientPool = new HttpClientPool(thread); } private void handleGzip(HttpResponse httpResponse) { diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientPool.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientPool.java index f3b72b6..c882836 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientPool.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientPool.java @@ -24,23 +24,19 @@ import java.util.Map; */ public class HttpClientPool { - public static volatile HttpClientPool INSTANCE; - - public static HttpClientPool getInstance(int poolSize) { - if (INSTANCE == null) { - synchronized (HttpClientPool.class) { - if (INSTANCE == null) { - INSTANCE = new HttpClientPool(poolSize); - } - } - } - return INSTANCE; - } - private int poolSize; - private HttpClientPool(int poolSize) { + private PoolingClientConnectionManager connectionManager; + + public HttpClientPool(int poolSize) { this.poolSize = poolSize; + SchemeRegistry schemeRegistry = new SchemeRegistry(); + schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); + schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory())); + + connectionManager = new PoolingClientConnectionManager(schemeRegistry); + connectionManager.setMaxTotal(poolSize); + connectionManager.setDefaultMaxPerRoute(100); } public HttpClient getClient(Site site) { @@ -58,7 +54,6 @@ public class HttpClientPool { params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 3000); } - params.setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.BEST_MATCH); HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params); paramsBean.setVersion(HttpVersion.HTTP_1_1); @@ -67,13 +62,6 @@ public class HttpClientPool { } paramsBean.setUseExpectContinue(false); - SchemeRegistry schemeRegistry = new SchemeRegistry(); - schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory())); - schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory())); - - PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(schemeRegistry); - connectionManager.setMaxTotal(poolSize); - connectionManager.setDefaultMaxPerRoute(100); DefaultHttpClient httpClient = new DefaultHttpClient(connectionManager, params); if (site != null) { generateCookie(httpClient, site); diff --git a/webmagic-samples/pom.xml b/webmagic-samples/pom.xml index bf97b75..79238b0 100644 --- a/webmagic-samples/pom.xml +++ b/webmagic-samples/pom.xml @@ -5,7 +5,7 @@ webmagic-parent us.codecraft - 0.3.2 + 0.3.3-SNAPSHOT 4.0.0