Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

#30 reuse PoolingClientConnectionManager for HttpClientDownloader

master
yihua.huang 2013-10-14 23:22:04 +08:00
parent 5a226387e0
commit 7fb44d2eec
3 changed files with 22 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -34,6 +34,8 @@ public class HttpClientDownloader implements Downloader {
private Logger logger = Logger.getLogger(getClass());
private HttpClientPool httpClientPool;
private int poolSize = 1;
/**
@ -58,6 +60,13 @@ public class HttpClientDownloader implements Downloader {
return (Html) page.getHtml();
}
private HttpClientPool getHttpClientPool(){
if (httpClientPool==null){
httpClientPool = new HttpClientPool(poolSize);
}
return httpClientPool;
}
@Override
public Page download(Request request, Task task) {
Site site = null;
@ -78,7 +87,7 @@ public class HttpClientDownloader implements Downloader {
acceptStatCode.add(200);
}
logger.info("downloading page " + request.getUrl());
HttpClient httpClient = HttpClientPool.getInstance(poolSize).getClient(site);
HttpClient httpClient = getHttpClientPool().getClient(site);
try {
HttpGet httpGet = new HttpGet(request.getUrl());
if (headers!=null){
@ -150,6 +159,7 @@ public class HttpClientDownloader implements Downloader {
@Override
public void setThread(int thread) {
poolSize = thread;
httpClientPool = new HttpClientPool(thread);
}
private void handleGzip(HttpResponse httpResponse) {

32
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientPool.java
View File

@ -24,23 +24,19 @@ import java.util.Map;
*/
public class HttpClientPool {
public static volatile HttpClientPool INSTANCE;
public static HttpClientPool getInstance(int poolSize) {
if (INSTANCE == null) {
synchronized (HttpClientPool.class) {
if (INSTANCE == null) {
INSTANCE = new HttpClientPool(poolSize);
}
}
}
return INSTANCE;
}
private int poolSize;
private HttpClientPool(int poolSize) {
private PoolingClientConnectionManager connectionManager;
public HttpClientPool(int poolSize) {
this.poolSize = poolSize;
SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));
connectionManager = new PoolingClientConnectionManager(schemeRegistry);
connectionManager.setMaxTotal(poolSize);
connectionManager.setDefaultMaxPerRoute(100);
}
public HttpClient getClient(Site site) {
@ -58,7 +54,6 @@ public class HttpClientPool {
params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 3000);
}
params.setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.BEST_MATCH);
HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params);
paramsBean.setVersion(HttpVersion.HTTP_1_1);
@ -67,13 +62,6 @@ public class HttpClientPool {
}
paramsBean.setUseExpectContinue(false);
SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));
PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(schemeRegistry);
connectionManager.setMaxTotal(poolSize);
connectionManager.setDefaultMaxPerRoute(100);
DefaultHttpClient httpClient = new DefaultHttpClient(connectionManager, params);
if (site != null) {
generateCookie(httpClient, site);

2
webmagic-samples/pom.xml
View File

@ -5,7 +5,7 @@
<parent>
<artifactId>webmagic-parent</artifactId>
<groupId>us.codecraft</groupId>
<version>0.3.2</version>
<version>0.3.3-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>