Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add some null pointer check for httpclientdownloader

master
yihua.huang 2013-08-14 13:30:09 +08:00
parent 6c61c5476d
commit 067f3ea0cb
2 changed files with 39 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -17,6 +17,8 @@ import us.codecraft.webmagic.selector.PlainText;
import us.codecraft.webmagic.utils.UrlUtils; import us.codecraft.webmagic.utils.UrlUtils;
import java.io.IOException; import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
/** /**
@ -34,10 +36,23 @@ public class HttpClientDownloader implements Downloader {
@Override @Override
public Page download(Request request, Task task) { public Page download(Request request, Task task) {
Site site = task.getSite(); Site site = null;
if (task != null) {
site = task.getSite();
}
int retryTimes = 0;
Set<Integer> acceptStatCode;
String charset = null;
if (site != null) {
retryTimes = site.getRetryTimes();
acceptStatCode = site.getAcceptStatCode();
charset = site.getCharset();
} else {
acceptStatCode = new HashSet<Integer>();
acceptStatCode.add(200);
}
logger.info("downloading page " + request.getUrl()); logger.info("downloading page " + request.getUrl());
HttpClient httpClient = HttpClientPool.getInstance(poolSize).getClient(site); HttpClient httpClient = HttpClientPool.getInstance(poolSize).getClient(site);
String charset = site.getCharset();
try { try {
HttpGet httpGet = new HttpGet(request.getUrl()); HttpGet httpGet = new HttpGet(request.getUrl());
HttpResponse httpResponse = null; HttpResponse httpResponse = null;
@ -49,7 +64,8 @@ public class HttpClientDownloader implements Downloader {
retry = false; retry = false;
} catch (IOException e) { } catch (IOException e) {
tried++; tried++;
if (tried > site.getRetryTimes()) {
if (tried > retryTimes) {
logger.warn("download page " + request.getUrl() + " error", e); logger.warn("download page " + request.getUrl() + " error", e);
return null; return null;
} }
@ -58,7 +74,7 @@ public class HttpClientDownloader implements Downloader {
} }
} while (retry); } while (retry);
int statusCode = httpResponse.getStatusLine().getStatusCode(); int statusCode = httpResponse.getStatusLine().getStatusCode();
if (site.getAcceptStatCode().contains(statusCode)) { if (acceptStatCode.contains(statusCode)) {
//charset //charset
if (charset == null) { if (charset == null) {
String value = httpResponse.getEntity().getContentType().getValue(); String value = httpResponse.getEntity().getContentType().getValue();

8
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientPool.java
View File

@ -50,13 +50,17 @@ public class HttpClientPool {
private HttpClient generateClient(Site site) { private HttpClient generateClient(Site site) {
HttpParams params = new BasicHttpParams(); HttpParams params = new BasicHttpParams();
if (site != null && site.getUserAgent() != null) {
params.setParameter(CoreProtocolPNames.USER_AGENT, site.getUserAgent()); params.setParameter(CoreProtocolPNames.USER_AGENT, site.getUserAgent());
}
params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, 1000); params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, 1000);
params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 2000); params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 2000);
HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params); HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params);
paramsBean.setVersion(HttpVersion.HTTP_1_1); paramsBean.setVersion(HttpVersion.HTTP_1_1);
if (site != null && site.getCharset() != null) {
paramsBean.setContentCharset(site.getCharset()); paramsBean.setContentCharset(site.getCharset());
}
paramsBean.setUseExpectContinue(false); paramsBean.setUseExpectContinue(false);
SchemeRegistry schemeRegistry = new SchemeRegistry(); SchemeRegistry schemeRegistry = new SchemeRegistry();
@ -67,7 +71,9 @@ public class HttpClientPool {
connectionManager.setMaxTotal(poolSize); connectionManager.setMaxTotal(poolSize);
connectionManager.setDefaultMaxPerRoute(100); connectionManager.setDefaultMaxPerRoute(100);
DefaultHttpClient httpClient = new DefaultHttpClient(connectionManager, params); DefaultHttpClient httpClient = new DefaultHttpClient(connectionManager, params);
if (site != null) {
generateCookie(httpClient, site); generateCookie(httpClient, site);
}
httpClient.getParams().setIntParameter("http.socket.timeout", 60000); httpClient.getParams().setIntParameter("http.socket.timeout", 60000);
httpClient.getParams().setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.BEST_MATCH); httpClient.getParams().setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.BEST_MATCH);
return httpClient; return httpClient;
@ -75,11 +81,13 @@ public class HttpClientPool {
private void generateCookie(DefaultHttpClient httpClient, Site site) { private void generateCookie(DefaultHttpClient httpClient, Site site) {
CookieStore cookieStore = new BasicCookieStore(); CookieStore cookieStore = new BasicCookieStore();
if (site.getCookies() != null) {
for (Map.Entry<String, String> cookieEntry : site.getCookies().entrySet()) { for (Map.Entry<String, String> cookieEntry : site.getCookies().entrySet()) {
BasicClientCookie cookie = new BasicClientCookie(cookieEntry.getKey(), cookieEntry.getValue()); BasicClientCookie cookie = new BasicClientCookie(cookieEntry.getKey(), cookieEntry.getValue());
cookie.setDomain(site.getDomain()); cookie.setDomain(site.getDomain());
cookieStore.addCookie(cookie); cookieStore.addCookie(cookie);
} }
}
httpClient.setCookieStore(cookieStore); httpClient.setCookieStore(cookieStore);
} }