Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Bug, add null check to site in HttpClientDownloader & HttpClientGenerator

master
wuyifan 2017-03-17 14:10:54 +08:00
parent e9341d0291
commit 79522f941e
2 changed files with 26 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -82,14 +82,14 @@ public class HttpClientDownloader extends AbstractDownloader {
} }
logger.info("downloading page {}", request.getUrl()); logger.info("downloading page {}", request.getUrl());
CloseableHttpResponse httpResponse = null; CloseableHttpResponse httpResponse = null;
int statusCode=0; int statusCode = 0;
try { try {
HttpHost proxyHost = null; HttpHost proxyHost = null;
Proxy proxy = null; //TODO Proxy proxy = null; //TODO
if (site.getHttpProxyPool() != null && site.getHttpProxyPool().isEnable()) { if (site != null && site.getHttpProxyPool() != null && site.getHttpProxyPool().isEnable()) {
proxy = site.getHttpProxyFromPool(); proxy = site.getHttpProxyFromPool();
proxyHost = proxy.getHttpHost(); proxyHost = proxy.getHttpHost();
} else if(site.getHttpProxy()!= null){ } else if (site != null && site.getHttpProxy() != null){
proxyHost = site.getHttpProxy(); proxyHost = site.getHttpProxy();
} }
@ -107,14 +107,14 @@ public class HttpClientDownloader extends AbstractDownloader {
} }
} catch (IOException e) { } catch (IOException e) {
logger.warn("download page {} error", request.getUrl(), e); logger.warn("download page {} error", request.getUrl(), e);
if (site.getCycleRetryTimes() > 0) { if (site != null && site.getCycleRetryTimes() > 0) {
return addToCycleRetry(request, site); return addToCycleRetry(request, site);
} }
onError(request); onError(request);
return null; return null;
} finally { } finally {
request.putExtra(Request.STATUS_CODE, statusCode); request.putExtra(Request.STATUS_CODE, statusCode);
if (site.getHttpProxyPool()!=null && site.getHttpProxyPool().isEnable()) { if (site != null && site.getHttpProxyPool() != null && site.getHttpProxyPool().isEnable()) {
site.returnHttpProxyToPool((HttpHost) request.getExtra(Request.PROXY), (Integer) request site.returnHttpProxyToPool((HttpHost) request.getExtra(Request.PROXY), (Integer) request
.getExtra(Request.STATUS_CODE)); .getExtra(Request.STATUS_CODE));
} }
@ -138,19 +138,23 @@ public class HttpClientDownloader extends AbstractDownloader {
return acceptStatCode.contains(statusCode); return acceptStatCode.contains(statusCode);
} }
protected HttpUriRequest getHttpUriRequest(Request request, Site site, Map<String, String> headers,HttpHost proxy) { protected HttpUriRequest getHttpUriRequest(Request request, Site site, Map<String, String> headers, HttpHost proxy) {
RequestBuilder requestBuilder = selectRequestMethod(request).setUri(request.getUrl()); RequestBuilder requestBuilder = selectRequestMethod(request).setUri(request.getUrl());
if (headers != null) { if (headers != null) {
for (Map.Entry<String, String> headerEntry : headers.entrySet()) { for (Map.Entry<String, String> headerEntry : headers.entrySet()) {
requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue()); requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue());
} }
} }
RequestConfig.Builder requestConfigBuilder = RequestConfig.custom()
.setConnectionRequestTimeout(site.getTimeOut()) RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
.setSocketTimeout(site.getTimeOut()) if (site != null) {
.setConnectTimeout(site.getTimeOut()) requestConfigBuilder.setConnectionRequestTimeout(site.getTimeOut())
.setCookieSpec(CookieSpecs.BEST_MATCH); .setSocketTimeout(site.getTimeOut())
if (proxy !=null) { .setConnectTimeout(site.getTimeOut())
.setCookieSpec(CookieSpecs.BEST_MATCH);
}
if (proxy != null) {
requestConfigBuilder.setProxy(proxy); requestConfigBuilder.setProxy(proxy);
request.putExtra(Request.PROXY, proxy); request.putExtra(Request.PROXY, proxy);
} }

15
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java
View File

@ -100,7 +100,7 @@ public class HttpClientGenerator {
CredentialsProvider credsProvider = null; CredentialsProvider credsProvider = null;
HttpClientBuilder httpClientBuilder = HttpClients.custom(); HttpClientBuilder httpClientBuilder = HttpClients.custom();
if(proxy!=null && StringUtils.isNotBlank(proxy.getUser()) && StringUtils.isNotBlank(proxy.getPassword())) if (proxy != null && StringUtils.isNotBlank(proxy.getUser()) && StringUtils.isNotBlank(proxy.getPassword()))
{ {
credsProvider= new BasicCredentialsProvider(); credsProvider= new BasicCredentialsProvider();
credsProvider.setCredentials( credsProvider.setCredentials(
@ -109,7 +109,7 @@ public class HttpClientGenerator {
httpClientBuilder.setDefaultCredentialsProvider(credsProvider); httpClientBuilder.setDefaultCredentialsProvider(credsProvider);
} }
if(site!=null&&site.getHttpProxy()!=null&&site.getUsernamePasswordCredentials()!=null){ if (site != null && site.getHttpProxy()!= null && site.getUsernamePasswordCredentials() != null){
credsProvider = new BasicCredentialsProvider(); credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials( credsProvider.setCredentials(
new AuthScope(site.getHttpProxy()),//可以访问的范围 new AuthScope(site.getHttpProxy()),//可以访问的范围
@ -137,14 +137,19 @@ public class HttpClientGenerator {
} }
//解决post/redirect/post 302跳转问题 //解决post/redirect/post 302跳转问题
httpClientBuilder.setRedirectStrategy(new CustomRedirectStrategy()); httpClientBuilder.setRedirectStrategy(new CustomRedirectStrategy());
SocketConfig socketConfig = SocketConfig.custom().setSoTimeout(site.getTimeOut()).setSoKeepAlive(true).setTcpNoDelay(true).build(); SocketConfig.Builder socketConfigBuilder = SocketConfig.custom();
socketConfigBuilder.setSoKeepAlive(true).setTcpNoDelay(true);
if (site != null) {
socketConfigBuilder.setSoTimeout(site.getTimeOut());
}
SocketConfig socketConfig = socketConfigBuilder.build();
httpClientBuilder.setDefaultSocketConfig(socketConfig); httpClientBuilder.setDefaultSocketConfig(socketConfig);
connectionManager.setDefaultSocketConfig(socketConfig); connectionManager.setDefaultSocketConfig(socketConfig);
if (site != null) { if (site != null) {
httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(site.getRetryTimes(), true)); httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(site.getRetryTimes(), true));
generateCookie(httpClientBuilder, site);
} }
generateCookie(httpClientBuilder, site);
return httpClientBuilder.build(); return httpClientBuilder.build();
} }