Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into 0.4.x

master
yihua.huang 2014-03-13 07:12:29 +08:00
parent d5a978e00f af07280176
commit 66d4d3c192
1 changed files with 3 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java
View File

@ -1,8 +1,9 @@
package us.codecraft.webmagic.downloader;
import org.apache.http.*;
import org.apache.http.HttpException;
import org.apache.http.HttpRequest;
import org.apache.http.HttpRequestInterceptor;
import org.apache.http.client.CookieStore;
import org.apache.http.client.protocol.ResponseContentEncoding;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.config.SocketConfig;
@ -66,24 +67,6 @@ public class HttpClientGenerator {
}
SocketConfig socketConfig = SocketConfig.custom().setSoKeepAlive(true).setTcpNoDelay(true).build();
httpClientBuilder.setDefaultSocketConfig(socketConfig);
// Http client has some problem handling compressing entity for redirect
// So I disable it and do it manually
// https://issues.apache.org/jira/browse/HTTPCLIENT-1432
httpClientBuilder.disableContentCompression();
httpClientBuilder.addInterceptorFirst(new HttpResponseInterceptor() {
private ResponseContentEncoding contentEncoding = new ResponseContentEncoding();
public void process(
final HttpResponse response,
final HttpContext context) throws HttpException, IOException {
if (response.getStatusLine().getStatusCode() == 301 || response.getStatusLine().getStatusCode() == 302) {
return;
}
contentEncoding.process(response, context);
}
});
if (site != null) {
httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(site.getRetryTimes(), true));
}