From af07280176845b9ceb0289fc1c42754096ea7b49 Mon Sep 17 00:00:00 2001 From: "yihua.huang" Date: Thu, 13 Mar 2014 07:11:56 +0800 Subject: [PATCH] remove defend code for httpclient 4.3.1 because it is fixed in 4.3.3 #59 --- .../downloader/HttpClientGenerator.java | 23 +++---------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java index 12f59d5..edb3a49 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java @@ -1,8 +1,9 @@ package us.codecraft.webmagic.downloader; -import org.apache.http.*; +import org.apache.http.HttpException; +import org.apache.http.HttpRequest; +import org.apache.http.HttpRequestInterceptor; import org.apache.http.client.CookieStore; -import org.apache.http.client.protocol.ResponseContentEncoding; import org.apache.http.config.Registry; import org.apache.http.config.RegistryBuilder; import org.apache.http.config.SocketConfig; @@ -66,24 +67,6 @@ public class HttpClientGenerator { } SocketConfig socketConfig = SocketConfig.custom().setSoKeepAlive(true).setTcpNoDelay(true).build(); httpClientBuilder.setDefaultSocketConfig(socketConfig); - // Http client has some problem handling compressing entity for redirect - // So I disable it and do it manually - // https://issues.apache.org/jira/browse/HTTPCLIENT-1432 - httpClientBuilder.disableContentCompression(); - httpClientBuilder.addInterceptorFirst(new HttpResponseInterceptor() { - - private ResponseContentEncoding contentEncoding = new ResponseContentEncoding(); - - public void process( - final HttpResponse response, - final HttpContext context) throws HttpException, IOException { - if (response.getStatusLine().getStatusCode() == 301 || response.getStatusLine().getStatusCode() == 302) { - return; - } - contentEncoding.process(response, context); - } - - }); if (site != null) { httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(site.getRetryTimes(), true)); }