From d0843bee0d1cce6a2e7f01f1182b493437f1a6fb Mon Sep 17 00:00:00 2001 From: Sutra Zhou Date: Sat, 2 Jan 2021 20:32:35 +0800 Subject: [PATCH] =?UTF-8?q?Revert=20"=E7=AE=80=E5=8C=96=E4=BB=A3=E7=A0=81"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 9cc5287743de9715ec3ac10a20636377be41d060. --- .../downloader/HttpClientDownloader.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java index 1d308bc..f204e39 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java @@ -32,21 +32,24 @@ import us.codecraft.webmagic.utils.HttpClientUtils; */ public class HttpClientDownloader extends AbstractDownloader { + private Logger logger = LoggerFactory.getLogger(getClass()); + private final Map httpClients = new HashMap(); - private final Logger logger = LoggerFactory.getLogger(getClass()); - private final HttpClientGenerator httpClientGenerator = new HttpClientGenerator(); + + private HttpClientGenerator httpClientGenerator = new HttpClientGenerator(); private HttpUriRequestConverter httpUriRequestConverter = new HttpUriRequestConverter(); private ProxyProvider proxyProvider; - private final boolean responseHeader = true; + private boolean responseHeader = true; + private volatile boolean refreshProxyOnError = false; - private Predicate refreshProxyOnError = t -> false; + private Predicate throwablePredicate = t->false; - public void setRefreshProxyOnError(Predicate proxyOnError) { - this.refreshProxyOnError = refreshProxyOnError; + public void setThrowablePredicate(Predicate predicate){ + this.throwablePredicate = predicate; } public void setHttpUriRequestConverter(HttpUriRequestConverter httpUriRequestConverter) { @@ -116,7 +119,7 @@ public class HttpClientDownloader extends AbstractDownloader { String contentType = httpResponse.getEntity().getContentType() == null ? "" : httpResponse.getEntity().getContentType().getValue(); Page page = new Page(); page.setBytes(bytes); - if (!request.isBinaryContent()) { + if (!request.isBinaryContent()){ if (charset == null) { charset = getHtmlCharset(contentType, bytes); }