From 4a6441e7c5923c14d889c7f54af0ef15e5a05cb9 Mon Sep 17 00:00:00 2001 From: yao Date: Mon, 21 Dec 2020 14:52:25 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BE=9B=E5=87=BA=E7=8E=B0=E6=9F=90?= =?UTF-8?q?=E7=A7=8D=E5=BC=82=E5=B8=B8=E5=88=B7=E6=96=B0=E4=BB=A3=E7=90=86?= =?UTF-8?q?=EF=BC=8C=E5=BC=82=E5=B8=B8=E5=8F=AF=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../webmagic/downloader/HttpClientDownloader.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java index 757cdd3..2dd340f 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.nio.charset.Charset; import java.util.HashMap; import java.util.Map; +import java.util.function.Predicate; /** @@ -43,6 +44,14 @@ public class HttpClientDownloader extends AbstractDownloader { private boolean responseHeader = true; + private volatile boolean refreshProxyOnError = false; + + private Predicate throwablePredicate = t->false; + + public void setThrowablePredicate(Predicate predicate){ + this.throwablePredicate = predicate; + } + public void setHttpUriRequestConverter(HttpUriRequestConverter httpUriRequestConverter) { this.httpUriRequestConverter = httpUriRequestConverter; } @@ -88,6 +97,9 @@ public class HttpClientDownloader extends AbstractDownloader { } catch (IOException e) { logger.warn("download page {} error", request.getUrl(), e); onError(request,e,proxyProvider); + if(proxyProvider != null && refreshProxyOnError && throwablePredicate.test(e)){ + proxyProvider.refreshProxy(task); + } return page; } finally { if (httpResponse != null) {