webMagic-Mobai
/
magic-Dependency
mirror of https://gitea.qinmian.online/Mobai/magic-Dependency.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

提供出现某种异常刷新代理，异常可配置

master
yao 2020-12-21 14:52:25 +08:00
parent ba69eba669
commit 4a6441e7c5
1 changed files with 12 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -21,6 +21,7 @@ import java.io.IOException;
import java.nio.charset.Charset; import java.nio.charset.Charset;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.function.Predicate;
/** /**
@ -43,6 +44,14 @@ public class HttpClientDownloader extends AbstractDownloader {
private boolean responseHeader = true; private boolean responseHeader = true;
private volatile boolean refreshProxyOnError = false;
private Predicate<Throwable> throwablePredicate = t->false;
public void setThrowablePredicate(Predicate<Throwable> predicate){
this.throwablePredicate = predicate;
}
public void setHttpUriRequestConverter(HttpUriRequestConverter httpUriRequestConverter) { public void setHttpUriRequestConverter(HttpUriRequestConverter httpUriRequestConverter) {
this.httpUriRequestConverter = httpUriRequestConverter; this.httpUriRequestConverter = httpUriRequestConverter;
} }
@ -88,6 +97,9 @@ public class HttpClientDownloader extends AbstractDownloader {
} catch (IOException e) { } catch (IOException e) {
logger.warn("download page {} error", request.getUrl(), e); logger.warn("download page {} error", request.getUrl(), e);
onError(request,e,proxyProvider); onError(request,e,proxyProvider);
if(proxyProvider != null && refreshProxyOnError && throwablePredicate.test(e)){
proxyProvider.refreshProxy(task);
}
return page; return page;
} finally { } finally {
if (httpResponse != null) { if (httpResponse != null) {