Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revert "简化代码" 

This reverts commit 9cc5287743.
master
Sutra Zhou 2021-01-02 20:32:35 +08:00
parent 5ceccc62e0
commit d0843bee0d
1 changed files with 10 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -32,21 +32,24 @@ import us.codecraft.webmagic.utils.HttpClientUtils;
*/ */
public class HttpClientDownloader extends AbstractDownloader { public class HttpClientDownloader extends AbstractDownloader {
private Logger logger = LoggerFactory.getLogger(getClass());
private final Map<String, CloseableHttpClient> httpClients = new HashMap<String, CloseableHttpClient>(); private final Map<String, CloseableHttpClient> httpClients = new HashMap<String, CloseableHttpClient>();
private final Logger logger = LoggerFactory.getLogger(getClass());
private final HttpClientGenerator httpClientGenerator = new HttpClientGenerator(); private HttpClientGenerator httpClientGenerator = new HttpClientGenerator();
private HttpUriRequestConverter httpUriRequestConverter = new HttpUriRequestConverter(); private HttpUriRequestConverter httpUriRequestConverter = new HttpUriRequestConverter();
private ProxyProvider proxyProvider; private ProxyProvider proxyProvider;
private final boolean responseHeader = true; private boolean responseHeader = true;
private volatile boolean refreshProxyOnError = false;
private Predicate<Throwable> refreshProxyOnError = t -> false; private Predicate<Throwable> throwablePredicate = t->false;
public void setRefreshProxyOnError(Predicate<Throwable> proxyOnError) { public void setThrowablePredicate(Predicate<Throwable> predicate){
this.refreshProxyOnError = refreshProxyOnError; this.throwablePredicate = predicate;
} }
public void setHttpUriRequestConverter(HttpUriRequestConverter httpUriRequestConverter) { public void setHttpUriRequestConverter(HttpUriRequestConverter httpUriRequestConverter) {
@ -116,7 +119,7 @@ public class HttpClientDownloader extends AbstractDownloader {
String contentType = httpResponse.getEntity().getContentType() == null ? "" : httpResponse.getEntity().getContentType().getValue(); String contentType = httpResponse.getEntity().getContentType() == null ? "" : httpResponse.getEntity().getContentType().getValue();
Page page = new Page(); Page page = new Page();
page.setBytes(bytes); page.setBytes(bytes);
if (!request.isBinaryContent()) { if (!request.isBinaryContent()){
if (charset == null) { if (charset == null) {
charset = getHtmlCharset(contentType, bytes); charset = getHtmlCharset(contentType, bytes);
} }