Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

简化代码

master
yao 2020-12-21 14:58:01 +08:00
parent 4a6441e7c5
commit 9cc5287743
1 changed files with 10 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -32,24 +32,21 @@ import java.util.function.Predicate;
*/ */
public class HttpClientDownloader extends AbstractDownloader { public class HttpClientDownloader extends AbstractDownloader {
private Logger logger = LoggerFactory.getLogger(getClass());
private final Map<String, CloseableHttpClient> httpClients = new HashMap<String, CloseableHttpClient>(); private final Map<String, CloseableHttpClient> httpClients = new HashMap<String, CloseableHttpClient>();
private final Logger logger = LoggerFactory.getLogger(getClass());
private HttpClientGenerator httpClientGenerator = new HttpClientGenerator(); private final HttpClientGenerator httpClientGenerator = new HttpClientGenerator();
private HttpUriRequestConverter httpUriRequestConverter = new HttpUriRequestConverter(); private HttpUriRequestConverter httpUriRequestConverter = new HttpUriRequestConverter();
private ProxyProvider proxyProvider; private ProxyProvider proxyProvider;
private boolean responseHeader = true; private final boolean responseHeader = true;
private volatile boolean refreshProxyOnError = false;
private Predicate<Throwable> throwablePredicate = t->false; private Predicate<Throwable> refreshProxyOnError = t -> false;
public void setThrowablePredicate(Predicate<Throwable> predicate){ public void setRefreshProxyOnError(Predicate<Throwable> proxyOnError) {
this.throwablePredicate = predicate; this.refreshProxyOnError = refreshProxyOnError;
} }
public void setHttpUriRequestConverter(HttpUriRequestConverter httpUriRequestConverter) { public void setHttpUriRequestConverter(HttpUriRequestConverter httpUriRequestConverter) {
@ -96,8 +93,8 @@ public class HttpClientDownloader extends AbstractDownloader {
return page; return page;
} catch (IOException e) { } catch (IOException e) {
logger.warn("download page {} error", request.getUrl(), e); logger.warn("download page {} error", request.getUrl(), e);
onError(request,e,proxyProvider); onError(request, e, proxyProvider);
if(proxyProvider != null && refreshProxyOnError && throwablePredicate.test(e)){ if (proxyProvider != null && refreshProxyOnError.test(e)) {
proxyProvider.refreshProxy(task); proxyProvider.refreshProxy(task);
} }
return page; return page;
@ -122,7 +119,7 @@ public class HttpClientDownloader extends AbstractDownloader {
String contentType = httpResponse.getEntity().getContentType() == null ? "" : httpResponse.getEntity().getContentType().getValue(); String contentType = httpResponse.getEntity().getContentType() == null ? "" : httpResponse.getEntity().getContentType().getValue();
Page page = new Page(); Page page = new Page();
page.setBytes(bytes); page.setBytes(bytes);
if (!request.isBinaryContent()){ if (!request.isBinaryContent()) {
if (charset == null) { if (charset == null) {
charset = getHtmlCharset(contentType, bytes); charset = getHtmlCharset(contentType, bytes);
} }