Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revert "提供异常刷新httpClient，异常可配置，重写getHttpClient代码" 

This reverts commit 19465089c3.
master
Sutra Zhou 2021-01-02 20:31:53 +08:00
parent 33e3fcdf22
commit 5ceccc62e0
1 changed files with 13 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -2,8 +2,8 @@ package us.codecraft.webmagic.downloader;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Predicate;
import org.apache.commons.io.IOUtils;
@ -24,7 +24,6 @@ import us.codecraft.webmagic.selector.PlainText;
import us.codecraft.webmagic.utils.CharsetUtils;
import us.codecraft.webmagic.utils.HttpClientUtils;
/**
* The http downloader based on HttpClient.
*
@ -33,7 +32,7 @@ import us.codecraft.webmagic.utils.HttpClientUtils;
*/
public class HttpClientDownloader extends AbstractDownloader {
private final Map<String, CloseableHttpClient> httpClients = new ConcurrentHashMap<>();
private final Map<String, CloseableHttpClient> httpClients = new HashMap<String, CloseableHttpClient>();
private final Logger logger = LoggerFactory.getLogger(getClass());
private final HttpClientGenerator httpClientGenerator = new HttpClientGenerator();
@ -46,13 +45,6 @@ public class HttpClientDownloader extends AbstractDownloader {
private Predicate<Throwable> refreshProxyOnError = t -> false;
private Predicate<Throwable> refreshClientOnError = t -> false;
public void setRefreshClientOnError(Predicate<Throwable> clientOnError){
this.refreshClientOnError = clientOnError;
}
public void setRefreshProxyOnError(Predicate<Throwable> proxyOnError) {
this.refreshProxyOnError = refreshProxyOnError;
}
@ -70,8 +62,17 @@ public class HttpClientDownloader extends AbstractDownloader {
return httpClientGenerator.getClient(null);
}
String domain = site.getDomain();
return httpClients.computeIfAbsent(domain,k->httpClientGenerator.getClient(site));
CloseableHttpClient httpClient = httpClients.get(domain);
if (httpClient == null) {
synchronized (this) {
httpClient = httpClients.get(domain);
if (httpClient == null) {
httpClient = httpClientGenerator.getClient(site);
httpClients.put(domain, httpClient);
}
}
}
return httpClient;
}
@Override