Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

提供异常刷新httpClient，异常可配置，重写getHttpClient代码

master
yao 2020-12-21 16:02:35 +08:00
parent 9cc5287743
commit 19465089c3
1 changed files with 14 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -21,6 +21,7 @@ import java.io.IOException;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Predicate;
@ -32,7 +33,7 @@ import java.util.function.Predicate;
*/
public class HttpClientDownloader extends AbstractDownloader {
private final Map<String, CloseableHttpClient> httpClients = new HashMap<String, CloseableHttpClient>();
private final Map<String, CloseableHttpClient> httpClients = new ConcurrentHashMap<>();
private final Logger logger = LoggerFactory.getLogger(getClass());
private final HttpClientGenerator httpClientGenerator = new HttpClientGenerator();
@ -45,6 +46,13 @@ public class HttpClientDownloader extends AbstractDownloader {
private Predicate<Throwable> refreshProxyOnError = t -> false;
private Predicate<Throwable> refreshClientOnError = t -> false;
public void setRefreshClientOnError(Predicate<Throwable> clientOnError){
this.refreshClientOnError = clientOnError;
}
public void setRefreshProxyOnError(Predicate<Throwable> proxyOnError) {
this.refreshProxyOnError = refreshProxyOnError;
}
@ -62,17 +70,8 @@ public class HttpClientDownloader extends AbstractDownloader {
return httpClientGenerator.getClient(null);
}
String domain = site.getDomain();
CloseableHttpClient httpClient = httpClients.get(domain);
if (httpClient == null) {
synchronized (this) {
httpClient = httpClients.get(domain);
if (httpClient == null) {
httpClient = httpClientGenerator.getClient(site);
httpClients.put(domain, httpClient);
}
}
}
return httpClient;
return httpClients.computeIfAbsent(domain,k->httpClientGenerator.getClient(site));
}
@Override
@ -97,6 +96,9 @@ public class HttpClientDownloader extends AbstractDownloader {
if (proxyProvider != null && refreshProxyOnError.test(e)) {
proxyProvider.refreshProxy(task);
}
if(refreshClientOnError.test(e)) {
httpClients.remove(task.getSite().getDomain());
}
return page;
} finally {
if (httpResponse != null) {