Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revert "代理接口的修改，提供刷星代理API。downloader 下载错误时，提供request,exception,proxyProvider三个参数，" 

This reverts commit ba69eba669.
master
Sutra Zhou 2021-01-02 20:27:28 +08:00
parent c489647c4b
commit 33e3fcdf22
4 changed files with 9 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/AbstractDownloader.java
View File

@ -3,7 +3,6 @@ package us.codecraft.webmagic.downloader;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.proxy.ProxyProvider;
import us.codecraft.webmagic.selector.Html;
/**
@ -39,7 +38,7 @@ public abstract class AbstractDownloader implements Downloader {
protected void onSuccess(Request request) {
}
protected void onError(Request request, Throwable throwable, ProxyProvider proxyProvider) {
protected void onError(Request request) {
}
}

22
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -1,5 +1,11 @@
package us.codecraft.webmagic.downloader;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Predicate;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.CloseableHttpResponse;
@ -7,6 +13,7 @@ import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
@ -17,13 +24,6 @@ import us.codecraft.webmagic.selector.PlainText;
import us.codecraft.webmagic.utils.CharsetUtils;
import us.codecraft.webmagic.utils.HttpClientUtils;
import java.io.IOException;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Predicate;
/**
* The http downloader based on HttpClient.
@ -92,13 +92,7 @@ public class HttpClientDownloader extends AbstractDownloader {
return page;
} catch (IOException e) {
logger.warn("download page {} error", request.getUrl(), e);
onError(request, e, proxyProvider);
if (proxyProvider != null && refreshProxyOnError.test(e)) {
proxyProvider.refreshProxy(task);
}
if(refreshClientOnError.test(e)) {
httpClients.remove(task.getSite().getDomain());
}
onError(request);
return page;
} finally {
if (httpResponse != null) {

7
webmagic-core/src/main/java/us/codecraft/webmagic/proxy/ProxyProvider.java
View File

@ -19,13 +19,6 @@ public interface ProxyProvider {
*/
void returnProxy(Proxy proxy, Page page, Task task);
/**
* 代理IP是珍贵资源，有可能代理提供者内部代理没有过期，就一直提供某个IP，但这个IP又不可以使用，所以提供一种方式通知提供者，这个代理该刷新了
*
* @param task 下载任务
*/
void refreshProxy(Task task);
/**
* Get a proxy for task by some strategy.
* @param task the download task

5
webmagic-core/src/main/java/us/codecraft/webmagic/proxy/SimpleProxyProvider.java
View File

@ -30,11 +30,6 @@ public class SimpleProxyProvider implements ProxyProvider {
this.pointer = pointer;
}
@Override
public void refreshProxy(Task task) {
}
public static SimpleProxyProvider from(Proxy... proxies) {
List<Proxy> proxiesTemp = new ArrayList<Proxy>(proxies.length);
for (Proxy proxy : proxies) {