代理接口的修改,提供刷星代理API。downloader 下载错误时,提供request,exception,proxyProvider三个参数,
parent
9a71f0ac92
commit
ba69eba669
|
@ -3,6 +3,7 @@ package us.codecraft.webmagic.downloader;
|
|||
import us.codecraft.webmagic.Page;
|
||||
import us.codecraft.webmagic.Request;
|
||||
import us.codecraft.webmagic.Site;
|
||||
import us.codecraft.webmagic.proxy.ProxyProvider;
|
||||
import us.codecraft.webmagic.selector.Html;
|
||||
|
||||
/**
|
||||
|
@ -38,7 +39,7 @@ public abstract class AbstractDownloader implements Downloader {
|
|||
protected void onSuccess(Request request) {
|
||||
}
|
||||
|
||||
protected void onError(Request request) {
|
||||
protected void onError(Request request, Throwable throwable, ProxyProvider proxyProvider) {
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -87,7 +87,7 @@ public class HttpClientDownloader extends AbstractDownloader {
|
|||
return page;
|
||||
} catch (IOException e) {
|
||||
logger.warn("download page {} error", request.getUrl(), e);
|
||||
onError(request);
|
||||
onError(request,e,proxyProvider);
|
||||
return page;
|
||||
} finally {
|
||||
if (httpResponse != null) {
|
||||
|
|
|
@ -19,6 +19,13 @@ public interface ProxyProvider {
|
|||
*/
|
||||
void returnProxy(Proxy proxy, Page page, Task task);
|
||||
|
||||
/**
|
||||
* 代理IP是珍贵资源,有可能代理提供者内部代理没有过期,就一直提供某个IP,但这个IP又不可以使用,所以提供一种方式通知提供者,这个代理该刷新了
|
||||
*
|
||||
* @param task 下载任务
|
||||
*/
|
||||
void refreshProxy(Task task);
|
||||
|
||||
/**
|
||||
* Get a proxy for task by some strategy.
|
||||
* @param task the download task
|
||||
|
|
|
@ -30,6 +30,11 @@ public class SimpleProxyProvider implements ProxyProvider {
|
|||
this.pointer = pointer;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void refreshProxy(Task task) {
|
||||
|
||||
}
|
||||
|
||||
public static SimpleProxyProvider from(Proxy... proxies) {
|
||||
List<Proxy> proxiesTemp = new ArrayList<Proxy>(proxies.length);
|
||||
for (Proxy proxy : proxies) {
|
||||
|
|
Loading…
Reference in New Issue