代理接口的修改,提供刷星代理API。downloader 下载错误时,提供request,exception,proxyProvider三个参数,

master
yao 2020-12-21 14:36:44 +08:00
parent 9a71f0ac92
commit ba69eba669
4 changed files with 15 additions and 2 deletions

View File

@ -3,6 +3,7 @@ package us.codecraft.webmagic.downloader;
import us.codecraft.webmagic.Page; import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request; import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site; import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.proxy.ProxyProvider;
import us.codecraft.webmagic.selector.Html; import us.codecraft.webmagic.selector.Html;
/** /**
@ -38,7 +39,7 @@ public abstract class AbstractDownloader implements Downloader {
protected void onSuccess(Request request) { protected void onSuccess(Request request) {
} }
protected void onError(Request request) { protected void onError(Request request, Throwable throwable, ProxyProvider proxyProvider) {
} }
} }

View File

@ -87,7 +87,7 @@ public class HttpClientDownloader extends AbstractDownloader {
return page; return page;
} catch (IOException e) { } catch (IOException e) {
logger.warn("download page {} error", request.getUrl(), e); logger.warn("download page {} error", request.getUrl(), e);
onError(request); onError(request,e,proxyProvider);
return page; return page;
} finally { } finally {
if (httpResponse != null) { if (httpResponse != null) {

View File

@ -19,6 +19,13 @@ public interface ProxyProvider {
*/ */
void returnProxy(Proxy proxy, Page page, Task task); void returnProxy(Proxy proxy, Page page, Task task);
/**
* IPIPIP使
*
* @param task
*/
void refreshProxy(Task task);
/** /**
* Get a proxy for task by some strategy. * Get a proxy for task by some strategy.
* @param task the download task * @param task the download task

View File

@ -30,6 +30,11 @@ public class SimpleProxyProvider implements ProxyProvider {
this.pointer = pointer; this.pointer = pointer;
} }
@Override
public void refreshProxy(Task task) {
}
public static SimpleProxyProvider from(Proxy... proxies) { public static SimpleProxyProvider from(Proxy... proxies) {
List<Proxy> proxiesTemp = new ArrayList<Proxy>(proxies.length); List<Proxy> proxiesTemp = new ArrayList<Proxy>(proxies.length);
for (Proxy proxy : proxies) { for (Proxy proxy : proxies) {