webMagic-Mobai
/
magic-Dependency
mirror of https://gitea.qinmian.online/Mobai/magic-Dependency.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix

master
yihua.huang 2017-03-18 12:18:00 +08:00
parent a7f9e7cad5
commit b71f379512
2 changed files with 7 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
webmagic-core/src/main/java/us/codecraft/webmagic/Site.java
View File

@ -513,8 +513,4 @@ public class Site {
return httpProxyPool.getProxy(); return httpProxyPool.getProxy();
} }
public void returnHttpProxyToPool(HttpHost proxy,int statusCode) {
httpProxyPool.returnProxy(proxy,statusCode);
}
} }

14
webmagic-core/src/main/java/us/codecraft/webmagic/proxy/TimerReuseProxyPool.java
View File

@ -186,31 +186,31 @@ public class TimerReuseProxyPool implements ProxyPool {
} }
public void returnProxy(HttpHost host, int statusCode) { public void returnProxy(HttpHost host, int statusCode) {
Proxy p = allProxy.get(host.getAddress().getHostAddress()); TimerReuseProxy p = allProxy.get(host.getAddress().getHostAddress());
if (p == null) { if (p == null) {
return; return;
} }
switch (statusCode) { switch (statusCode) {
case Proxy.SUCCESS: case TimerReuseProxy.SUCCESS:
p.setReuseTimeInterval(reuseInterval); p.setReuseTimeInterval(reuseInterval);
p.setFailedNum(0); p.setFailedNum(0);
p.setFailedErrorType(new ArrayList<Integer>()); p.setFailedErrorType(new ArrayList<Integer>());
p.recordResponse(); p.recordResponse();
p.successNumIncrement(1); p.successNumIncrement(1);
break; break;
case Proxy.ERROR_403: case TimerReuseProxy.ERROR_403:
// banned,try longer interval // banned,try longer interval
p.fail(Proxy.ERROR_403); p.fail(TimerReuseProxy.ERROR_403);
p.setReuseTimeInterval(reuseInterval * p.getFailedNum()); p.setReuseTimeInterval(reuseInterval * p.getFailedNum());
logger.info(host + " >>>> reuseTimeInterval is >>>> " + p.getReuseTimeInterval() / 1000.0); logger.info(host + " >>>> reuseTimeInterval is >>>> " + p.getReuseTimeInterval() / 1000.0);
break; break;
case Proxy.ERROR_BANNED: case TimerReuseProxy.ERROR_BANNED:
p.fail(Proxy.ERROR_BANNED); p.fail(TimerReuseProxy.ERROR_BANNED);
p.setReuseTimeInterval(10 * 60 * 1000 * p.getFailedNum()); p.setReuseTimeInterval(10 * 60 * 1000 * p.getFailedNum());
logger.warn("this proxy is banned >>>> " + p.getHttpHost()); logger.warn("this proxy is banned >>>> " + p.getHttpHost());
logger.info(host + " >>>> reuseTimeInterval is >>>> " + p.getReuseTimeInterval() / 1000.0); logger.info(host + " >>>> reuseTimeInterval is >>>> " + p.getReuseTimeInterval() / 1000.0);
break; break;
case Proxy.ERROR_404: case TimerReuseProxy.ERROR_404:
// p.fail(Proxy.ERROR_404); // p.fail(Proxy.ERROR_404);
// p.setReuseTimeInterval(reuseInterval * p.getFailedNum()); // p.setReuseTimeInterval(reuseInterval * p.getFailedNum());
break; break;