Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

new proxy pool api

master
yihua.huang 2017-03-20 22:47:24 +08:00
parent 46297deaa1
commit 25c81013ca
3 changed files with 7 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
webmagic-core/src/main/java/us/codecraft/webmagic/Site.java
View File

@ -2,7 +2,6 @@ package us.codecraft.webmagic;
import org.apache.http.HttpHost;
import org.apache.http.auth.UsernamePasswordCredentials;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.proxy.ProxyPool;
import us.codecraft.webmagic.proxy.TimerReuseProxyPool;
import us.codecraft.webmagic.utils.UrlUtils;
@ -509,8 +508,4 @@ public class Site {
return httpProxyPool;
}
public Proxy getHttpProxyFromPool() {
return httpProxyPool.getProxy();
}
}

6
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -79,8 +79,8 @@ public class HttpClientDownloader extends AbstractDownloader {
Site site = task.getSite();
Proxy proxy = null;
HttpContext httpContext = new BasicHttpContext();
if (site.getHttpProxyPool() != null && site.getHttpProxyPool().isEnable()) {
proxy = site.getHttpProxyFromPool();
if (site.getHttpProxyPool() != null) {
proxy = site.getHttpProxyPool().getProxy(task);
request.putExtra(Request.PROXY, proxy);
AuthState authState = new AuthState();
authState.update(new BasicScheme(), new UsernamePasswordCredentials(proxy.getUsername(), proxy.getPassword()));
@ -112,7 +112,7 @@ public class HttpClientDownloader extends AbstractDownloader {
EntityUtils.consumeQuietly(httpResponse.getEntity());
}
if (proxy != null) {
site.getHttpProxyPool().returnProxy(proxy, statusCode);
site.getHttpProxyPool().returnProxy(proxy, statusCode, task);
}
}
}

7
webmagic-core/src/main/java/us/codecraft/webmagic/proxy/ProxyPool.java
View File

@ -1,13 +1,14 @@
package us.codecraft.webmagic.proxy;
import us.codecraft.webmagic.Task;
/**
* Created by edwardsbean on 15-2-28.
*/
public interface ProxyPool {
void returnProxy(Proxy proxy, int statusCode);
void returnProxy(Proxy proxy, int statusCode, Task task);
Proxy getProxy();
Proxy getProxy(Task task);
boolean isEnable();
}