Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Expose Request to ProxyProvider.

master
Joe Zhou 2023-06-22 00:27:30 +08:00
parent 244ade7b4c
commit 58fd08bcf8
4 changed files with 22 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -77,7 +77,7 @@ public class HttpClientDownloader extends AbstractDownloader {
} }
CloseableHttpResponse httpResponse = null; CloseableHttpResponse httpResponse = null;
CloseableHttpClient httpClient = getHttpClient(task.getSite()); CloseableHttpClient httpClient = getHttpClient(task.getSite());
Proxy proxy = proxyProvider != null ? proxyProvider.getProxy(task) : null; Proxy proxy = proxyProvider != null ? proxyProvider.getProxy(request, task) : null;
HttpClientRequestContext requestContext = httpUriRequestConverter.convert(request, task.getSite(), proxy); HttpClientRequestContext requestContext = httpUriRequestConverter.convert(request, task.getSite(), proxy);
Page page = Page.fail(); Page page = Page.fail();
try { try {

19
webmagic-core/src/main/java/us/codecraft/webmagic/proxy/ProxyProvider.java
View File

@ -1,6 +1,7 @@
package us.codecraft.webmagic.proxy; package us.codecraft.webmagic.proxy;
import us.codecraft.webmagic.Page; import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Task; import us.codecraft.webmagic.Task;
/** /**
@ -23,7 +24,23 @@ public interface ProxyProvider {
* Get a proxy for task by some strategy. * Get a proxy for task by some strategy.
* @param task the download task * @param task the download task
* @return proxy * @return proxy
* @deprecated Use {@link #getProxy(Request, Task)} instead.
*/ */
Proxy getProxy(Task task); @Deprecated
default Proxy getProxy(Task task) {
throw new UnsupportedOperationException();
}
/**
* Returns a proxy for the request.
*
* @param request the request
* @param task the download task
* @return proxy
* @since 0.9.0
*/
default Proxy getProxy(Request request, Task task) {
return this.getProxy(task);
}
} }

3
webmagic-core/src/main/java/us/codecraft/webmagic/proxy/SimpleProxyProvider.java
View File

@ -1,6 +1,7 @@
package us.codecraft.webmagic.proxy; package us.codecraft.webmagic.proxy;
import us.codecraft.webmagic.Page; import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Task; import us.codecraft.webmagic.Task;
import java.util.ArrayList; import java.util.ArrayList;
@ -44,7 +45,7 @@ public class SimpleProxyProvider implements ProxyProvider {
} }
@Override @Override
public Proxy getProxy(Task task) { public Proxy getProxy(Request request, Task task) {
return proxies.get(incrForLoop()); return proxies.get(incrForLoop());
} }

2
webmagic-core/src/test/java/us/codecraft/webmagic/proxy/SimpleProxyProviderTest.java
View File

@ -20,7 +20,7 @@ public class SimpleProxyProviderTest {
Proxy originProxy1 = new Proxy("127.0.0.1", 1087); Proxy originProxy1 = new Proxy("127.0.0.1", 1087);
Proxy originProxy2 = new Proxy("127.0.0.1", 1088); Proxy originProxy2 = new Proxy("127.0.0.1", 1088);
SimpleProxyProvider proxyProvider = SimpleProxyProvider.from(originProxy1, originProxy2); SimpleProxyProvider proxyProvider = SimpleProxyProvider.from(originProxy1, originProxy2);
Proxy proxy = proxyProvider.getProxy(TASK); Proxy proxy = proxyProvider.getProxy(null, TASK);
assertThat(proxy).isEqualTo(originProxy1); assertThat(proxy).isEqualTo(originProxy1);
proxy = proxyProvider.getProxy(TASK); proxy = proxyProvider.getProxy(TASK);
assertThat(proxy).isEqualTo(originProxy2); assertThat(proxy).isEqualTo(originProxy2);