Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HttpUriRequestConverter

master
yihua.huang 2017-03-20 22:43:32 +08:00
parent 1d86f7c048
commit 46297deaa1
2 changed files with 6 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -86,7 +86,7 @@ public class HttpClientDownloader extends AbstractDownloader {
authState.update(new BasicScheme(), new UsernamePasswordCredentials(proxy.getUsername(), proxy.getPassword())); authState.update(new BasicScheme(), new UsernamePasswordCredentials(proxy.getUsername(), proxy.getPassword()));
httpContext.setAttribute(HttpClientContext.PROXY_AUTH_STATE, authState); httpContext.setAttribute(HttpClientContext.PROXY_AUTH_STATE, authState);
} }
HttpUriRequest httpUriRequest = httpUriRequestConverter.convert(request, site); HttpUriRequest httpUriRequest = httpUriRequestConverter.convert(request, site, proxy);
CloseableHttpClient httpClient = getHttpClient(site); CloseableHttpClient httpClient = getHttpClient(site);
try { try {
httpResponse = httpClient.execute(httpUriRequest, httpContext); httpResponse = httpClient.execute(httpUriRequest, httpContext);

13
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpUriRequestConverter.java
View File

@ -10,6 +10,7 @@ import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.message.BasicNameValuePair; import org.apache.http.message.BasicNameValuePair;
import us.codecraft.webmagic.Request; import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site; import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.proxy.Proxy;
import us.codecraft.webmagic.utils.HttpConstant; import us.codecraft.webmagic.utils.HttpConstant;
import java.nio.charset.Charset; import java.nio.charset.Charset;
@ -25,14 +26,10 @@ import java.util.Map;
*/ */
public class HttpUriRequestConverter { public class HttpUriRequestConverter {
public HttpUriRequest convert(Request request, Site site) { public HttpUriRequest convert(Request request, Site site, Proxy proxy) {
return null;
}
private HttpUriRequest getHttpUriRequest(Request request, Site site, Map<String, String> headers, HttpHost proxy) {
RequestBuilder requestBuilder = selectRequestMethod(request).setUri(request.getUrl()); RequestBuilder requestBuilder = selectRequestMethod(request).setUri(request.getUrl());
if (headers != null) { if (site.getHeaders() != null) {
for (Map.Entry<String, String> headerEntry : headers.entrySet()) { for (Map.Entry<String, String> headerEntry : site.getHeaders().entrySet()) {
requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue()); requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue());
} }
} }
@ -46,7 +43,7 @@ public class HttpUriRequestConverter {
} }
if (proxy != null) { if (proxy != null) {
requestConfigBuilder.setProxy(proxy); requestConfigBuilder.setProxy(new HttpHost(proxy.getProxyHost().getHost(), proxy.getProxyHost().getPort()));
} }
requestBuilder.setConfig(requestConfigBuilder.build()); requestBuilder.setConfig(requestConfigBuilder.build());
return requestBuilder.build(); return requestBuilder.build();