diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java index 052c6fa..7ca483a 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java @@ -86,7 +86,7 @@ public class HttpClientDownloader extends AbstractDownloader { authState.update(new BasicScheme(), new UsernamePasswordCredentials(proxy.getUsername(), proxy.getPassword())); httpContext.setAttribute(HttpClientContext.PROXY_AUTH_STATE, authState); } - HttpUriRequest httpUriRequest = httpUriRequestConverter.convert(request, site); + HttpUriRequest httpUriRequest = httpUriRequestConverter.convert(request, site, proxy); CloseableHttpClient httpClient = getHttpClient(site); try { httpResponse = httpClient.execute(httpUriRequest, httpContext); diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpUriRequestConverter.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpUriRequestConverter.java index beda2e6..0ec4b0e 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpUriRequestConverter.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpUriRequestConverter.java @@ -10,6 +10,7 @@ import org.apache.http.client.methods.RequestBuilder; import org.apache.http.message.BasicNameValuePair; import us.codecraft.webmagic.Request; import us.codecraft.webmagic.Site; +import us.codecraft.webmagic.proxy.Proxy; import us.codecraft.webmagic.utils.HttpConstant; import java.nio.charset.Charset; @@ -25,14 +26,10 @@ import java.util.Map; */ public class HttpUriRequestConverter { - public HttpUriRequest convert(Request request, Site site) { - return null; - } - - private HttpUriRequest getHttpUriRequest(Request request, Site site, Map headers, HttpHost proxy) { + public HttpUriRequest convert(Request request, Site site, Proxy proxy) { RequestBuilder requestBuilder = selectRequestMethod(request).setUri(request.getUrl()); - if (headers != null) { - for (Map.Entry headerEntry : headers.entrySet()) { + if (site.getHeaders() != null) { + for (Map.Entry headerEntry : site.getHeaders().entrySet()) { requestBuilder.addHeader(headerEntry.getKey(), headerEntry.getValue()); } } @@ -46,7 +43,7 @@ public class HttpUriRequestConverter { } if (proxy != null) { - requestConfigBuilder.setProxy(proxy); + requestConfigBuilder.setProxy(new HttpHost(proxy.getProxyHost().getHost(), proxy.getProxyHost().getPort())); } requestBuilder.setConfig(requestConfigBuilder.build()); return requestBuilder.build();