Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

change header from Authorization to Proxy-Authorization for Proxy Authorization #596

master
yihua.huang 2017-06-17 16:08:39 +08:00
parent 4111b07263
commit 3ee00015c2
2 changed files with 16 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpUriRequestConverter.java
View File

@ -2,6 +2,7 @@ package us.codecraft.webmagic.downloader;
import org.apache.http.HttpHost; import org.apache.http.HttpHost;
import org.apache.http.auth.AuthState; import org.apache.http.auth.AuthState;
import org.apache.http.auth.ChallengeState;
import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CookieStore; import org.apache.http.client.CookieStore;
import org.apache.http.client.config.CookieSpecs; import org.apache.http.client.config.CookieSpecs;
@ -41,7 +42,7 @@ public class HttpUriRequestConverter {
HttpClientContext httpContext = new HttpClientContext(); HttpClientContext httpContext = new HttpClientContext();
if (proxy != null && proxy.getUsername() != null) { if (proxy != null && proxy.getUsername() != null) {
AuthState authState = new AuthState(); AuthState authState = new AuthState();
authState.update(new BasicScheme(), new UsernamePasswordCredentials(proxy.getUsername(), proxy.getPassword())); authState.update(new BasicScheme(ChallengeState.PROXY), new UsernamePasswordCredentials(proxy.getUsername(), proxy.getPassword()));
httpContext.setAttribute(HttpClientContext.PROXY_AUTH_STATE, authState); httpContext.setAttribute(HttpClientContext.PROXY_AUTH_STATE, authState);
} }
if (request.getCookies() != null && !request.getCookies().isEmpty()) { if (request.getCookies() != null && !request.getCookies().isEmpty()) {

23
webmagic-core/src/test/java/us/codecraft/webmagic/downloader/HttpClientDownloaderTest.java
View File

@ -10,7 +10,6 @@ import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils; import org.apache.http.util.EntityUtils;
import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import us.codecraft.webmagic.Page; import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request; import us.codecraft.webmagic.Request;
@ -255,15 +254,21 @@ public class HttpClientDownloaderTest {
}); });
} }
@Ignore("need proxy server")
@Test @Test
public void test_download_by_SimpleProxyProvider(){ public void test_download_auth_by_SimpleProxyProvider() throws Exception {
HttpClientDownloader httpClientDownloader = new HttpClientDownloader(); HttpServer server = httpServer(13423);
httpClientDownloader.setProxyProvider(SimpleProxyProvider.from(new Proxy("127.0.0.1", 1087))); server.get(eq(header("Proxy-Authorization"), "Basic dXNlcm5hbWU6cGFzc3dvcmQ=")).response("ok");
Request request = new Request(); Runner.running(server, new Runnable() {
request.setUrl("https://www.baidu.com"); @Override
Page page = httpClientDownloader.download(request, Site.me().toTask()); public void run() throws Exception {
assertThat(page.isDownloadSuccess()); HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
httpClientDownloader.setProxyProvider(SimpleProxyProvider.from(new Proxy("127.0.0.1", 13423, "username", "password")));
Request request = new Request();
request.setUrl("http://www.baidu.com");
Page page = httpClientDownloader.download(request, Site.me().toTask());
assertThat(page.getRawText()).isEqualTo("ok");
}
});
} }
} }