webMagic-Mobai
/
magic-Dependency
mirror of https://gitea.qinmian.online/Mobai/magic-Dependency.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add cookie supoort

master
yihua.huang 2013-11-04 00:59:48 +08:00
parent 3c6fced48e
commit a37f40e6e6
2 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -3,6 +3,7 @@ package us.codecraft.webmagic.downloader;
import com.google.common.collect.Sets;
import org.apache.http.HttpResponse;
import org.apache.http.annotation.ThreadSafe;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.RequestBuilder;
@ -102,8 +103,9 @@ public class HttpClientDownloader implements Downloader {
}
RequestConfig.Builder requestConfigBuilder = RequestConfig.custom()
.setConnectionRequestTimeout(site.getTimeOut())
.setConnectTimeout(site.getTimeOut());
if (site.getHttpProxy()!=null){
.setConnectTimeout(site.getTimeOut())
.setCookieSpec(CookieSpecs.BEST_MATCH);
if (site.getHttpProxy() != null) {
requestConfigBuilder.setProxy(site.getHttpProxy());
}
requestBuilder.setConfig(requestConfigBuilder.build());

5
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientPool.java
View File

@ -81,10 +81,11 @@ public class HttpClientPool {
if (site!=null){
httpClientBuilder.setRetryHandler(new DefaultHttpRequestRetryHandler(site.getRetryTimes(),true));
}
generateCookie(httpClientBuilder,site);
return httpClientBuilder.build();
}
private void generateCookie(DefaultHttpClient httpClient, Site site) {
private void generateCookie(HttpClientBuilder httpClientBuilder, Site site) {
CookieStore cookieStore = new BasicCookieStore();
if (site.getCookies() != null) {
for (Map.Entry<String, String> cookieEntry : site.getCookies().entrySet()) {
@ -93,7 +94,7 @@ public class HttpClientPool {
cookieStore.addCookie(cookie);
}
}
httpClient.setCookieStore(cookieStore);
httpClientBuilder.setDefaultCookieStore(cookieStore);
}
}