diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/Site.java b/webmagic-core/src/main/java/us/codecraft/webmagic/Site.java index d8f8050..d4fc727 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/Site.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/Site.java @@ -6,6 +6,7 @@ import org.apache.http.HttpHost; import us.codecraft.webmagic.proxy.Proxy; import us.codecraft.webmagic.proxy.SimpleProxyPool; +import org.apache.http.auth.UsernamePasswordCredentials; import us.codecraft.webmagic.proxy.ProxyPool; import us.codecraft.webmagic.utils.UrlUtils; @@ -53,6 +54,8 @@ public class Site { private HttpHost httpProxy; + private UsernamePasswordCredentials usernamePasswordCredentials; //代理用户名密码设置 + private ProxyPool httpProxyPool; private boolean useGzip = true; @@ -486,6 +489,15 @@ public class Site { return this; } + public UsernamePasswordCredentials getUsernamePasswordCredentials() { + return usernamePasswordCredentials; + } + + public Site setUsernamePasswordCredentials(UsernamePasswordCredentials usernamePasswordCredentials) { + this.usernamePasswordCredentials = usernamePasswordCredentials; + return this; + } + public ProxyPool getHttpProxyPool() { return httpProxyPool; } diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java index 0befdd6..225297c 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java @@ -81,6 +81,15 @@ public class HttpClientGenerator { } }); } + + if(site!=null&&site.getHttpProxy()!=null&&site.getUsernamePasswordCredentials()!=null){ + CredentialsProvider credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials( + new AuthScope(site.getHttpProxy()),//可以访问的范围 + site.getUsernamePasswordCredentials());//用户名和密码 + httpClientBuilder.setDefaultCredentialsProvider(credsProvider); + } + SocketConfig socketConfig = SocketConfig.custom().setSoKeepAlive(true).setTcpNoDelay(true).build(); httpClientBuilder.setDefaultSocketConfig(socketConfig); if (site != null) {