Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat:update host verify (#1112)

master
Tanky-Zhang 2023-03-22 22:25:51 +08:00 committed by GitHub
parent 717931166a
commit 244ade7b4c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 20 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientGenerator.java
View File

@ -1,16 +1,5 @@
package us.codecraft.webmagic.downloader; package us.codecraft.webmagic.downloader;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.Map;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import org.apache.commons.lang3.JavaVersion; import org.apache.commons.lang3.JavaVersion;
import org.apache.commons.lang3.SystemUtils; import org.apache.commons.lang3.SystemUtils;
import org.apache.http.HttpException; import org.apache.http.HttpException;
@ -22,28 +11,32 @@ import org.apache.http.config.RegistryBuilder;
import org.apache.http.config.SocketConfig; import org.apache.http.config.SocketConfig;
import org.apache.http.conn.socket.ConnectionSocketFactory; import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory; import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.DefaultHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.BasicCookieStore; import org.apache.http.impl.client.*;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.impl.cookie.BasicClientCookie; import org.apache.http.impl.cookie.BasicClientCookie;
import org.apache.http.protocol.HttpContext; import org.apache.http.protocol.HttpContext;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.Site; import us.codecraft.webmagic.Site;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import java.io.IOException;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.Map;
/** /**
* @author code4crafter@gmail.com <br> * @author code4crafter@gmail.com <br>
* @since 0.4.0 * @since 0.4.0
*/ */
public class HttpClientGenerator { public class HttpClientGenerator {
private transient Logger logger = LoggerFactory.getLogger(getClass()); private transient Logger logger = LoggerFactory.getLogger(getClass());
private PoolingHttpClientConnectionManager connectionManager; private PoolingHttpClientConnectionManager connectionManager;
@ -61,21 +54,20 @@ public class HttpClientGenerator {
SSLContext sslContext = createIgnoreVerifySSL(); SSLContext sslContext = createIgnoreVerifySSL();
String[] supportedProtocols; String[] supportedProtocols;
if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_11)) { if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_11)) {
supportedProtocols = new String[] { "SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3" }; supportedProtocols = new String[]{"SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3"};
} else { } else {
supportedProtocols = new String[] { "SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2" }; supportedProtocols = new String[]{"SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2"};
} }
logger.debug("supportedProtocols: {}", String.join(", ", supportedProtocols)); logger.debug("supportedProtocols: {}", String.join(", ", supportedProtocols));
return new SSLConnectionSocketFactory(sslContext, supportedProtocols, return new SSLConnectionSocketFactory(sslContext, supportedProtocols,
null, null,
new DefaultHostnameVerifier()); // 优先绕过安全证书 //不进行主机校验
} catch (KeyManagementException e) { (host, sslSession) -> true); // 优先绕过安全证书
logger.error("ssl connection fail", e); } catch (KeyManagementException | NoSuchAlgorithmException e) {
} catch (NoSuchAlgorithmException e) {
logger.error("ssl connection fail", e); logger.error("ssl connection fail", e);
} }
return SSLConnectionSocketFactory.getSocketFactory(); return SSLConnectionSocketFactory.getSocketFactory();
} }
private SSLContext createIgnoreVerifySSL() throws NoSuchAlgorithmException, KeyManagementException { private SSLContext createIgnoreVerifySSL() throws NoSuchAlgorithmException, KeyManagementException {
// 实现一个X509TrustManager接口，用于绕过验证，不用修改里面的方法 // 实现一个X509TrustManager接口，用于绕过验证，不用修改里面的方法
@ -97,9 +89,9 @@ public class HttpClientGenerator {
}; };
SSLContext sc = SSLContext.getInstance("TLS"); SSLContext sc = SSLContext.getInstance("TLS");
sc.init(null, new TrustManager[] { trustManager }, null); sc.init(null, new TrustManager[]{trustManager}, null);
return sc; return sc;
} }
public HttpClientGenerator setPoolSize(int poolSize) { public HttpClientGenerator setPoolSize(int poolSize) {
connectionManager.setMaxTotal(poolSize); connectionManager.setMaxTotal(poolSize);