主要是 目前 webmagic
最新版是0.7.3版本,在爬取只支持 TLS1.2
的 https
站点会报错,修改HttpClientGenerator中的buildSSLConnectionSocketFactory方法,
private SSLConnectionSocketFactory buildSSLConnectionSocketFactory() {
try {
//return new SSLConnectionSocketFactory(createIgnoreVerifySSL()); // 优先绕过安全证书
return new SSLConnectionSocketFactory(createIgnoreVerifySSL(), new String[]{"SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2"},
null, new DefaultHostnameVerifier());
} catch (KeyManagementException e) {
logger.error("ssl connection fail", e);
} catch (NoSuchAlgorithmException e) {
logger.error("ssl connection fail", e);
}
return SSLConnectionSocketFactory.getSocketFactory();
}