From 5a226387e0f344555603e762b69afc4a9f0ebd57 Mon Sep 17 00:00:00 2001 From: "yihua.huang" Date: Fri, 11 Oct 2013 11:32:44 +0800 Subject: [PATCH] #27 nullpointer fix --- .../codecraft/webmagic/downloader/HttpClientPool.java | 10 +++++++--- .../webmagic/downloader/HttpClientDownloaderTest.java | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientPool.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientPool.java index c256ac4..f3b72b6 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientPool.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientPool.java @@ -51,9 +51,14 @@ public class HttpClientPool { HttpParams params = new BasicHttpParams(); if (site != null && site.getUserAgent() != null) { params.setParameter(CoreProtocolPNames.USER_AGENT, site.getUserAgent()); + params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, site.getTimeOut()); + params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, site.getTimeOut()); + } else { + params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, 3000); + params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 3000); } - params.setIntParameter(CoreConnectionPNames.SO_TIMEOUT, site.getTimeOut()); - params.setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, site.getTimeOut()); + + params.setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.BEST_MATCH); HttpProtocolParamBean paramsBean = new HttpProtocolParamBean(params); paramsBean.setVersion(HttpVersion.HTTP_1_1); @@ -73,7 +78,6 @@ public class HttpClientPool { if (site != null) { generateCookie(httpClient, site); } - return httpClient; } diff --git a/webmagic-core/src/test/java/us/codecraft/webmagic/downloader/HttpClientDownloaderTest.java b/webmagic-core/src/test/java/us/codecraft/webmagic/downloader/HttpClientDownloaderTest.java index 936aece..b5ecada 100644 --- a/webmagic-core/src/test/java/us/codecraft/webmagic/downloader/HttpClientDownloaderTest.java +++ b/webmagic-core/src/test/java/us/codecraft/webmagic/downloader/HttpClientDownloaderTest.java @@ -22,4 +22,5 @@ public class HttpClientDownloaderTest { Page download = httpClientDownloader.download(new Request("http://www.diandian.com"), site.toTask()); Assert.assertTrue(download.getHtml().toString().contains("flashsword30")); } + }