From 8d67fd0357ca37dea868b0dacd9c26632c9f5712 Mon Sep 17 00:00:00 2001 From: "yihua.huang" Date: Wed, 28 May 2014 08:08:51 +0800 Subject: [PATCH] change back return proxy from spider to httpclientdownloader #128 --- .../src/main/java/us/codecraft/webmagic/Spider.java | 5 +++++ .../codecraft/webmagic/downloader/HttpClientDownloader.java | 4 ---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java index 81cf179..6f6453b 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java @@ -2,6 +2,7 @@ package us.codecraft.webmagic; import com.google.common.collect.Lists; import org.apache.commons.collections.CollectionUtils; +import org.apache.http.HttpHost; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import us.codecraft.webmagic.downloader.Downloader; @@ -324,6 +325,10 @@ public class Spider implements Runnable, Task { onError(requestFinal); logger.error("process request " + requestFinal + " error", e); } finally { + if (site.getHttpProxyPool().isEnable()) { + site.returnHttpProxyToPool((HttpHost) requestFinal.getExtra(Request.PROXY), (Integer) requestFinal + .getExtra(Request.STATUS_CODE)); + } pageCount.incrementAndGet(); signalNewUrl(); } diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java index f2f6d39..bdafea7 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java @@ -117,10 +117,6 @@ public class HttpClientDownloader extends AbstractDownloader { } catch (IOException e) { logger.warn("close response fail", e); } - if (site.getHttpProxyPool().isEnable()) { - site.returnHttpProxyToPool((HttpHost) request.getExtra(Request.PROXY), (Integer) request - .getExtra(Request.STATUS_CODE)); - } } }