diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java index 0fd770d..a2be633 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java @@ -325,10 +325,6 @@ public class Spider implements Runnable, Task { onError(requestFinal); logger.error("process request " + requestFinal + " error", e); } finally { - if (site.getHttpProxyPool()!=null && site.getHttpProxyPool().isEnable()) { - site.returnHttpProxyToPool((HttpHost) requestFinal.getExtra(Request.PROXY), (Integer) requestFinal - .getExtra(Request.STATUS_CODE)); - } pageCount.incrementAndGet(); signalNewUrl(); } @@ -408,9 +404,7 @@ public class Spider implements Runnable, Task { protected void processRequest(Request request) { Page page = downloader.download(request, this); if (page == null) { - sleep(site.getRetrySleepTime()); - onError(request); - return; + throw new RuntimeException("unaccpetable response status"); } // for cycle retry if (page.isNeedCycleRetry()) { diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java index 733d79c..d2cb0ab 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java @@ -119,6 +119,10 @@ public class HttpClientDownloader extends AbstractDownloader { return null; } finally { request.putExtra(Request.STATUS_CODE, statusCode); + if (site.getHttpProxyPool()!=null && site.getHttpProxyPool().isEnable()) { + site.returnHttpProxyToPool((HttpHost) request.getExtra(Request.PROXY), (Integer) request + .getExtra(Request.STATUS_CODE)); + } try { if (httpResponse != null) { //ensure the connection is released back to pool