webMagic-Mobai
/
magic-Dependency
mirror of https://gitea.qinmian.online/Mobai/magic-Dependency.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'kapsterio-fix'

master
yihua.huang 2016-05-08 20:54:57 +08:00
parent 37cb43b667 8730e3e97a
commit 3a6e246350
2 changed files with 5 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
View File

@ -325,10 +325,6 @@ public class Spider implements Runnable, Task {
onError(requestFinal); onError(requestFinal);
logger.error("process request " + requestFinal + " error", e); logger.error("process request " + requestFinal + " error", e);
} finally { } finally {
if (site.getHttpProxyPool()!=null && site.getHttpProxyPool().isEnable()) {
site.returnHttpProxyToPool((HttpHost) requestFinal.getExtra(Request.PROXY), (Integer) requestFinal
.getExtra(Request.STATUS_CODE));
}
pageCount.incrementAndGet(); pageCount.incrementAndGet();
signalNewUrl(); signalNewUrl();
} }
@ -408,9 +404,7 @@ public class Spider implements Runnable, Task {
protected void processRequest(Request request) { protected void processRequest(Request request) {
Page page = downloader.download(request, this); Page page = downloader.download(request, this);
if (page == null) { if (page == null) {
sleep(site.getRetrySleepTime()); throw new RuntimeException("unaccpetable response status");
onError(request);
return;
} }
// for cycle retry // for cycle retry
if (page.isNeedCycleRetry()) { if (page.isNeedCycleRetry()) {

4
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -119,6 +119,10 @@ public class HttpClientDownloader extends AbstractDownloader {
return null; return null;
} finally { } finally {
request.putExtra(Request.STATUS_CODE, statusCode); request.putExtra(Request.STATUS_CODE, statusCode);
if (site.getHttpProxyPool()!=null && site.getHttpProxyPool().isEnable()) {
site.returnHttpProxyToPool((HttpHost) request.getExtra(Request.PROXY), (Integer) request
.getExtra(Request.STATUS_CODE));
}
try { try {
if (httpResponse != null) { if (httpResponse != null) {
//ensure the connection is released back to pool //ensure the connection is released back to pool