Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

change back return proxy from spider to httpclientdownloader #128

master
yihua.huang 2014-05-28 08:08:51 +08:00
parent 40bf8ca58f
commit 8d67fd0357
2 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
View File

@ -2,6 +2,7 @@ package us.codecraft.webmagic;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.http.HttpHost;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.downloader.Downloader; import us.codecraft.webmagic.downloader.Downloader;
@ -324,6 +325,10 @@ public class Spider implements Runnable, Task {
onError(requestFinal); onError(requestFinal);
logger.error("process request " + requestFinal + " error", e); logger.error("process request " + requestFinal + " error", e);
} finally { } finally {
if (site.getHttpProxyPool().isEnable()) {
site.returnHttpProxyToPool((HttpHost) requestFinal.getExtra(Request.PROXY), (Integer) requestFinal
.getExtra(Request.STATUS_CODE));
}
pageCount.incrementAndGet(); pageCount.incrementAndGet();
signalNewUrl(); signalNewUrl();
} }

4
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -117,10 +117,6 @@ public class HttpClientDownloader extends AbstractDownloader {
} catch (IOException e) { } catch (IOException e) {
logger.warn("close response fail", e); logger.warn("close response fail", e);
} }
if (site.getHttpProxyPool().isEnable()) {
site.returnHttpProxyToPool((HttpHost) request.getExtra(Request.PROXY), (Integer) request
.getExtra(Request.STATUS_CODE));
}
} }
} }