Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

change return proxy from spider to httpclientdownloader #128

master
yihua.huang 2014-05-28 07:53:23 +08:00
parent 1f21d9cc14
commit 40bf8ca58f
2 changed files with 4 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
View File

@ -2,7 +2,6 @@ package us.codecraft.webmagic;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.http.HttpHost;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.downloader.Downloader; import us.codecraft.webmagic.downloader.Downloader;
@ -325,10 +324,6 @@ public class Spider implements Runnable, Task {
onError(requestFinal); onError(requestFinal);
logger.error("process request " + requestFinal + " error", e); logger.error("process request " + requestFinal + " error", e);
} finally { } finally {
if (site.getHttpProxyPool().isEnable()) {
site.returnHttpProxyToPool((HttpHost) requestFinal.getExtra(Request.PROXY), (Integer) requestFinal
.getExtra(Request.STATUS_CODE));
}
pageCount.incrementAndGet(); pageCount.incrementAndGet();
signalNewUrl(); signalNewUrl();
} }

4
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -117,6 +117,10 @@ public class HttpClientDownloader extends AbstractDownloader {
} catch (IOException e) { } catch (IOException e) {
logger.warn("close response fail", e); logger.warn("close response fail", e);
} }
if (site.getHttpProxyPool().isEnable()) {
site.returnHttpProxyToPool((HttpHost) request.getExtra(Request.PROXY), (Integer) request
.getExtra(Request.STATUS_CODE));
}
} }
} }