Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

todo bugfix

master
yihua.huang 2013-11-03 23:10:09 +08:00
parent 583a0eba8c
commit 160a149b05
2 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
View File

@ -482,7 +482,7 @@ public class Spider implements Runnable, Task {
/**
* Exit when complete. <br/>
* True: exit when all url of the site is downloaded. <br/>
* False: not exit until call stop manually.<br/>
* False: not exit until call stop() manually.<br/>
*
* @param exitWhenComplete
* @return

5
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -90,11 +90,15 @@ public class HttpClientDownloader implements Downloader {
HttpClient httpClient = getHttpClientPool().getClient(site);
try {
HttpGet httpGet = new HttpGet(request.getUrl());
if (headers!=null){
for (Map.Entry<String, String> headerEntry : headers.entrySet()) {
httpGet.addHeader(headerEntry.getKey(),headerEntry.getValue());
}
}
if (!httpGet.containsHeader("Accept-Encoding")) {
httpGet.addHeader("Accept-Encoding", "gzip");
}
HttpResponse httpResponse = null;
int tried = 0;
boolean retry;
@ -168,6 +172,7 @@ public class HttpClientDownloader implements Downloader {
HeaderElement[] codecs = ceheader.getElements();
for (HeaderElement codec : codecs) {
if (codec.getName().equalsIgnoreCase("gzip")) {
//todo bugfix
httpResponse.setEntity(
new GzipDecompressingEntity(httpResponse.getEntity()));
}