Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

todo bugfix

master
yihua.huang 2013-11-03 23:10:09 +08:00
parent 583a0eba8c
commit 160a149b05
2 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
View File

@ -482,7 +482,7 @@ public class Spider implements Runnable, Task {
/** /**
* Exit when complete. <br/> * Exit when complete. <br/>
* True: exit when all url of the site is downloaded. <br/> * True: exit when all url of the site is downloaded. <br/>
* False: not exit until call stop manually.<br/> * False: not exit until call stop() manually.<br/>
* *
* @param exitWhenComplete * @param exitWhenComplete
* @return * @return

5
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -90,11 +90,15 @@ public class HttpClientDownloader implements Downloader {
HttpClient httpClient = getHttpClientPool().getClient(site); HttpClient httpClient = getHttpClientPool().getClient(site);
try { try {
HttpGet httpGet = new HttpGet(request.getUrl()); HttpGet httpGet = new HttpGet(request.getUrl());
if (headers!=null){ if (headers!=null){
for (Map.Entry<String, String> headerEntry : headers.entrySet()) { for (Map.Entry<String, String> headerEntry : headers.entrySet()) {
httpGet.addHeader(headerEntry.getKey(),headerEntry.getValue()); httpGet.addHeader(headerEntry.getKey(),headerEntry.getValue());
} }
} }
if (!httpGet.containsHeader("Accept-Encoding")) {
httpGet.addHeader("Accept-Encoding", "gzip");
}
HttpResponse httpResponse = null; HttpResponse httpResponse = null;
int tried = 0; int tried = 0;
boolean retry; boolean retry;
@ -168,6 +172,7 @@ public class HttpClientDownloader implements Downloader {
HeaderElement[] codecs = ceheader.getElements(); HeaderElement[] codecs = ceheader.getElements();
for (HeaderElement codec : codecs) { for (HeaderElement codec : codecs) {
if (codec.getName().equalsIgnoreCase("gzip")) { if (codec.getName().equalsIgnoreCase("gzip")) {
//todo bugfix
httpResponse.setEntity( httpResponse.setEntity(
new GzipDecompressingEntity(httpResponse.getEntity())); new GzipDecompressingEntity(httpResponse.getEntity()));
} }