Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix charset bug

master
yihua.huang 2013-06-08 20:29:53 +08:00
parent d1fc1cf305
commit 3c3f001186
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/main/java/us/codecraft/spider/downloader/HttpClientDownloader.java
View File

@ -31,8 +31,12 @@ public class HttpClientDownloader implements Downloader {
HttpResponse httpResponse = httpClient.execute(httpGet); HttpResponse httpResponse = httpClient.execute(httpGet);
int statusCode = httpResponse.getStatusLine().getStatusCode(); int statusCode = httpResponse.getStatusLine().getStatusCode();
if (site.getAcceptStatCode().contains(statusCode)) { if (site.getAcceptStatCode().contains(statusCode)) {
if (site.getEncoding() == null){
String value = httpResponse.getEntity().getContentType().getValue();
site.setEncoding(new PlainText(value).r("charset=([^\\s]+)").toString());
}
String content = IOUtils.toString(httpResponse.getEntity().getContent(), String content = IOUtils.toString(httpResponse.getEntity().getContent(),
site.getEncoding() == null ? httpResponse.getEntity().getContentType().getValue() : site.getEncoding()); site.getEncoding());
Page page = new Page(); Page page = new Page();
page.setHtml(new Html(UrlUtils.fixAllRelativeHrefs(content, request.getUrl()))); page.setHtml(new Html(UrlUtils.fixAllRelativeHrefs(content, request.getUrl())));
page.setUrl(new PlainText(request.getUrl())); page.setUrl(new PlainText(request.getUrl()));