Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add charset specific for easy call of HttpClientDownloader

master
yihua.huang 2013-08-25 15:41:43 +08:00
parent 478ace7e97
commit 0cc0ccee35
1 changed files with 12 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -46,6 +46,17 @@ public class HttpClientDownloader implements Downloader {
return (Html) page.getHtml();
}
/**
* A simple method to download a url.
*
* @param url
* @return html
*/
public Html download(String url,String charset) {
Page page = download(new Request(url), Site.me().setCharset(charset).toTask());
return (Html) page.getHtml();
}
@Override
public Page download(Request request, Task task) {
Site site = null;
@ -87,13 +98,12 @@ public class HttpClientDownloader implements Downloader {
} while (retry);
int statusCode = httpResponse.getStatusLine().getStatusCode();
if (acceptStatCode.contains(statusCode)) {
handleGzip(httpResponse);
//charset
if (charset == null) {
String value = httpResponse.getEntity().getContentType().getValue();
charset = UrlUtils.getCharset(value);
}
//
handleGzip(httpResponse);
return handleResponse(request, charset, httpResponse, task);
} else {
logger.warn("code error " + statusCode + "\t" + request.getUrl());