Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

#627 set charset to request

master
yihua.huang 2017-07-29 11:27:56 +08:00
parent 32f1f2cf44
commit 6f5b9e448e
3 changed files with 44 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
webmagic-core/src/main/java/us/codecraft/webmagic/Request.java
View File

@ -51,6 +51,8 @@ public class Request implements Serializable {
*/
private boolean binaryContent = false;
private String charset;
public Request() {
}
@ -176,6 +178,14 @@ public class Request implements Serializable {
this.binaryContent = binaryContent;
}
public String getCharset() {
return charset;
}
public void setCharset(String charset) {
this.charset = charset;
}
@Override
public String toString() {
return "Request{" +

2
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -83,7 +83,7 @@ public class HttpClientDownloader extends AbstractDownloader {
Page page = Page.fail();
try {
httpResponse = httpClient.execute(requestContext.getHttpUriRequest(), requestContext.getHttpClientContext());
page = handleResponse(request, task.getSite().getCharset(), httpResponse, task);
page = handleResponse(request, request.getCharset() != null ? request.getCharset() : task.getSite().getCharset(), httpResponse, task);
onSuccess(request);
logger.info("downloading page success {}", request.getUrl());
return page;

33
webmagic-core/src/test/java/us/codecraft/webmagic/downloader/HttpClientDownloaderTest.java
View File

@ -289,4 +289,37 @@ public class HttpClientDownloaderTest {
});
}
@Test
public void test_download_set_charset() throws Exception {
HttpServer server = httpServer(13423);
server.response(header("Content-Type","text/html; charset=utf-8")).response("hello world!");
Runner.running(server, new Runnable() {
@Override
public void run() throws Exception {
final HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
Request request = new Request();
request.setUrl("http://127.0.0.1:13423/");
Page page = httpClientDownloader.download(request, Site.me().toTask());
assertThat(page.getCharset()).isEqualTo("utf-8");
}
});
}
@Test
public void test_download_set_request_charset() throws Exception {
HttpServer server = httpServer(13423);
server.response("hello world!");
Runner.running(server, new Runnable() {
@Override
public void run() throws Exception {
final HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
Request request = new Request();
request.setCharset("utf-8");
request.setUrl("http://127.0.0.1:13423/");
Page page = httpClientDownloader.download(request, Site.me().setCharset("gbk").toTask());
assertThat(page.getCharset()).isEqualTo("utf-8");
}
});
}
}