Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge tag 'WebMagic-0.10.1' into develop 

Tag hotfix
master
Sutra Zhou 2024-04-23 20:41:02 +08:00
parent 884f51ba3b 2994716ba5
commit 2217b1c8f2
2 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -7,6 +7,7 @@ import java.util.Map;
import java.util.Optional;
import org.apache.commons.io.IOUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.impl.client.CloseableHttpClient;
@ -85,13 +86,13 @@ public class HttpClientDownloader extends AbstractDownloader {
page = handleResponse(request, request.getCharset() != null ? request.getCharset() : task.getSite().getCharset(), httpResponse, task);
onSuccess(page, task);
logger.info("downloading page success {}", request.getUrl());
logger.info("Download page success: {}", request.getUrl());
return page;
} catch (IOException e) {
onError(page, task, e);
logger.info("download page {} error", request.getUrl(), e);
logger.info("Download page error: {}", request.getUrl(), e);
return page;
} finally {
@ -111,7 +112,8 @@ public class HttpClientDownloader extends AbstractDownloader {
}
protected Page handleResponse(Request request, String charset, HttpResponse httpResponse, Task task) throws IOException {
byte[] bytes = IOUtils.toByteArray(httpResponse.getEntity().getContent());
HttpEntity entity = httpResponse.getEntity();
byte[] bytes = entity != null ? IOUtils.toByteArray(entity.getContent()) : new byte[0];;
String contentType = httpResponse.getEntity().getContentType() == null ? "" : httpResponse.getEntity().getContentType().getValue();
Page page = new Page();
page.setBytes(bytes);

2
webmagic-samples/pom.xml
View File

@ -28,7 +28,7 @@
<dependency>
<groupId>org.mapdb</groupId>
<artifactId>mapdb</artifactId>
<version>3.0.10</version>
<version>3.1.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>