From 4281f82352505021a36fab064a1bfff9bbb55c0d Mon Sep 17 00:00:00 2001 From: Yihua Huang Date: Thu, 11 Jan 2024 08:53:43 +0800 Subject: [PATCH 1/6] fix: webmagic-samples/pom.xml to reduce vulnerabilities (#1142) The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JAVA-ORGJETBRAINSKOTLIN-2393744 - https://snyk.io/vuln/SNYK-JAVA-ORGJETBRAINSKOTLIN-2628385 Co-authored-by: snyk-bot --- webmagic-samples/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webmagic-samples/pom.xml b/webmagic-samples/pom.xml index 08e70c1..906606f 100644 --- a/webmagic-samples/pom.xml +++ b/webmagic-samples/pom.xml @@ -27,7 +27,7 @@ org.mapdb mapdb - 3.0.10 + 3.1.0 com.fasterxml.jackson.core From 65fd8f3779580f925e0ec54e1d8037859a6ce303 Mon Sep 17 00:00:00 2001 From: Yihua Huang Date: Mon, 22 Jan 2024 12:50:01 +0800 Subject: [PATCH 2/6] fix: pom.xml to reduce vulnerabilities (#1146) The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JAVA-COMJAYWAYJSONPATH-6140361 Co-authored-by: snyk-bot --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5f1bdf9..2fb56e7 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 4.4.15 3.7.1 9.3.9.0 - 2.8.0 + 2.9.0 4.13.2 2.7.3 1.2.17 From 76ef0332f1a29361e77458338e29db10506a48a7 Mon Sep 17 00:00:00 2001 From: Sutra Zhou Date: Tue, 23 Apr 2024 20:06:47 +0800 Subject: [PATCH 3/6] Update versions for hotfix --- pom.xml | 2 +- webmagic-core/pom.xml | 2 +- webmagic-coverage/pom.xml | 2 +- webmagic-extension/pom.xml | 2 +- webmagic-samples/pom.xml | 2 +- webmagic-saxon/pom.xml | 2 +- webmagic-scripts/pom.xml | 2 +- webmagic-selenium/pom.xml | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index 2fb56e7..15722ec 100644 --- a/pom.xml +++ b/pom.xml @@ -1,7 +1,7 @@ us.codecraft - 0.10.0 + 0.10.1 4.0.0 pom diff --git a/webmagic-core/pom.xml b/webmagic-core/pom.xml index 021a83f..dcf2e4c 100644 --- a/webmagic-core/pom.xml +++ b/webmagic-core/pom.xml @@ -3,7 +3,7 @@ us.codecraft webmagic-parent - 0.10.0 + 0.10.1 4.0.0 diff --git a/webmagic-coverage/pom.xml b/webmagic-coverage/pom.xml index 4109c49..3e441ec 100644 --- a/webmagic-coverage/pom.xml +++ b/webmagic-coverage/pom.xml @@ -8,7 +8,7 @@ us.codecraft webmagic-parent - 0.10.0 + 0.10.1 webmagic-coverage diff --git a/webmagic-extension/pom.xml b/webmagic-extension/pom.xml index b47ae36..d99bf82 100644 --- a/webmagic-extension/pom.xml +++ b/webmagic-extension/pom.xml @@ -3,7 +3,7 @@ us.codecraft webmagic-parent - 0.10.0 + 0.10.1 4.0.0 diff --git a/webmagic-samples/pom.xml b/webmagic-samples/pom.xml index 906606f..bae376e 100644 --- a/webmagic-samples/pom.xml +++ b/webmagic-samples/pom.xml @@ -3,7 +3,7 @@ webmagic-parent us.codecraft - 0.10.0 + 0.10.1 4.0.0 diff --git a/webmagic-saxon/pom.xml b/webmagic-saxon/pom.xml index 4a2b358..9709a04 100644 --- a/webmagic-saxon/pom.xml +++ b/webmagic-saxon/pom.xml @@ -3,7 +3,7 @@ webmagic-parent us.codecraft - 0.10.0 + 0.10.1 4.0.0 diff --git a/webmagic-scripts/pom.xml b/webmagic-scripts/pom.xml index 9291465..2939b82 100644 --- a/webmagic-scripts/pom.xml +++ b/webmagic-scripts/pom.xml @@ -3,7 +3,7 @@ webmagic-parent us.codecraft - 0.10.0 + 0.10.1 4.0.0 diff --git a/webmagic-selenium/pom.xml b/webmagic-selenium/pom.xml index 5c2e50b..cd12130 100644 --- a/webmagic-selenium/pom.xml +++ b/webmagic-selenium/pom.xml @@ -3,7 +3,7 @@ webmagic-parent us.codecraft - 0.10.0 + 0.10.1 4.0.0 From a0ff4a2d2e3834e68ca399c0621b15bc798b4f83 Mon Sep 17 00:00:00 2001 From: Sutra Zhou Date: Tue, 23 Apr 2024 20:09:15 +0800 Subject: [PATCH 4/6] Fix log message. --- .../codecraft/webmagic/downloader/HttpClientDownloader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java index 80e7b72..05d6e1a 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java @@ -85,13 +85,13 @@ public class HttpClientDownloader extends AbstractDownloader { page = handleResponse(request, request.getCharset() != null ? request.getCharset() : task.getSite().getCharset(), httpResponse, task); onSuccess(page, task); - logger.info("downloading page success {}", request.getUrl()); + logger.info("Download page success: {}", request.getUrl()); return page; } catch (IOException e) { onError(page, task, e); - logger.info("download page {} error", request.getUrl(), e); + logger.info("Download page error: {}", request.getUrl(), e); return page; } finally { From 5afe3e7b81b905386f622d07b3fbe1a276da7a71 Mon Sep 17 00:00:00 2001 From: Sutra Zhou Date: Tue, 23 Apr 2024 20:21:45 +0800 Subject: [PATCH 5/6] Revert "fix: pom.xml to reduce vulnerabilities (#1134)" This reverts commit 19288e9c11551e7b0e3a2533183942405f0fa521. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 15722ec..eecbdb9 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ 1.3.0 1.2.0 11.4 - 4.14.1 + 3.141.59 2.0.4 4.0.0.RELEASE 0.3.5 From 78740a4e283bdbe8d5f237d32f7b5a35eb634d9d Mon Sep 17 00:00:00 2001 From: Sutra Zhou Date: Tue, 23 Apr 2024 20:26:38 +0800 Subject: [PATCH 6/6] Fix NPE. --- .../codecraft/webmagic/downloader/HttpClientDownloader.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java index 05d6e1a..31eebc7 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java @@ -7,6 +7,7 @@ import java.util.Map; import java.util.Optional; import org.apache.commons.io.IOUtils; +import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.impl.client.CloseableHttpClient; @@ -111,7 +112,8 @@ public class HttpClientDownloader extends AbstractDownloader { } protected Page handleResponse(Request request, String charset, HttpResponse httpResponse, Task task) throws IOException { - byte[] bytes = IOUtils.toByteArray(httpResponse.getEntity().getContent()); + HttpEntity entity = httpResponse.getEntity(); + byte[] bytes = entity != null ? IOUtils.toByteArray(entity.getContent()) : new byte[0];; String contentType = httpResponse.getEntity().getContentType() == null ? "" : httpResponse.getEntity().getContentType().getValue(); Page page = new Page(); page.setBytes(bytes);