Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Pass the task to onSuccess & onError.

master
Sutra Zhou 2022-10-19 22:10:04 +08:00
parent 5f80e02abd
commit d2b2eed9df
4 changed files with 20 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/AbstractDownloader.java
View File

@ -3,6 +3,7 @@ package us.codecraft.webmagic.downloader;
import us.codecraft.webmagic.Page; import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request; import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site; import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.selector.Html; import us.codecraft.webmagic.selector.Html;
/** /**
@ -35,15 +36,26 @@ public abstract class AbstractDownloader implements Downloader {
return (Html) page.getHtml(); return (Html) page.getHtml();
} }
@Deprecated
protected void onSuccess(Request request) { protected void onSuccess(Request request) {
} }
/**
* @since 0.7.6
*/
protected void onSuccess(Request request, Task task) {
this.onSuccess(request);
}
@Deprecated @Deprecated
protected void onError(Request request) { protected void onError(Request request) {
this.onError(request, null);
} }
protected void onError(Request request, Throwable e) { /**
* @since 0.7.6
*/
protected void onError(Request request, Task task, Throwable e) {
this.onError(request);
} }
} }

4
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -82,12 +82,12 @@ public class HttpClientDownloader extends AbstractDownloader {
try { try {
httpResponse = httpClient.execute(requestContext.getHttpUriRequest(), requestContext.getHttpClientContext()); httpResponse = httpClient.execute(requestContext.getHttpUriRequest(), requestContext.getHttpClientContext());
page = handleResponse(request, request.getCharset() != null ? request.getCharset() : task.getSite().getCharset(), httpResponse, task); page = handleResponse(request, request.getCharset() != null ? request.getCharset() : task.getSite().getCharset(), httpResponse, task);
onSuccess(request); onSuccess(request, task);
logger.info("downloading page success {}", request.getUrl()); logger.info("downloading page success {}", request.getUrl());
return page; return page;
} catch (IOException e) { } catch (IOException e) {
logger.warn("download page {} error", request.getUrl(), e); logger.warn("download page {} error", request.getUrl(), e);
onError(request, e); onError(request, task, e);
return page; return page;
} finally { } finally {
if (httpResponse != null) { if (httpResponse != null) {

4
webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/PhantomJSDownloader.java
View File

@ -98,9 +98,9 @@ public class PhantomJSDownloader extends AbstractDownloader {
page.setRequest(request); page.setRequest(request);
page.setStatusCode(200); page.setStatusCode(200);
} }
onSuccess(request); onSuccess(request, task);
} catch (Exception e) { } catch (Exception e) {
onError(request, e); onError(request, task, e);
logger.warn("download page {} error", request.getUrl(), e); logger.warn("download page {} error", request.getUrl(), e);
} }
return page; return page;

4
webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java
View File

@ -111,10 +111,10 @@ public class SeleniumDownloader extends AbstractDownloader implements Closeable
page.setHtml(new Html(content, request.getUrl())); page.setHtml(new Html(content, request.getUrl()));
page.setUrl(new PlainText(request.getUrl())); page.setUrl(new PlainText(request.getUrl()));
page.setRequest(request); page.setRequest(request);
onSuccess(request); onSuccess(request, task);
} catch (Exception e) { } catch (Exception e) {
logger.warn("download page {} error", request.getUrl(), e); logger.warn("download page {} error", request.getUrl(), e);
onError(request, e); onError(request, task, e);
} finally { } finally {
if (webDriver != null) { if (webDriver != null) {
webDriverPool.returnToPool(webDriver); webDriverPool.returnToPool(webDriver);