Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #974 from itranlin/master 

子任务可以使用不同的下载器
master
Sutra Zhou 2020-12-19 18:12:17 +08:00 committed by GitHub
parent 4b902270b4 fc7ae9ce28
commit b2aa5e2677
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 20 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
webmagic-core/src/main/java/us/codecraft/webmagic/Request.java
View File

@ -1,5 +1,6 @@
package us.codecraft.webmagic;
import us.codecraft.webmagic.downloader.Downloader;
import us.codecraft.webmagic.model.HttpRequestBody;
import us.codecraft.webmagic.utils.Experimental;
@ -26,6 +27,11 @@ public class Request implements Serializable {
private HttpRequestBody requestBody;
/**
* this req use this downloader
*/
private Downloader downloader;
/**
* Store additional information in extras.
*/
@ -175,6 +181,14 @@ public class Request implements Serializable {
return binaryContent;
}
public Downloader getDownloader() {
return downloader;
}
public void setDownloader(Downloader downloader) {
this.downloader = downloader;
}
public Request setBinaryContent(boolean binaryContent) {
this.binaryContent = binaryContent;
return this;

7
webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
View File

@ -401,7 +401,12 @@ public class Spider implements Runnable, Task {
}
private void processRequest(Request request) {
Page page = downloader.download(request, this);
Page page;
if (null != request.getDownloader()){
page = request.getDownloader().download(request,this);
}else {
page = downloader.download(request, this);
}
if (page.isDownloadSuccess()){
onDownloadSuccess(request, page);
} else {