Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

closable

master
yihua.huang 2013-08-17 21:19:24 +08:00
parent d01c0eb8ce
commit 5073258237
4 changed files with 13 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
View File

@ -2,7 +2,6 @@ package us.codecraft.webmagic;
import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger;
import us.codecraft.webmagic.downloader.Destroyable;
import us.codecraft.webmagic.downloader.Downloader;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
@ -12,6 +11,8 @@ import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.scheduler.Scheduler;
import us.codecraft.webmagic.utils.ThreadUtils;
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
@ -279,8 +280,12 @@ public class Spider implements Runnable, Task {
}
private void destroyEach(Object object) {
if (object instanceof Destroyable) {
((Destroyable) object).destroy();
if (object instanceof Closeable) {
try {
((Closeable) object).close();
} catch (IOException e) {
e.printStackTrace();
}
}
}

13
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/Destroyable.java
View File

@ -1,13 +0,0 @@
package us.codecraft.webmagic.downloader;
/**
* 比较占用资源的服务可以实现该接口，Spider会在结束时调用destroy()释放资源。<br>
* @author code4crafter@gmail.com <br>
* Date: 13-7-26 <br>
* Time: 下午3:10 <br>
*/
public interface Destroyable {
public void destroy();
}

2
webmagic-extension/src/main/java/us/codecraft/webmagic/model/OOSpider.java
View File

@ -56,7 +56,7 @@ public class OOSpider extends Spider {
public OOSpider(Site site, PageModelPipeline pageModelPipeline, Class... pageModels) {
this(ModelPageProcessor.create(site, pageModels));
this.modelPipeline = new ModelPipeline();
super.pipeline(modelPipeline);
super.addPipeline(modelPipeline);
if (pageModelPipeline!=null){
for (Class pageModel : pageModels) {
this.modelPipeline.put(pageModel, pageModelPipeline);

7
webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java
View File

@ -9,12 +9,13 @@ import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.downloader.Destroyable;
import us.codecraft.webmagic.downloader.Downloader;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.PlainText;
import us.codecraft.webmagic.utils.UrlUtils;
import java.io.Closeable;
import java.io.IOException;
import java.util.Map;
/**
@ -25,7 +26,7 @@ import java.util.Map;
* Date: 13-7-26 <br>
* Time: 下午1:37 <br>
*/
public class SeleniumDownloader implements Downloader, Destroyable {
public class SeleniumDownloader implements Downloader, Closeable {
private volatile WebDriverPool webDriverPool;
@ -104,7 +105,7 @@ public class SeleniumDownloader implements Downloader, Destroyable {
}
@Override
public void destroy() {
public void close() throws IOException {
webDriverPool.closeAll();
}
}