diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
index 7e499be..bf3343b 100644
--- a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
+++ b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
@@ -374,6 +374,7 @@ public class Spider implements Runnable, Task {
public void close() {
destroyEach(downloader);
destroyEach(pageProcessor);
+ destroyEach(scheduler);
for (Pipeline pipeline : pipelines) {
destroyEach(pipeline);
}
diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/FileCacheQueueScheduler.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/FileCacheQueueScheduler.java
index 407200a..1582fc9 100644
--- a/webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/FileCacheQueueScheduler.java
+++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/FileCacheQueueScheduler.java
@@ -12,17 +12,19 @@ import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
+
/**
* Store urls and cursor in files so that a Spider can resume the status when shutdown.
*
* @author code4crafter@gmail.com
* @since 0.2.0
*/
-public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implements MonitorableScheduler {
+public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implements MonitorableScheduler,Closeable {
private String filePath = System.getProperty("java.io.tmpdir");
@@ -43,6 +45,8 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
private BlockingQueue queue;
private Set urls;
+
+ private ScheduledExecutorService flushThreadPool;
public FileCacheQueueScheduler(String filePath) {
if (!filePath.endsWith("/") && !filePath.endsWith("\\")) {
@@ -94,7 +98,8 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
}
private void initFlushThread() {
- Executors.newScheduledThreadPool(1).scheduleAtFixedRate(new Runnable() {
+ flushThreadPool = Executors.newScheduledThreadPool(1);
+ flushThreadPool.scheduleAtFixedRate(new Runnable() {
@Override
public void run() {
flush();
@@ -161,6 +166,12 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
}
}
}
+
+ public void close() throws IOException {
+ flushThreadPool.shutdown();
+ fileUrlWriter.close();
+ fileCursorWriter.close();
+ }
private String getFileName(String filename) {
return filePath + task.getUUID() + filename;