webMagic-Mobai
/
magic-Dependency
mirror of https://gitea.qinmian.online/Mobai/magic-Dependency.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #161 from zhugw/patch-4 

Update FileCacheQueueScheduler.java
master
Yihua Huang 2014-09-29 14:46:35 +08:00
parent 3a9c1d3002 eb3c78b9d8
commit 20422f1b63
1 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/FileCacheQueueScheduler.java
View File

@ -49,6 +49,7 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
filePath += "/"; filePath += "/";
} }
this.filePath = filePath; this.filePath = filePath;
initDuplicateRemover();
} }
private void flush() { private void flush() {
@ -74,6 +75,9 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
new DuplicateRemover() { new DuplicateRemover() {
@Override @Override
public boolean isDuplicate(Request request, Task task) { public boolean isDuplicate(Request request, Task task) {
if (!inited.get()) {
init(task);
}
return !urls.add(request.getUrl()); return !urls.add(request.getUrl());
} }
@ -113,7 +117,7 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
urls = new LinkedHashSet<String>(); urls = new LinkedHashSet<String>();
readCursorFile(); readCursorFile();
readUrlFile(); readUrlFile();
initDuplicateRemover(); // initDuplicateRemover();
} catch (FileNotFoundException e) { } catch (FileNotFoundException e) {
//init //init
logger.info("init cache file " + getFileName(fileUrlAllName)); logger.info("init cache file " + getFileName(fileUrlAllName));
@ -164,9 +168,9 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
@Override @Override
protected void pushWhenNoDuplicate(Request request, Task task) { protected void pushWhenNoDuplicate(Request request, Task task) {
if (!inited.get()) { /* if (!inited.get()) {
init(task); init(task);
} }*/
queue.add(request); queue.add(request);
fileUrlWriter.println(request.getUrl()); fileUrlWriter.println(request.getUrl());
} }