Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #161 from zhugw/patch-4 

Update FileCacheQueueScheduler.java
master
Yihua Huang 2014-09-29 14:46:35 +08:00
parent 3a9c1d3002 eb3c78b9d8
commit 20422f1b63
1 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/FileCacheQueueScheduler.java
View File

@ -49,6 +49,7 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
filePath += "/";
}
this.filePath = filePath;
initDuplicateRemover();
}
private void flush() {
@ -74,6 +75,9 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
new DuplicateRemover() {
@Override
public boolean isDuplicate(Request request, Task task) {
if (!inited.get()) {
init(task);
}
return !urls.add(request.getUrl());
}
@ -113,7 +117,7 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
urls = new LinkedHashSet<String>();
readCursorFile();
readUrlFile();
initDuplicateRemover();
// initDuplicateRemover();
} catch (FileNotFoundException e) {
//init
logger.info("init cache file " + getFileName(fileUrlAllName));
@ -164,9 +168,9 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
@Override
protected void pushWhenNoDuplicate(Request request, Task task) {
if (!inited.get()) {
/* if (!inited.get()) {
init(task);
}
}*/
queue.add(request);
fileUrlWriter.println(request.getUrl());
}