Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add FileCacheQueueScheduler support for cycleRetryTimes

master
yihua.huang 2014-04-07 11:00:09 +08:00
parent 969ad1766b
commit c1e7207869
1 changed files with 4 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/FileCacheQueueScheduler.java
View File

@ -23,7 +23,7 @@ import java.util.concurrent.atomic.AtomicInteger;
* @author code4crafter@gmail.com <br> * @author code4crafter@gmail.com <br>
* @since 0.2.0 * @since 0.2.0
*/ */
public class FileCacheQueueScheduler implements Scheduler { public class FileCacheQueueScheduler extends LocalDuplicatedRemovedScheduler {
private Logger logger = LoggerFactory.getLogger(getClass()); private Logger logger = LoggerFactory.getLogger(getClass());
@ -145,18 +145,14 @@ public class FileCacheQueueScheduler implements Scheduler {
} }
@Override @Override
public synchronized void push(Request request, Task task) { protected void pushWhenNoDuplicate(Request request, Task task) {
if (!inited.get()) { if (!inited.get()) {
init(task); init(task);
} }
logger.debug("push to queue {}" , request.getUrl());
if (urls.add(request.getUrl())) {
queue.add(request); queue.add(request);
fileUrlWriter.println(request.getUrl()); fileUrlWriter.println(request.getUrl());
} }
}
@Override @Override
public synchronized Request poll(Task task) { public synchronized Request poll(Task task) {
if (!inited.get()) { if (!inited.get()) {