Merge pull request #157 from zhugw/patch-1

Update FileCacheQueueScheduler.java
2014-09-12 07:37:56 +08:00 · 2014-09-12 07:37:56 +08:00 · 689e89a9b2
parent 147401ce5e 1db940a088
commit 689e89a9b2
1 changed files with 2 additions and 0 deletions
--- a/webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/FileCacheQueueScheduler.java
+++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/FileCacheQueueScheduler.java
@ -145,6 +145,8 @@ public class FileCacheQueueScheduler extends DuplicateRemovedScheduler implement
        if (!inited.get()) {
            init(task);
        }
+        if(urls.contains(request.getUrl())) //已存在此URL 表示已抓取过 跳过
+            return;
        queue.add(request);
        fileUrlWriter.println(request.getUrl());
    }