Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

change check-and-set to atomic sadd for redis DuplicateRemover #368

master
yihua.huang 2017-03-01 22:24:34 +08:00
parent d6cd92b1a8
commit d87c73b472
1 changed files with 1 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/RedisScheduler.java
View File

@ -48,11 +48,7 @@ public class RedisScheduler extends DuplicateRemovedScheduler implements Monitor
public boolean isDuplicate(Request request, Task task) { public boolean isDuplicate(Request request, Task task) {
Jedis jedis = pool.getResource(); Jedis jedis = pool.getResource();
try { try {
boolean isDuplicate = jedis.sismember(getSetKey(task), request.getUrl()); return jedis.sadd(getSetKey(task), request.getUrl()) > 0;
if (!isDuplicate) {
jedis.sadd(getSetKey(task), request.getUrl());
}
return isDuplicate;
} finally { } finally {
pool.returnResource(jedis); pool.returnResource(jedis);
} }