webMagic-Mobai
/
magic-Dependency
mirror of https://gitea.qinmian.online/Mobai/magic-Dependency.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

json fix

master
yihua.huang 2013-08-06 22:36:37 +08:00
parent e5f4b3916f
commit c7005a0227
3 changed files with 19 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
webmagic-core/src/main/java/us/codecraft/webmagic/Request.java
View File

@ -40,6 +40,9 @@ public class Request implements Serializable {
private double priority; private double priority;
public Request() {
}
/** /**
* 构建一个request对象 * 构建一个request对象
* *
@ -102,4 +105,12 @@ public class Request implements Serializable {
public int hashCode() { public int hashCode() {
return url.hashCode(); return url.hashCode();
} }
public void setExtras(Map<String, Object> extras) {
this.extras = extras;
}
public void setUrl(String url) {
this.url = url;
}
} }

8
webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/scheduler/RedisScheduler.java
View File

@ -40,8 +40,8 @@ public class RedisScheduler implements Scheduler {
jedis.zadd(SET_PREFIX + task.getUUID(), request.getPriority(), request.getUrl()); jedis.zadd(SET_PREFIX + task.getUUID(), request.getPriority(), request.getUrl());
if (request.getExtras() != null) { if (request.getExtras() != null) {
String key = ITEM_PREFIX + DigestUtils.shaHex(request.getUrl()); String key = ITEM_PREFIX + DigestUtils.shaHex(request.getUrl());
byte[] serialize = JSON.toJSONBytes(request); byte[] bytes = JSON.toJSONString(request).getBytes();
jedis.set(key.getBytes(), serialize); jedis.set(key.getBytes(), bytes);
} }
} }
pool.returnResource(jedis); pool.returnResource(jedis);
@ -57,8 +57,8 @@ public class RedisScheduler implements Scheduler {
String key = ITEM_PREFIX + DigestUtils.shaHex(url); String key = ITEM_PREFIX + DigestUtils.shaHex(url);
byte[] bytes = jedis.get(key.getBytes()); byte[] bytes = jedis.get(key.getBytes());
if (bytes != null) { if (bytes != null) {
Object o = JSON.parse(bytes); Request o = JSON.parseObject(new String(bytes),Request.class);
return (Request) o; return o;
} }
pool.returnResource(jedis); pool.returnResource(jedis);
return new Request(url); return new Request(url);

6
webmagic-plugin/webmagic-misc/src/test/java/us/codecraft/webmagic/scheduler/RedisSchedulerTest.java
View File

@ -1,7 +1,6 @@
package us.codecraft.webmagic.scheduler; package us.codecraft.webmagic.scheduler;
import org.junit.Before; import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import us.codecraft.webmagic.Request; import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site; import us.codecraft.webmagic.Site;
@ -35,8 +34,11 @@ public class RedisSchedulerTest {
return null; return null;
} }
}; };
redisScheduler.push(new Request("http://www.ibm.com/developerworks/cn/java/j-javadev2-22/"), task); Request request = new Request("http://www.ibm.com/developerworks/cn/java/j-javadev2-22/");
request.putExtra("1","2");
redisScheduler.push(request, task);
Request poll = redisScheduler.poll(task); Request poll = redisScheduler.poll(task);
System.out.println(poll);
} }
} }