update request
parent
d56c681be1
commit
dc9f574e27
|
@ -1,5 +1,8 @@
|
||||||
package us.codecraft.webmagic;
|
package us.codecraft.webmagic;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Request对象封装了待抓取的url信息。<br/>
|
* Request对象封装了待抓取的url信息。<br/>
|
||||||
* 在PageProcessor中,Request对象可以通过{@link us.codecraft.webmagic.Page#getRequest()} 获取。<br/>
|
* 在PageProcessor中,Request对象可以通过{@link us.codecraft.webmagic.Page#getRequest()} 获取。<br/>
|
||||||
|
@ -18,6 +21,7 @@ package us.codecraft.webmagic;
|
||||||
* String linktext = (String)page.getRequest().getExtra()[0];
|
* String linktext = (String)page.getRequest().getExtra()[0];
|
||||||
* }
|
* }
|
||||||
* </pre>
|
* </pre>
|
||||||
|
*
|
||||||
* @author code4crafter@gmail.com <br>
|
* @author code4crafter@gmail.com <br>
|
||||||
* Date: 13-4-21
|
* Date: 13-4-21
|
||||||
* Time: 上午11:37
|
* Time: 上午11:37
|
||||||
|
@ -26,18 +30,20 @@ public class Request {
|
||||||
|
|
||||||
private String url;
|
private String url;
|
||||||
|
|
||||||
private Object[] extra;
|
/**
|
||||||
|
* 额外参数,可以保存一些需要的上下文信息
|
||||||
|
*/
|
||||||
|
private Map<String, Object> extras = new HashMap<String, Object>();
|
||||||
|
|
||||||
private double priority;
|
private double priority;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 构建一个request对象
|
* 构建一个request对象
|
||||||
|
*
|
||||||
* @param url 必须参数,待抓取的url
|
* @param url 必须参数,待抓取的url
|
||||||
* @param extra 额外参数,可以保存一些需要的上下文信息
|
|
||||||
*/
|
*/
|
||||||
public Request(String url, Object... extra) {
|
public Request(String url) {
|
||||||
this.url = url;
|
this.url = url;
|
||||||
this.extra = extra;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public double getPriority() {
|
public double getPriority() {
|
||||||
|
@ -49,16 +55,18 @@ public class Request {
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
public Object getExtra(String key) {
|
||||||
* 获取预存的对象
|
return extras.get(key);
|
||||||
* @return object[] 预存的对象数组
|
}
|
||||||
*/
|
|
||||||
public Object[] getExtra() {
|
public Request putExtra(String key,Object value) {
|
||||||
return extra;
|
extras.put(key,value);
|
||||||
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 获取待抓取的url
|
* 获取待抓取的url
|
||||||
|
*
|
||||||
* @return url 待抓取的url
|
* @return url 待抓取的url
|
||||||
*/
|
*/
|
||||||
public String getUrl() {
|
public String getUrl() {
|
||||||
|
|
|
@ -33,7 +33,7 @@ public class RedisScheduler implements Scheduler {
|
||||||
if (jedis.zrank(SET_PREFIX + task.getUUID(), request.getUrl()) == null) {
|
if (jedis.zrank(SET_PREFIX + task.getUUID(), request.getUrl()) == null) {
|
||||||
//使用List保存队列
|
//使用List保存队列
|
||||||
jedis.rpush(QUEUE_PREFIX + task.getUUID(), request.getUrl());
|
jedis.rpush(QUEUE_PREFIX + task.getUUID(), request.getUrl());
|
||||||
jedis.zadd(SET_PREFIX + task.getUUID(), System.currentTimeMillis(), request.getUrl());
|
jedis.zadd(SET_PREFIX + task.getUUID(), request.getPriority(), request.getUrl());
|
||||||
}
|
}
|
||||||
pool.returnResource(jedis);
|
pool.returnResource(jedis);
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue