Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

change serilizer to fastjson

master
yihua.huang 2013-08-06 22:25:27 +08:00
parent 5436ecbb7b
commit a2d830f7b0
4 changed files with 67 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
webmagic-plugin/webmagic-misc/pom.xml
View File

@ -13,9 +13,9 @@
<dependencies>
<dependency>
<groupId>org.resthub</groupId>
<artifactId>hessian</artifactId>
<version>4.0.8</version>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.1.35</version>
</dependency>
<dependency>
<groupId>redis.clients</groupId>

58
webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/pipeline/JsonFilePipeline.java 100644
View File

@ -0,0 +1,58 @@
package us.codecraft.webmagic.pipeline;
import com.alibaba.fastjson.JSON;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.log4j.Logger;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
/**
* 持久化到文件的接口。
*
* @author code4crafter@gmail.com <br>
* Date: 13-4-21
* Time: 下午6:28
*/
public class JsonFilePipeline implements Pipeline {
private String path = "/data/temp/webmagic/";
private Logger logger = Logger.getLogger(getClass());
/**
* 新建一个FilePipeline，使用默认保存路径"/data/temp/webmagic/"
*/
public JsonFilePipeline() {
}
/**
* 新建一个FilePipeline
*
* @param path 文件保存路径
*/
public JsonFilePipeline(String path) {
this.path = path;
}
@Override
public void process(ResultItems resultItems, Task task) {
String path = this.path + "/" + task.getUUID() + "/";
File file = new File(path);
if (!file.exists()) {
file.mkdirs();
}
try {
PrintWriter printWriter = new PrintWriter(new FileWriter(path + DigestUtils.md5Hex(resultItems.getRequest().getUrl()) + ".json"));
printWriter.write(JSON.toJSONString(resultItems.getAll()));
printWriter.close();
} catch (IOException e) {
logger.warn("write file error", e);
}
}
}

33
webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/scheduler/HessianSerializer.java
View File

@ -1,33 +0,0 @@
package us.codecraft.webmagic.scheduler;
import com.caucho.hessian.io.Hessian2Input;
import com.caucho.hessian.io.Hessian2Output;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
/**
* @author code4crafter@gmail.com <br>
* @date: 13-7-14 <br>
* Time: 下午9:20 <br>
*/
public enum HessianSerializer {
INSTANCE;
public <T> byte[] serialize(T v) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
Hessian2Output hessian2Output = new Hessian2Output(baos);
hessian2Output.writeObject(v);
hessian2Output.close();
return baos.toByteArray();
}
@SuppressWarnings("unchecked")
public <T> T deSerialize(byte[] bytes) throws IOException {
ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
Hessian2Input hessian2Input = new Hessian2Input(bais);
T t = (T) hessian2Input.readObject();
hessian2Input.close();
return t;
}
}

19
webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/scheduler/RedisScheduler.java
View File

@ -1,5 +1,6 @@
package us.codecraft.webmagic.scheduler;
import com.alibaba.fastjson.JSON;
import org.apache.commons.codec.digest.DigestUtils;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
@ -8,8 +9,6 @@ import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.schedular.Scheduler;
import java.io.IOException;
/**
* 使用redis管理url，构建一个分布式的爬虫。<br>
*
@ -41,12 +40,8 @@ public class RedisScheduler implements Scheduler {
jedis.zadd(SET_PREFIX + task.getUUID(), request.getPriority(), request.getUrl());
if (request.getExtras() != null) {
String key = ITEM_PREFIX + DigestUtils.shaHex(request.getUrl());
try {
byte[] serialize = HessianSerializer.INSTANCE.serialize(request);
byte[] serialize = JSON.toJSONBytes(request);
jedis.set(key.getBytes(), serialize);
} catch (IOException e) {
e.printStackTrace();
}
}
}
pool.returnResource(jedis);
@ -61,13 +56,9 @@ public class RedisScheduler implements Scheduler {
}
String key = ITEM_PREFIX + DigestUtils.shaHex(url);
byte[] bytes = jedis.get(key.getBytes());
if (bytes!=null){
try {
Object o = HessianSerializer.INSTANCE.deSerialize(bytes);
return (Request)o;
} catch (Exception e) {
e.printStackTrace();
}
if (bytes != null) {
Object o = JSON.parse(bytes);
return (Request) o;
}
pool.returnResource(jedis);
return new Request(url);