Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update email

master
yihua.huang 2013-08-03 14:01:18 +08:00
parent 7c9e9ce869
commit 866ab0a056
17 changed files with 110 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
webmagic-core/src/main/java/us/codecraft/webmagic/ResultItems.java
View File

@ -5,7 +5,7 @@ import java.util.Map;
/**
* 保存抽取结果的类，由PageProcessor处理得到，传递给{@link us.codecraft.webmagic.pipeline.Pipeline}进行持久化。<br>
* @author yihua.huang@dianping.com <br>
* @author code4crafter@gmail.com <br>
* @date: 13-7-25 <br>
* Time: 下午12:20 <br>
*/

4
webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
View File

@ -90,10 +90,6 @@ public class Spider implements Runnable, Task {
return new Spider(pageProcessor);
}
public static Spider create(Site site,Class... pageModels) {
return new Spider(ObjectPageProcessor.create(site,pageModels));
}
/**
* 重新设置startUrls，会覆盖Site本身的startUrls。
*

15
webmagic-core/src/main/java/us/codecraft/webmagic/annotation/AfterExtractor.java 100644
View File

@ -0,0 +1,15 @@
package us.codecraft.webmagic.annotation;
import us.codecraft.webmagic.Page;
/**
* 实现这个接口即可在抽取后进行后处理。<br>
*
* @author code4crafter@gmail.com <br>
* @date: 13-8-3 <br>
* Time: 上午9:42 <br>
*/
public interface AfterExtractor<T> {
public void afterProcess(Page page, T t);
}

2
webmagic-core/src/main/java/us/codecraft/webmagic/annotation/ExtractBy.java
View File

@ -5,7 +5,7 @@ import java.lang.annotation.Retention;
import java.lang.annotation.Target;
/**
* @author yihua.huang@dianping.com <br>
* @author code4crafter@gmail.com <br>
* @date: 13-8-1 <br>
* Time: 下午8:40 <br>
*/

2
webmagic-core/src/main/java/us/codecraft/webmagic/annotation/ExtractByUrl.java
View File

@ -5,7 +5,7 @@ import java.lang.annotation.Retention;
import java.lang.annotation.Target;
/**
* @author yihua.huang@dianping.com <br>
* @author code4crafter@gmail.com <br>
* @date: 13-8-1 <br>
* Time: 下午8:40 <br>
*/

2
webmagic-core/src/main/java/us/codecraft/webmagic/annotation/FieldExtractor.java
View File

@ -6,7 +6,7 @@ import java.lang.reflect.Field;
import java.lang.reflect.Method;
/**
* @author yihua.huang@dianping.com <br>
* @author code4crafter@gmail.com <br>
* @date: 13-8-1 <br>
* Time: 下午9:48 <br>
*/

2
webmagic-core/src/main/java/us/codecraft/webmagic/annotation/HelpUrl.java
View File

@ -5,7 +5,7 @@ import java.lang.annotation.Retention;
import java.lang.annotation.Target;
/**
* @author yihua.huang@dianping.com <br>
* @author code4crafter@gmail.com <br>
* @date: 13-8-1 <br>
* Time: 下午8:40 <br>
*/

29
webmagic-core/src/main/java/us/codecraft/webmagic/annotation/OOSpider.java 100644
View File

@ -0,0 +1,29 @@
package us.codecraft.webmagic.annotation;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
/**
* @author code4crafter@gmail.com <br>
* @date: 13-8-3 <br>
* Time: 上午9:51 <br>
*/
public class OOSpider extends Spider{
/**
* 使用已定义的抽取规则新建一个Spider。
*
* @param pageProcessor 已定义的抽取规则
*/
public OOSpider(PageProcessor pageProcessor) {
super(pageProcessor);
}
public static OOSpider create(Site site,Class... pageModels) {
OOSpider ooSpider = new OOSpider(ObjectPageProcessor.create(site, pageModels));
ooSpider.pipeline(new ObjectPipeline());
return ooSpider;
}
}

2
webmagic-core/src/main/java/us/codecraft/webmagic/annotation/ObjectPageProcessor.java
View File

@ -12,7 +12,7 @@ import java.util.Set;
import java.util.regex.Pattern;
/**
* @author yihua.huang@dianping.com <br>
* @author code4crafter@gmail.com <br>
* @date: 13-8-1 <br>
* Time: 下午8:46 <br>
*/

30
webmagic-core/src/main/java/us/codecraft/webmagic/annotation/ObjectPipeline.java
View File

@ -4,18 +4,36 @@ import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.Pipeline;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* @author yihua.huang@dianping.com <br>
* @author code4crafter@gmail.com <br>
* @date: 13-8-2 <br>
* Time: 上午10:47 <br>
*/
public class ObjectPipeline implements Pipeline {
private Map<Class, PageModelPipeline> pageModelPipelines = new ConcurrentHashMap<Class, PageModelPipeline>();
public ObjectPipeline() {
}
public ObjectPipeline put(Class clazz, PageModelPipeline pageModelPipeline) {
pageModelPipelines.put(clazz, pageModelPipeline);
return this;
}
@Override
public void process(ResultItems resultItems, Task task) {
}
public <T> T read() {
return null;
if (resultItems.isSkip()) {
return;
}
for (Map.Entry<Class, PageModelPipeline> classPageModelPipelineEntry : pageModelPipelines.entrySet()) {
Object o = resultItems.get(classPageModelPipelineEntry.getKey().getCanonicalName());
if (o != null) {
classPageModelPipelineEntry.getValue().process(o, task);
}
}
}
}

14
webmagic-core/src/main/java/us/codecraft/webmagic/annotation/PageModelExtractor.java
View File

@ -16,7 +16,7 @@ import java.util.List;
import java.util.regex.Pattern;
/**
* @author yihua.huang@dianping.com <br>
* @author code4crafter@gmail.com <br>
* @date: 13-8-1 <br>
* Time: 下午9:33 <br>
*/
@ -30,6 +30,8 @@ class PageModelExtractor {
private List<FieldExtractor> fieldExtractors;
private AfterExtractor afterExtractor;
public static PageModelExtractor create(Class clazz) {
PageModelExtractor pageModelExtractor = new PageModelExtractor();
pageModelExtractor.init(clazz);
@ -40,6 +42,13 @@ class PageModelExtractor {
this.clazz = clazz;
initTargetUrlPatterns();
fieldExtractors = new ArrayList<FieldExtractor>();
if (clazz.isAssignableFrom(AfterExtractor.class)){
try {
afterExtractor=(AfterExtractor)clazz.newInstance();
} catch (Exception e) {
throw new IllegalArgumentException(e);
}
}
for (Field field : clazz.getDeclaredFields()) {
field.setAccessible(true);
if (!field.getType().isAssignableFrom(String.class)){
@ -147,6 +156,9 @@ class PageModelExtractor {
}
setField(o, fieldExtractor, value);
}
if (afterExtractor!=null){
afterExtractor.afterProcess(page,o);
}
} catch (InstantiationException e) {
e.printStackTrace();
} catch (IllegalAccessException e) {

14
webmagic-core/src/main/java/us/codecraft/webmagic/annotation/PageModelPipeline.java 100644
View File

@ -0,0 +1,14 @@
package us.codecraft.webmagic.annotation;
import us.codecraft.webmagic.Task;
/**
* @author code4crafter@gmail.com <br>
* @date: 13-8-3 <br>
* Time: 上午9:34 <br>
*/
public interface PageModelPipeline<T> {
public void process(T t, Task task);
}

3
webmagic-core/src/main/java/us/codecraft/webmagic/annotation/TargetUrl.java
View File

@ -5,7 +5,7 @@ import java.lang.annotation.Retention;
import java.lang.annotation.Target;
/**
* @author yihua.huang@dianping.com <br>
* @author code4crafter@gmail.com <br>
* @date: 13-8-1 <br>
* Time: 下午8:40 <br>
*/
@ -14,4 +14,5 @@ import java.lang.annotation.Target;
public @interface TargetUrl {
String[] value();
}

2
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/Destroyable.java
View File

@ -2,7 +2,7 @@ package us.codecraft.webmagic.downloader;
/**
* 比较占用资源的服务可以实现该接口，Spider会在结束时调用destroy()释放资源。<br>
* @author yihua.huang@dianping.com <br>
* @author code4crafter@gmail.com <br>
* @date: 13-7-26 <br>
* Time: 下午3:10 <br>
*/

8
webmagic-core/src/test/java/us/codecraft/webmagic/annotation/TestFetcher.java
View File

@ -3,7 +3,6 @@ package us.codecraft.webmagic.annotation;
import org.junit.Ignore;
import org.junit.Test;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
/**
* @author yihua.huang@dianping.com <br>
@ -16,12 +15,9 @@ public class TestFetcher {
@Test
public void test() {
ObjectPipeline objectPipeline = new ObjectPipeline();
Spider.create(ObjectPageProcessor.create(Site.me().addStartUrl("http://my.oschina.net/flashsword/blog/145796"), OschinaBlog.class))
.pipeline(objectPipeline).runAsync();
OOSpider.create(Site.me().addStartUrl("http://my.oschina.net/flashsword/blog/145796"), OschinaBlog.class)
.pipeline(objectPipeline);
OschinaBlog oschinaBlog = null;
while ((oschinaBlog = objectPipeline.read()) != null) {
System.out.println(oschinaBlog);
}
}

4
webmagic-samples/src/main/java/us/codecraft/webmagic/annotation/samples/IteyeBlog.java
View File

@ -1,8 +1,8 @@
package us.codecraft.webmagic.annotation.samples;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.annotation.ExtractBy;
import us.codecraft.webmagic.annotation.OOSpider;
import us.codecraft.webmagic.annotation.TargetUrl;
/**
@ -28,7 +28,7 @@ public class IteyeBlog implements Blog{
}
public static void main(String[] args) {
Spider.create(Site.me().addStartUrl("http://dengminhui.iteye.com/blog"),IteyeBlog.class).run();
OOSpider.create(Site.me().addStartUrl("http://dengminhui.iteye.com/blog"), IteyeBlog.class).run();
}
public String getTitle() {

4
webmagic-samples/src/main/java/us/codecraft/webmagic/annotation/samples/OschinaBlog.java
View File

@ -1,8 +1,8 @@
package us.codecraft.webmagic.annotation.samples;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.annotation.ExtractBy;
import us.codecraft.webmagic.annotation.OOSpider;
import us.codecraft.webmagic.annotation.TargetUrl;
/**
@ -28,7 +28,7 @@ public class OschinaBlog implements Blog{
}
public static void main(String[] args) {
Spider.create(Site.me().addStartUrl("http://my.oschina.net/flashsword/blog"),OschinaBlog.class).run();
OOSpider.create(Site.me().addStartUrl("http://my.oschina.net/flashsword/blog"), OschinaBlog.class).run();
}
public String getTitle() {