Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

rename

master
yihua.huang 2013-08-02 06:33:48 +08:00
parent c5cf05640a
commit f08ffc34fd
5 changed files with 30 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
webmagic-core/src/main/java/us/codecraft/webmagic/annotation/Fetcher.java → webmagic-core/src/main/java/us/codecraft/webmagic/annotation/ExtractBy.java
View File

@ -11,7 +11,7 @@ import java.lang.annotation.Target;
*/
@Retention(java.lang.annotation.RetentionPolicy.RUNTIME)
@Target({ElementType.FIELD})
public @interface Fetcher {
public @interface ExtractBy {
String value();

4
webmagic-core/src/main/java/us/codecraft/webmagic/annotation/FieldFetcher.java → webmagic-core/src/main/java/us/codecraft/webmagic/annotation/FieldExtractor.java
View File

@ -9,13 +9,13 @@ import java.lang.reflect.Field;
* @date: 13-8-1 <br>
* Time: 下午9:48 <br>
*/
class FieldFetcher {
class FieldExtractor {
private final Field field;
private final Selector selector;
FieldFetcher(Field field, Selector selector) {
FieldExtractor(Field field, Selector selector) {
this.field = field;
this.selector = selector;
}

24
webmagic-core/src/main/java/us/codecraft/webmagic/annotation/ObjectPageProcessor.java
View File

@ -18,36 +18,36 @@ import java.util.regex.Pattern;
*/
public class ObjectPageProcessor implements PageProcessor {
private List<PageModelFetcher> pageModelFetcherList;
private List<PageModelExtractor> pageModelExtractorList;
private Site site;
private Set<Pattern> targetUrlPatterns;
public static ObjectPageProcessor create(Site site, Class... clazzs) {
List<PageModelFetcher> pageModelFetcherList = new ArrayList<PageModelFetcher>();
List<PageModelExtractor> pageModelExtractorList = new ArrayList<PageModelExtractor>();
for (Class clazz : clazzs) {
PageModelFetcher pageModelFetcher = PageModelFetcher.create(clazz);
pageModelFetcherList.add(pageModelFetcher);
PageModelExtractor pageModelExtractor = PageModelExtractor.create(clazz);
pageModelExtractorList.add(pageModelExtractor);
}
ObjectPageProcessor objectPageProcessor = new ObjectPageProcessor(site, pageModelFetcherList);
ObjectPageProcessor objectPageProcessor = new ObjectPageProcessor(site, pageModelExtractorList);
return objectPageProcessor;
}
private ObjectPageProcessor(Site site, List<PageModelFetcher> pageModelFetcherList) {
private ObjectPageProcessor(Site site, List<PageModelExtractor> pageModelExtractorList) {
this.site = site;
this.pageModelFetcherList = pageModelFetcherList;
this.pageModelExtractorList = pageModelExtractorList;
targetUrlPatterns = new HashSet<Pattern>();
for (PageModelFetcher pageModelFetcher : pageModelFetcherList) {
targetUrlPatterns.addAll(pageModelFetcher.getTargetUrlPatterns());
for (PageModelExtractor pageModelExtractor : pageModelExtractorList) {
targetUrlPatterns.addAll(pageModelExtractor.getTargetUrlPatterns());
}
}
@Override
public void process(Page page) {
for (PageModelFetcher pageModelFetcher : pageModelFetcherList) {
Object process = pageModelFetcher.process(page);
page.putField(pageModelFetcher.getClazz().getCanonicalName(), process);
for (PageModelExtractor pageModelExtractor : pageModelExtractorList) {
Object process = pageModelExtractor.process(page);
page.putField(pageModelExtractor.getClazz().getCanonicalName(), process);
}
for (String link : page.getHtml().links().all()) {
for (Pattern targetUrlPattern : targetUrlPatterns) {

26
webmagic-core/src/main/java/us/codecraft/webmagic/annotation/PageModelFetcher.java → webmagic-core/src/main/java/us/codecraft/webmagic/annotation/PageModelExtractor.java
View File

@ -17,30 +17,30 @@ import java.util.regex.Pattern;
* @date: 13-8-1 <br>
* Time: 下午9:33 <br>
*/
class PageModelFetcher {
class PageModelExtractor {
private List<Pattern> targetUrlPatterns;
private Class clazz;
private List<FieldFetcher> fieldFetchers;
private List<FieldExtractor> fieldExtractors;
public static PageModelFetcher create(Class clazz) {
PageModelFetcher pageModelFetcher = new PageModelFetcher();
pageModelFetcher.init(clazz);
return pageModelFetcher;
public static PageModelExtractor create(Class clazz) {
PageModelExtractor pageModelExtractor = new PageModelExtractor();
pageModelExtractor.init(clazz);
return pageModelExtractor;
}
private void init(Class clazz) {
this.clazz = clazz;
initTargetUrlPatterns();
fieldFetchers = new ArrayList<FieldFetcher>();
fieldExtractors = new ArrayList<FieldExtractor>();
for (Field field : clazz.getDeclaredFields()) {
field.setAccessible(true);
Fetcher fetcher = field.getAnnotation(Fetcher.class);
String value = fetcher.value();
ExtractBy extractBy = field.getAnnotation(ExtractBy.class);
String value = extractBy.value();
Selector selector;
switch (fetcher.type()) {
switch (extractBy.type()) {
case Css:
selector = new CssSelector(value);
break;
@ -53,7 +53,7 @@ class PageModelFetcher {
default:
selector = new XpathSelector(value);
}
fieldFetchers.add(new FieldFetcher(field, selector));
fieldExtractors.add(new FieldExtractor(field, selector));
}
}
@ -83,8 +83,8 @@ class PageModelFetcher {
Object o = null;
try {
o = clazz.newInstance();
for (FieldFetcher fieldFetcher : fieldFetchers) {
fieldFetcher.getField().set(o, fieldFetcher.getSelector().select(page.getHtml().toString()));
for (FieldExtractor fieldExtractor : fieldExtractors) {
fieldExtractor.getField().set(o, fieldExtractor.getSelector().select(page.getHtml().toString()));
}
} catch (InstantiationException e) {
e.printStackTrace();

4
webmagic-core/src/test/java/us/codecraft/webmagic/annotation/Blog.java
View File

@ -8,10 +8,10 @@ package us.codecraft.webmagic.annotation;
@TargetUrl("http://my.oschina.net/flashsword/blog/*")
public class Blog {
@Fetcher("//title")
@ExtractBy("//title")
private String title;
@Fetcher(value = "div.BlogContent",type = Fetcher.Type.Css)
@ExtractBy(value = "div.BlogContent",type = ExtractBy.Type.Css)
private String content;
@Override