Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add setter support

master
yihua.huang 2013-08-02 07:32:37 +08:00
parent abba3b7bff
commit 06a39af0f3
2 changed files with 52 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
webmagic-core/src/main/java/us/codecraft/webmagic/annotation/FieldExtractor.java
View File

@ -3,6 +3,7 @@ package us.codecraft.webmagic.annotation;
import us.codecraft.webmagic.selector.Selector; import us.codecraft.webmagic.selector.Selector;
import java.lang.reflect.Field; import java.lang.reflect.Field;
import java.lang.reflect.Method;
/** /**
* @author yihua.huang@dianping.com <br> * @author yihua.huang@dianping.com <br>
@ -17,6 +18,8 @@ class FieldExtractor {
private final Source source; private final Source source;
private Method setterMethod;
static enum Source {Html, Url} static enum Source {Html, Url}
public FieldExtractor(Field field, Selector selector) { public FieldExtractor(Field field, Selector selector) {
@ -42,4 +45,12 @@ class FieldExtractor {
Source getSource() { Source getSource() {
return source; return source;
} }
void setSetterMethod(Method setterMethod) {
this.setterMethod = setterMethod;
}
Method getSetterMethod() {
return setterMethod;
}
} }

46
webmagic-core/src/main/java/us/codecraft/webmagic/annotation/PageModelExtractor.java
View File

@ -1,5 +1,6 @@
package us.codecraft.webmagic.annotation; package us.codecraft.webmagic.annotation;
import org.apache.commons.lang3.StringUtils;
import us.codecraft.webmagic.Page; import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.CssSelector; import us.codecraft.webmagic.selector.CssSelector;
import us.codecraft.webmagic.selector.RegexSelector; import us.codecraft.webmagic.selector.RegexSelector;
@ -8,6 +9,8 @@ import us.codecraft.webmagic.selector.XpathSelector;
import java.lang.annotation.Annotation; import java.lang.annotation.Annotation;
import java.lang.reflect.Field; import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.regex.Pattern; import java.util.regex.Pattern;
@ -54,7 +57,12 @@ class PageModelExtractor {
default: default:
selector = new XpathSelector(value); selector = new XpathSelector(value);
} }
fieldExtractors.add(new FieldExtractor(field, selector)); FieldExtractor fieldExtractor = new FieldExtractor(field, selector);
Method setterMethod = getSetterMethod(clazz, field);
if (setterMethod != null) {
fieldExtractor.setSetterMethod(setterMethod);
}
fieldExtractors.add(fieldExtractor);
} }
ExtractByUrl extractByUrl = field.getAnnotation(ExtractByUrl.class); ExtractByUrl extractByUrl = field.getAnnotation(ExtractByUrl.class);
if (extractByUrl != null) { if (extractByUrl != null) {
@ -62,8 +70,24 @@ class PageModelExtractor {
if (regexPattern.trim().equals("")) { if (regexPattern.trim().equals("")) {
regexPattern = ".*"; regexPattern = ".*";
} }
fieldExtractors.add(new FieldExtractor(field, new RegexSelector(regexPattern), FieldExtractor.Source.Url)); FieldExtractor fieldExtractor = new FieldExtractor(field, new RegexSelector(regexPattern), FieldExtractor.Source.Url);
Method setterMethod = getSetterMethod(clazz, field);
if (setterMethod != null) {
fieldExtractor.setSetterMethod(setterMethod);
} }
fieldExtractors.add(fieldExtractor);
}
}
}
public static Method getSetterMethod(Class clazz, Field field) {
String name = "set" + StringUtils.capitalize(field.getName());
try {
Method declaredMethod = clazz.getDeclaredMethod(name, field.getType());
declaredMethod.setAccessible(true);
return declaredMethod;
} catch (NoSuchMethodException e) {
return null;
} }
} }
@ -94,24 +118,36 @@ class PageModelExtractor {
try { try {
o = clazz.newInstance(); o = clazz.newInstance();
for (FieldExtractor fieldExtractor : fieldExtractors) { for (FieldExtractor fieldExtractor : fieldExtractors) {
String value;
switch (fieldExtractor.getSource()) { switch (fieldExtractor.getSource()) {
case Html: case Html:
fieldExtractor.getField().set(o, fieldExtractor.getSelector().select(page.getHtml().toString())); value = fieldExtractor.getSelector().select(page.getHtml().toString());
break; break;
case Url: case Url:
fieldExtractor.getField().set(o, fieldExtractor.getSelector().select(page.getUrl().toString())); value = fieldExtractor.getSelector().select(page.getUrl().toString());
break; break;
default:
value = fieldExtractor.getSelector().select(page.getHtml().toString());
} }
setField(o,fieldExtractor,value);
} }
} catch (InstantiationException e) { } catch (InstantiationException e) {
e.printStackTrace(); e.printStackTrace();
} catch (IllegalAccessException e) { } catch (IllegalAccessException e) {
e.printStackTrace(); e.printStackTrace();
} catch (InvocationTargetException e) {
e.printStackTrace();
} }
return o; return o;
} }
private void setField(Object o, FieldExtractor fieldExtractor, String value) throws IllegalAccessException, InvocationTargetException {
if (fieldExtractor.getSetterMethod()!=null){
fieldExtractor.getSetterMethod().invoke(o,value);
}
fieldExtractor.getField().set(o, value);
}
Class getClazz() { Class getClazz() {
return clazz; return clazz;
} }