Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixed #301 修复使用注解抽取JSON数据的问题

master
郭玉昆 2016-08-29 17:07:37 +08:00
parent e22d6426fc
commit 700898fe8a
4 changed files with 32 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
webmagic-extension/src/main/java/us/codecraft/webmagic/example/GithubRepoApi.java
View File

@ -15,19 +15,19 @@ import java.util.List;
*/
public class GithubRepoApi implements HasKey {
@ExtractBy(type = ExtractBy.Type.JsonPath, value = "$.name")
@ExtractBy(type = ExtractBy.Type.JsonPath, value = "$.name", source = ExtractBy.Source.RawText)
private String name;
@ExtractBy(type = ExtractBy.Type.JsonPath, value = "$..owner.login")
@ExtractBy(type = ExtractBy.Type.JsonPath, value = "$..owner.login", source = ExtractBy.Source.RawText)
private String author;
@ExtractBy(type = ExtractBy.Type.JsonPath, value = "$.language",multi = true)
@ExtractBy(type = ExtractBy.Type.JsonPath, value = "$.language",multi = true, source = ExtractBy.Source.RawText)
private List<String> language;
@ExtractBy(type = ExtractBy.Type.JsonPath, value = "$.stargazers_count")
@ExtractBy(type = ExtractBy.Type.JsonPath, value = "$.stargazers_count", source = ExtractBy.Source.RawText)
private int star;
@ExtractBy(type = ExtractBy.Type.JsonPath, value = "$.homepage")
@ExtractBy(type = ExtractBy.Type.JsonPath, value = "$.forks_count", source = ExtractBy.Source.RawText)
private int fork;
@ExtractByUrl

2
webmagic-extension/src/main/java/us/codecraft/webmagic/model/Extractor.java
View File

@ -17,7 +17,7 @@ class Extractor {
protected final boolean multi;
static enum Source {Html, Url, RawHtml}
static enum Source {Html, Url, RawHtml, RawText}
public Extractor(Selector selector, Source source, boolean notNull, boolean multi) {
this.selector = selector;

25
webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelExtractor.java
View File

@ -179,7 +179,24 @@ class PageModelExtractor {
ExtractBy extractBy = field.getAnnotation(ExtractBy.class);
if (extractBy != null) {
Selector selector = ExtractorUtils.getSelector(extractBy);
fieldExtractor = new FieldExtractor(field, selector, extractBy.source() == ExtractBy.Source.RawHtml ? FieldExtractor.Source.RawHtml : FieldExtractor.Source.Html,
FieldExtractor.Source source = null;
switch (extractBy.source()){
case RawText:
source = FieldExtractor.Source.RawText;
break;
case RawHtml:
source = FieldExtractor.Source.RawHtml;
break;
case SelectedHtml:
source =FieldExtractor.Source.Html;
break;
default:
source =FieldExtractor.Source.Html;
}
fieldExtractor = new FieldExtractor(field, selector, source,
extractBy.notNull(), extractBy.multi() || List.class.isAssignableFrom(field.getType()));
Method setterMethod = getSetterMethod(clazz, field);
if (setterMethod != null) {
@ -284,6 +301,9 @@ class PageModelExtractor {
case Url:
value = fieldExtractor.getSelector().selectList(page.getUrl().toString());
break;
case RawText:
value = fieldExtractor.getSelector().selectList(page.getRawText());
break;
default:
value = fieldExtractor.getSelector().selectList(html);
}
@ -312,6 +332,9 @@ class PageModelExtractor {
case Url:
value = fieldExtractor.getSelector().select(page.getUrl().toString());
break;
case RawText:
value = fieldExtractor.getSelector().select(page.getRawText());
break;
default:
value = fieldExtractor.getSelector().select(html);
}

3
webmagic-extension/src/main/java/us/codecraft/webmagic/model/annotation/ExtractBy.java
View File

@ -52,7 +52,8 @@ public @interface ExtractBy {
/**
* extract from the raw html
*/
RawHtml
RawHtml,
RawText
}
/**