From 486d9d276ff5b580af9788e7fb81e118821c4368 Mon Sep 17 00:00:00 2001 From: "yihua.huang" Date: Thu, 28 Nov 2013 18:23:51 +0800 Subject: [PATCH] #45 Remove multi in ExtractBy --- .../java/us/codecraft/webmagic/example/AppStore.java | 6 +++++- .../codecraft/webmagic/model/PageModelExtractor.java | 10 ++++++---- .../webmagic/model/annotation/ComboExtract.java | 2 ++ .../codecraft/webmagic/model/annotation/ExtractBy.java | 2 ++ .../webmagic/model/annotation/ExtractByUrl.java | 2 ++ 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/example/AppStore.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/example/AppStore.java index fcc937b..d0d056f 100644 --- a/webmagic-extension/src/main/java/us/codecraft/webmagic/example/AppStore.java +++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/example/AppStore.java @@ -23,14 +23,18 @@ public class AppStore { @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$..userRatingCount") private int userRatingCount; - @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$..screenshotUrls",multi = true) + @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$..screenshotUrls") private List screenshotUrls; + @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$..supportedDevices") + private List supportedDevices; + public static void main(String[] args) { AppStore appStore = OOSpider.create(Site.me(), AppStore.class).get("http://itunes.apple.com/lookup?id=653350791&country=cn&entity=software"); System.out.println(appStore.trackName); System.out.println(appStore.description); System.out.println(appStore.userRatingCount); System.out.println(appStore.screenshotUrls); + System.out.println(appStore.supportedDevices); } } diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelExtractor.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelExtractor.java index d7da0c9..62b6de0 100644 --- a/webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelExtractor.java +++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelExtractor.java @@ -131,7 +131,9 @@ class PageModelExtractor { if (regexPattern.trim().equals("")) { regexPattern = ".*"; } - fieldExtractor = new FieldExtractor(field, new RegexSelector(regexPattern), FieldExtractor.Source.Url, extractByUrl.notNull(), extractByUrl.multi()); + fieldExtractor = new FieldExtractor(field, + new RegexSelector(regexPattern), FieldExtractor.Source.Url, extractByUrl.notNull(), + extractByUrl.multi() || List.class.isAssignableFrom(field.getType())); Method setterMethod = getSetterMethod(clazz, field); if (setterMethod != null) { fieldExtractor.setSetterMethod(setterMethod); @@ -157,7 +159,7 @@ class PageModelExtractor { selector = new AndSelector(ExtractorUtils.getSelectors(extractBies)); } fieldExtractor = new FieldExtractor(field, selector, comboExtract.source() == ComboExtract.Source.RawHtml ? FieldExtractor.Source.RawHtml : FieldExtractor.Source.Html, - comboExtract.notNull(), comboExtract.multi()); + comboExtract.notNull(), comboExtract.multi() || List.class.isAssignableFrom(field.getType())); Method setterMethod = getSetterMethod(clazz, field); if (setterMethod != null) { fieldExtractor.setSetterMethod(setterMethod); @@ -172,7 +174,7 @@ class PageModelExtractor { if (extractBy != null) { Selector selector = ExtractorUtils.getSelector(extractBy); fieldExtractor = new FieldExtractor(field, selector, extractBy.source() == ExtractBy.Source.RawHtml ? FieldExtractor.Source.RawHtml : FieldExtractor.Source.Html, - extractBy.notNull(), extractBy.multi()); + extractBy.notNull(), extractBy.multi() || List.class.isAssignableFrom(field.getType())); Method setterMethod = getSetterMethod(clazz, field); if (setterMethod != null) { fieldExtractor.setSetterMethod(setterMethod); @@ -359,7 +361,7 @@ class PageModelExtractor { } private void setField(Object o, FieldExtractor fieldExtractor, Object value) throws IllegalAccessException, InvocationTargetException { - if (value==null){ + if (value == null) { return; } if (fieldExtractor.getSetterMethod() != null) { diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/model/annotation/ComboExtract.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/annotation/ComboExtract.java index 5268a25..6d2ce6c 100644 --- a/webmagic-extension/src/main/java/us/codecraft/webmagic/model/annotation/ComboExtract.java +++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/annotation/ComboExtract.java @@ -75,6 +75,8 @@ public @interface ComboExtract { * Define whether the extractor return more than one result. * When set to 'true', the extractor return a list of string (so you should define the field as List).
* + * Deprecated since 0.4.2. This option is determined automatically by the class of field. + * @deprecated since 0.4.2 * @return whether the extractor return more than one result */ boolean multi() default false; diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/model/annotation/ExtractBy.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/annotation/ExtractBy.java index 8fddccf..2e23aa0 100644 --- a/webmagic-extension/src/main/java/us/codecraft/webmagic/model/annotation/ExtractBy.java +++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/annotation/ExtractBy.java @@ -67,6 +67,8 @@ public @interface ExtractBy { * Define whether the extractor return more than one result. * When set to 'true', the extractor return a list of string (so you should define the field as List).
* + * Deprecated since 0.4.2. This option is determined automatically by the class of field. + * @deprecated since 0.4.2 * @return whether the extractor return more than one result */ boolean multi() default false; diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/model/annotation/ExtractByUrl.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/annotation/ExtractByUrl.java index 328c079..6c77862 100644 --- a/webmagic-extension/src/main/java/us/codecraft/webmagic/model/annotation/ExtractByUrl.java +++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/annotation/ExtractByUrl.java @@ -33,6 +33,8 @@ public @interface ExtractByUrl { * Define whether the extractor return more than one result. * When set to 'true', the extractor return a list of string (so you should define the field as List).
* + * Deprecated since 0.4.2. This option is determined automatically by the class of field. + * @deprecated since 0.4.2 * @return whether the extractor return more than one result */ boolean multi() default false;