diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/model/ModelPageProcessor.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/ModelPageProcessor.java index 8a40dae..bc2afcf 100644 --- a/webmagic-extension/src/main/java/us/codecraft/webmagic/model/ModelPageProcessor.java +++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/ModelPageProcessor.java @@ -55,11 +55,14 @@ class ModelPageProcessor implements PageProcessor { extractLinks(page, pageModelExtractor.getTargetUrlRegionSelector(), pageModelExtractor.getTargetUrlPatterns()); Object process = pageModelExtractor.process(page); if (process == null || (process instanceof List && ((List) process).size() == 0)) { - page.getResultItems().setSkip(true); + continue; } postProcessPageModel(pageModelExtractor.getClazz(), process); page.putField(pageModelExtractor.getClazz().getCanonicalName(), process); } + if (page.getResultItems().getAll().size() == 0) { + page.getResultItems().setSkip(true); + } } private void extractLinks(Page page, Selector urlRegionSelector, List urlPatterns) { diff --git a/webmagic-extension/src/test/java/us/codecraft/webmagic/model/ModelPageProcessorTest.java b/webmagic-extension/src/test/java/us/codecraft/webmagic/model/ModelPageProcessorTest.java new file mode 100644 index 0000000..74f3f6a --- /dev/null +++ b/webmagic-extension/src/test/java/us/codecraft/webmagic/model/ModelPageProcessorTest.java @@ -0,0 +1,45 @@ +package us.codecraft.webmagic.model; + +import org.junit.Test; +import us.codecraft.webmagic.Page; +import us.codecraft.webmagic.Request; +import us.codecraft.webmagic.model.annotation.ExtractBy; +import us.codecraft.webmagic.model.annotation.TargetUrl; +import us.codecraft.webmagic.selector.PlainText; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author code4crafter@gmail.com + * @date 14-4-4 + */ +public class ModelPageProcessorTest { + + @TargetUrl("http://codecraft.us/foo") + public static class ModelFoo { + + @ExtractBy(value = "//div/@foo", notNull = true) + private String foo; + + } + + @TargetUrl("http://codecraft.us/bar") + public static class ModelBar { + + @ExtractBy(value = "//div/@bar", notNull = true) + private String bar; + + } + + @Test + public void testMultiModel_should_not_skip_when_match() throws Exception { + Page page = new Page(); + page.setRawText("
"); + page.setRequest(new Request("http://codecraft.us/foo")); + page.setUrl(PlainText.create("http://codecraft.us/foo")); + ModelPageProcessor modelPageProcessor = ModelPageProcessor.create(null, ModelFoo.class, ModelBar.class); + modelPageProcessor.process(page); + assertThat(page.getResultItems().isSkip()).isFalse(); + + } +}