From 6e32a19f807fbbf7821dee4b9ff57db021f7672b Mon Sep 17 00:00:00 2001
From: "yihua.huang" <code4crafter@gmail.com>
Date: Wed, 6 Nov 2013 12:46:50 +0800
Subject: [PATCH] update api for direct download

---
 .../java/us/codecraft/webmagic/Spider.java    | 15 ++++--
 .../webmagic/pipeline/CollectorPipeline.java  | 25 ++++------
 .../ResultItemsCollectorPipeline.java         | 26 ++++++++++
 .../example/BaiduBaikePageProcesser.java      |  2 +-
 .../webmagic/example/BaiduBaike.java          | 28 +++++++----
 .../webmagic/example/GithubRepo.java          |  5 +-
 .../webmagic/example/OschinaBlog.java         |  5 +-
 .../model/ConsolePageModelPipeline.java       |  1 +
 .../webmagic/model/ModelPipeline.java         |  1 +
 .../us/codecraft/webmagic/model/OOSpider.java | 50 +++++++------------
 .../model/PageModelCollectorPipeline.java     | 46 +++++++++++++++++
 .../webmagic/model/PageModelExtractor.java    |  2 +-
 .../pipeline/CollectorPageModelPipeline.java  | 23 +++++++++
 .../pipeline/FilePageModelPipeline.java       |  1 -
 .../pipeline/JsonFilePageModelPipeline.java   |  1 -
 .../PageModelPipeline.java                    |  2 +-
 .../webmagic/MockPageModelPipeline.java       |  2 +-
 .../webmagic/model/GithubRepoTest.java        |  1 +
 .../webmagic/model/samples/Kr36NewsModel.java |  2 +-
 .../webmagic/model/samples/OschinaBlog.java   |  2 +-
 .../webmagic/model/OOSpider-cmnt.xml          |  2 +-
 .../webmagic/model/PageModelPipeline-cmnt.xml |  2 +-
 22 files changed, 167 insertions(+), 77 deletions(-)
 create mode 100644 webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/ResultItemsCollectorPipeline.java
 create mode 100644 webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelCollectorPipeline.java
 create mode 100644 webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/CollectorPageModelPipeline.java
 rename webmagic-extension/src/main/java/us/codecraft/webmagic/{model => pipeline}/PageModelPipeline.java (86%)

diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
index 9a580bd..667a71e 100644
--- a/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
+++ b/webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
@@ -6,6 +6,7 @@ import org.apache.log4j.Logger;
 import us.codecraft.webmagic.downloader.Downloader;
 import us.codecraft.webmagic.downloader.HttpClientDownloader;
 import us.codecraft.webmagic.pipeline.CollectorPipeline;
+import us.codecraft.webmagic.pipeline.ResultItemsCollectorPipeline;
 import us.codecraft.webmagic.pipeline.ConsolePipeline;
 import us.codecraft.webmagic.pipeline.Pipeline;
 import us.codecraft.webmagic.processor.PageProcessor;
@@ -424,21 +425,25 @@ public class Spider implements Runnable, Task {
      * @param urls
      * @return
      */
-    public List<ResultItems> getAll(Collection<String> urls) {
+    public <T> List<T> getAll(Collection<String> urls) {
         destroyWhenExit = false;
         spawnUrl = false;
         startRequests = UrlUtils.convertToRequests(urls);
-        CollectorPipeline collectorPipeline = new CollectorPipeline();
+        CollectorPipeline collectorPipeline = getCollectorPipeline();
         pipelines.add(collectorPipeline);
         run();
         spawnUrl = true;
         destroyWhenExit = true;
-        return collectorPipeline.getCollector();
+        return collectorPipeline.getCollected();
     }
 
-    public ResultItems get(String url) {
+    protected CollectorPipeline getCollectorPipeline() {
+        return new ResultItemsCollectorPipeline();
+    }
+
+    public <T> T get(String url) {
         List<String> urls = Lists.newArrayList(url);
-        List<ResultItems> resultItemses = getAll(urls);
+        List<T> resultItemses = getAll(urls);
         if (resultItemses != null && resultItemses.size() > 0) {
             return resultItemses.get(0);
         } else {
diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/CollectorPipeline.java b/webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/CollectorPipeline.java
index 012c4c5..7242f43 100644
--- a/webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/CollectorPipeline.java
+++ b/webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/CollectorPipeline.java
@@ -1,25 +1,20 @@
 package us.codecraft.webmagic.pipeline;
 
-import us.codecraft.webmagic.ResultItems;
-import us.codecraft.webmagic.Task;
-
-import java.util.ArrayList;
 import java.util.List;
 
 /**
+ * Pipeline that can collect and store results. <br>
+ * Used for {@link us.codecraft.webmagic.Spider#getAll(java.util.Collection)}
+ *
  * @author code4crafter@gmail.com
  * @since 0.4.0
  */
-public class CollectorPipeline implements Pipeline{
+public interface CollectorPipeline<T> extends Pipeline {
 
-    private List<ResultItems> collector = new ArrayList<ResultItems>();
-
-    @Override
-    public void process(ResultItems resultItems, Task task) {
-        collector.add(resultItems);
-    }
-
-    public List<ResultItems> getCollector() {
-        return collector;
-    }
+    /**
+     * Get all results collected.
+     *
+     * @return collected results
+     */
+    public List<T> getCollected();
 }
diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/ResultItemsCollectorPipeline.java b/webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/ResultItemsCollectorPipeline.java
new file mode 100644
index 0000000..cf45ec8
--- /dev/null
+++ b/webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/ResultItemsCollectorPipeline.java
@@ -0,0 +1,26 @@
+package us.codecraft.webmagic.pipeline;
+
+import us.codecraft.webmagic.ResultItems;
+import us.codecraft.webmagic.Task;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author code4crafter@gmail.com
+ * @since 0.4.0
+ */
+public class ResultItemsCollectorPipeline implements CollectorPipeline<ResultItems> {
+
+    private List<ResultItems> collector = new ArrayList<ResultItems>();
+
+    @Override
+    public void process(ResultItems resultItems, Task task) {
+        collector.add(resultItems);
+    }
+
+    @Override
+    public List<ResultItems> getCollected() {
+        return collector;
+    }
+}
diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/BaiduBaikePageProcesser.java b/webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/BaiduBaikePageProcesser.java
index af03166..34f0134 100644
--- a/webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/BaiduBaikePageProcesser.java
+++ b/webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/BaiduBaikePageProcesser.java
@@ -38,7 +38,7 @@ public class BaiduBaikePageProcesser implements PageProcessor {
         list.add(String.format(urlTemplate,"太阳能"));
         list.add(String.format(urlTemplate,"地热发电"));
         list.add(String.format(urlTemplate,"地热发电"));
-        List<ResultItems> resultItemses = spider.getAll(list);
+        List<ResultItems> resultItemses = spider.<ResultItems>getAll(list);
         for (ResultItems resultItemse : resultItemses) {
             System.out.println(resultItemse.getAll());
         }
diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/example/BaiduBaike.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/example/BaiduBaike.java
index 6f901ff..0fd138e 100644
--- a/webmagic-extension/src/main/java/us/codecraft/webmagic/example/BaiduBaike.java
+++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/example/BaiduBaike.java
@@ -1,10 +1,8 @@
 package us.codecraft.webmagic.example;
 
-import us.codecraft.webmagic.Page;
-import us.codecraft.webmagic.model.AfterExtractor;
+import us.codecraft.webmagic.Site;
 import us.codecraft.webmagic.model.OOSpider;
 import us.codecraft.webmagic.model.annotation.ExtractBy;
-import us.codecraft.webmagic.model.direct.Param;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -13,21 +11,31 @@ import java.util.List;
  * @since 0.4.0
  * @author code4crafter@gmail.com
  */
-public class BaiduBaike implements AfterExtractor{
+public class BaiduBaike{
 
-    private String word;
+    @ExtractBy("//h1[@class=title]/div[@class=lemmaTitleH1]/text()")
+    private String name;
 
     @ExtractBy("//div[@id='lemmaContent-0']//div[@class='para']/allText()")
     private String description;
 
     @Override
-    public void afterProcess(Page page) {
-
+    public String toString() {
+        return "BaiduBaike{" +
+                "name='" + name + '\'' +
+                ", description='" + description + '\'' +
+                '}';
     }
 
     public static void main(String[] args) {
-        List<Param> words = new ArrayList<Param>();
-        words.add(new Param().put("word","红烧肉"));
-        OOSpider.direct(words, BaiduBaike.class).thread(10).run();
+        List<String> list = new ArrayList<String>();
+        String urlTemplate = "http://baike.baidu.com/search/word?word=%s&pic=1&sug=1&enc=utf8";
+        list.add(String.format(urlTemplate,"水力发电"));
+        list.add(String.format(urlTemplate,"风力发电"));
+        list.add(String.format(urlTemplate,"太阳能"));
+        list.add(String.format(urlTemplate,"地热发电"));
+        list.add(String.format(urlTemplate, "地热发电"));
+        List<BaiduBaike> baiduBaikes = OOSpider.create(Site.me().setSleepTime(100), BaiduBaike.class).<BaiduBaike>getAll(list);
+        System.out.println(baiduBaikes);
     }
 }
diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/example/GithubRepo.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/example/GithubRepo.java
index 58441cb..427cdf7 100644
--- a/webmagic-extension/src/main/java/us/codecraft/webmagic/example/GithubRepo.java
+++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/example/GithubRepo.java
@@ -41,8 +41,9 @@ public class GithubRepo implements HasKey {
     private String url;
 
     public static void main(String[] args) {
-        OOSpider.create(Site.me().addStartUrl("https://github.com/code4craft").setSleepTime(100)
-                , new ConsolePageModelPipeline(), GithubRepo.class).thread(10).run();
+        OOSpider.create(Site.me().setSleepTime(100)
+                , new ConsolePageModelPipeline(), GithubRepo.class)
+                .addUrl("https://github.com/code4craft").thread(10).run();
     }
 
     @Override
diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/example/OschinaBlog.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/example/OschinaBlog.java
index 1545f88..5bd8ddd 100644
--- a/webmagic-extension/src/main/java/us/codecraft/webmagic/example/OschinaBlog.java
+++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/example/OschinaBlog.java
@@ -31,8 +31,9 @@ public class OschinaBlog {
     private Date date;
 
     public static void main(String[] args) {
-        OOSpider.create(Site.me().addStartUrl("http://my.oschina.net/flashsword/blog")
-                , new JsonFilePageModelPipeline("/data/webmagic/"), OschinaBlog.class).run();
+        OOSpider.create(Site.me()
+                , new JsonFilePageModelPipeline("/data/webmagic/"), OschinaBlog.class)
+                .addUrl("http://my.oschina.net/flashsword/blog").run();
     }
 
     public String getTitle() {
diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/model/ConsolePageModelPipeline.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/ConsolePageModelPipeline.java
index abf411c..f4740c9 100644
--- a/webmagic-extension/src/main/java/us/codecraft/webmagic/model/ConsolePageModelPipeline.java
+++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/ConsolePageModelPipeline.java
@@ -2,6 +2,7 @@ package us.codecraft.webmagic.model;
 
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import us.codecraft.webmagic.Task;
+import us.codecraft.webmagic.pipeline.PageModelPipeline;
 
 /**
  * Print page model in console.<br>
diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/model/ModelPipeline.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/ModelPipeline.java
index 071cb26..593178f 100644
--- a/webmagic-extension/src/main/java/us/codecraft/webmagic/model/ModelPipeline.java
+++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/ModelPipeline.java
@@ -3,6 +3,7 @@ package us.codecraft.webmagic.model;
 import us.codecraft.webmagic.ResultItems;
 import us.codecraft.webmagic.Task;
 import us.codecraft.webmagic.model.annotation.ExtractBy;
+import us.codecraft.webmagic.pipeline.PageModelPipeline;
 import us.codecraft.webmagic.pipeline.Pipeline;
 
 import java.lang.annotation.Annotation;
diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/model/OOSpider.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/OOSpider.java
index c43b6e8..3133308 100644
--- a/webmagic-extension/src/main/java/us/codecraft/webmagic/model/OOSpider.java
+++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/OOSpider.java
@@ -2,10 +2,12 @@ package us.codecraft.webmagic.model;
 
 import us.codecraft.webmagic.Site;
 import us.codecraft.webmagic.Spider;
-import us.codecraft.webmagic.model.direct.Param;
+import us.codecraft.webmagic.pipeline.CollectorPipeline;
+import us.codecraft.webmagic.pipeline.PageModelPipeline;
 import us.codecraft.webmagic.processor.PageProcessor;
 
-import java.util.Collection;
+import java.util.ArrayList;
+import java.util.List;
 
 /**
  * The spider for page model extractor.<br>
@@ -36,12 +38,16 @@ import java.util.Collection;
  * @author code4crafter@gmail.com <br>
  * @since 0.2.0
  */
-public class OOSpider extends Spider {
+public class OOSpider<T> extends Spider {
 
     private ModelPageProcessor modelPageProcessor;
 
     private ModelPipeline modelPipeline;
 
+    private PageModelPipeline pageModelPipeline;
+
+    private List<Class> pageModelClasses = new ArrayList<Class>();
+
     protected OOSpider(ModelPageProcessor modelPageProcessor) {
         super(modelPageProcessor);
         this.modelPageProcessor = modelPageProcessor;
@@ -62,13 +68,19 @@ public class OOSpider extends Spider {
         this(ModelPageProcessor.create(site, pageModels));
         this.modelPipeline = new ModelPipeline();
         super.addPipeline(modelPipeline);
-        if (pageModelPipeline != null) {
-            for (Class pageModel : pageModels) {
+        for (Class pageModel : pageModels) {
+            if (pageModelPipeline != null) {
                 this.modelPipeline.put(pageModel, pageModelPipeline);
             }
+            pageModelClasses.add(pageModel);
         }
     }
 
+    @Override
+    protected CollectorPipeline getCollectorPipeline() {
+        return new PageModelCollectorPipeline<T>(pageModelClasses.get(0));
+    }
+
     public static OOSpider create(Site site, Class... pageModels) {
         return new OOSpider(site, null, pageModels);
     }
@@ -77,34 +89,6 @@ public class OOSpider extends Spider {
         return new OOSpider(site, pageModelPipeline, pageModels);
     }
 
-    /**
-     * @since 0.3.3
-     * NO implement yet!
-     */
-    public static OOSpider direct(Site site, PageModelPipeline pageModelPipeline, Class... pageModels) {
-        return new OOSpider(site, pageModelPipeline, pageModels);
-    }
-
-    /**
-     * @since 0.3.3
-     * NO implement yet!
-     */
-    public static OOSpider direct(PageModelPipeline pageModelPipeline, Class... pageModels) {
-        return new OOSpider(null, pageModelPipeline, pageModels);
-    }
-
-    /**
-     * @since 0.3.3
-     * NO implement yet!
-     */
-    public static OOSpider direct(Class... pageModels) {
-        return new OOSpider(null, null, pageModels);
-    }
-
-    public static OOSpider direct(Collection<Param> params, Class... pageModels) {
-        return new OOSpider(null, null, pageModels);
-    }
-
     public OOSpider addPageModel(PageModelPipeline pageModelPipeline, Class... pageModels) {
         for (Class pageModel : pageModels) {
             modelPageProcessor.addPageModel(pageModel);
diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelCollectorPipeline.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelCollectorPipeline.java
new file mode 100644
index 0000000..b61f112
--- /dev/null
+++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelCollectorPipeline.java
@@ -0,0 +1,46 @@
+package us.codecraft.webmagic.model;
+
+import us.codecraft.webmagic.ResultItems;
+import us.codecraft.webmagic.Task;
+import us.codecraft.webmagic.model.annotation.ExtractBy;
+import us.codecraft.webmagic.pipeline.CollectorPageModelPipeline;
+import us.codecraft.webmagic.pipeline.CollectorPipeline;
+
+import java.lang.annotation.Annotation;
+import java.util.List;
+
+/**
+ * @author code4crafter@gmail.com
+ * @since 0.4.0
+ */
+class PageModelCollectorPipeline<T> implements CollectorPipeline<T> {
+
+    private final CollectorPageModelPipeline<T> classPipeline = new CollectorPageModelPipeline<T>();
+
+    private final Class<?> clazz;
+
+    PageModelCollectorPipeline(Class<?> clazz) {
+        this.clazz = clazz;
+    }
+
+    @Override
+    public List<T> getCollected() {
+        return classPipeline.getCollected();
+    }
+
+    @Override
+    public synchronized void process(ResultItems resultItems, Task task) {
+        Object o = resultItems.get(clazz.getCanonicalName());
+        if (o != null) {
+            Annotation annotation = clazz.getAnnotation(ExtractBy.class);
+            if (annotation == null || !((ExtractBy) annotation).multi()) {
+                classPipeline.process((T) o, task);
+            } else {
+                List<Object> list = (List<Object>) o;
+                for (Object o1 : list) {
+                   classPipeline.process((T) o1, task);
+                }
+            }
+        }
+    }
+}
diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelExtractor.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelExtractor.java
index c78bd31..a079988 100644
--- a/webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelExtractor.java
+++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelExtractor.java
@@ -195,7 +195,7 @@ class PageModelExtractor {
     private void initClassExtractors() {
         Annotation annotation = clazz.getAnnotation(TargetUrl.class);
         if (annotation == null) {
-            targetUrlPatterns.add(Pattern.compile(".*"));
+            targetUrlPatterns.add(Pattern.compile("(.*)"));
         } else {
             TargetUrl targetUrl = (TargetUrl) annotation;
             String[] value = targetUrl.value();
diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/CollectorPageModelPipeline.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/CollectorPageModelPipeline.java
new file mode 100644
index 0000000..b6e0b1b
--- /dev/null
+++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/CollectorPageModelPipeline.java
@@ -0,0 +1,23 @@
+package us.codecraft.webmagic.pipeline;
+
+import us.codecraft.webmagic.Task;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @author code4crafter@gmail.com
+ */
+public class CollectorPageModelPipeline<T> implements PageModelPipeline<T> {
+
+    private List<T> collected = new ArrayList<T>();
+
+    @Override
+    public synchronized void process(T t, Task task) {
+        collected.add(t);
+    }
+
+    public List<T> getCollected() {
+        return collected;
+    }
+}
diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/FilePageModelPipeline.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/FilePageModelPipeline.java
index 5586863..273b18b 100644
--- a/webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/FilePageModelPipeline.java
+++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/FilePageModelPipeline.java
@@ -5,7 +5,6 @@ import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.log4j.Logger;
 import us.codecraft.webmagic.Task;
 import us.codecraft.webmagic.model.HasKey;
-import us.codecraft.webmagic.model.PageModelPipeline;
 import us.codecraft.webmagic.utils.FilePersistentBase;
 
 import java.io.FileWriter;
diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/JsonFilePageModelPipeline.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/JsonFilePageModelPipeline.java
index 228ec8c..4e35dfe 100644
--- a/webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/JsonFilePageModelPipeline.java
+++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/JsonFilePageModelPipeline.java
@@ -6,7 +6,6 @@ import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.log4j.Logger;
 import us.codecraft.webmagic.Task;
 import us.codecraft.webmagic.model.HasKey;
-import us.codecraft.webmagic.model.PageModelPipeline;
 import us.codecraft.webmagic.utils.FilePersistentBase;
 
 import java.io.FileWriter;
diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelPipeline.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/PageModelPipeline.java
similarity index 86%
rename from webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelPipeline.java
rename to webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/PageModelPipeline.java
index 2cb3808..382f71d 100644
--- a/webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelPipeline.java
+++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/PageModelPipeline.java
@@ -1,4 +1,4 @@
-package us.codecraft.webmagic.model;
+package us.codecraft.webmagic.pipeline;
 
 import us.codecraft.webmagic.Task;
 
diff --git a/webmagic-extension/src/test/java/us/codecraft/webmagic/MockPageModelPipeline.java b/webmagic-extension/src/test/java/us/codecraft/webmagic/MockPageModelPipeline.java
index ea7601b..f94efce 100644
--- a/webmagic-extension/src/test/java/us/codecraft/webmagic/MockPageModelPipeline.java
+++ b/webmagic-extension/src/test/java/us/codecraft/webmagic/MockPageModelPipeline.java
@@ -1,7 +1,7 @@
 package us.codecraft.webmagic;
 
 import junit.framework.Assert;
-import us.codecraft.webmagic.model.PageModelPipeline;
+import us.codecraft.webmagic.pipeline.PageModelPipeline;
 
 /**
  * @author code4crafter@gmail.com
diff --git a/webmagic-extension/src/test/java/us/codecraft/webmagic/model/GithubRepoTest.java b/webmagic-extension/src/test/java/us/codecraft/webmagic/model/GithubRepoTest.java
index d6e1bf0..b719bf0 100644
--- a/webmagic-extension/src/test/java/us/codecraft/webmagic/model/GithubRepoTest.java
+++ b/webmagic-extension/src/test/java/us/codecraft/webmagic/model/GithubRepoTest.java
@@ -6,6 +6,7 @@ import us.codecraft.webmagic.MockDownloader;
 import us.codecraft.webmagic.Site;
 import us.codecraft.webmagic.Task;
 import us.codecraft.webmagic.example.GithubRepo;
+import us.codecraft.webmagic.pipeline.PageModelPipeline;
 
 /**
  * @author code4crafter@gmail.com <br>
diff --git a/webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/Kr36NewsModel.java b/webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/Kr36NewsModel.java
index 472b47e..936f132 100644
--- a/webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/Kr36NewsModel.java
+++ b/webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/Kr36NewsModel.java
@@ -3,7 +3,7 @@ package us.codecraft.webmagic.model.samples;
 import us.codecraft.webmagic.Site;
 import us.codecraft.webmagic.Task;
 import us.codecraft.webmagic.model.OOSpider;
-import us.codecraft.webmagic.model.PageModelPipeline;
+import us.codecraft.webmagic.pipeline.PageModelPipeline;
 import us.codecraft.webmagic.model.annotation.ExtractBy;
 import us.codecraft.webmagic.model.annotation.ExtractByUrl;
 import us.codecraft.webmagic.model.annotation.HelpUrl;
diff --git a/webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/OschinaBlog.java b/webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/OschinaBlog.java
index 9e7fa88..468b855 100644
--- a/webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/OschinaBlog.java
+++ b/webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/OschinaBlog.java
@@ -3,7 +3,7 @@ package us.codecraft.webmagic.model.samples;
 import us.codecraft.webmagic.Site;
 import us.codecraft.webmagic.Task;
 import us.codecraft.webmagic.model.OOSpider;
-import us.codecraft.webmagic.model.PageModelPipeline;
+import us.codecraft.webmagic.pipeline.PageModelPipeline;
 import us.codecraft.webmagic.model.annotation.ExtractBy;
 import us.codecraft.webmagic.model.annotation.TargetUrl;
 
diff --git a/zh_docs/us/codecraft/webmagic/model/OOSpider-cmnt.xml b/zh_docs/us/codecraft/webmagic/model/OOSpider-cmnt.xml
index 2fd60a7..232c509 100644
--- a/zh_docs/us/codecraft/webmagic/model/OOSpider-cmnt.xml
+++ b/zh_docs/us/codecraft/webmagic/model/OOSpider-cmnt.xml
@@ -12,7 +12,7 @@
 ]]></data>
 </comment>
 <comment>
-<key><![CDATA[us.codecraft.webmagic.model.OOSpider(us.codecraft.webmagic.Site, us.codecraft.webmagic.model.PageModelPipeline, java.lang.Class...)]]></key>
+<key><![CDATA[us.codecraft.webmagic.model.OOSpider(us.codecraft.webmagic.Site, us.codecraft.webmagic.pipeline.PageModelPipeline, java.lang.Class...)]]></key>
 <data><![CDATA[ 创建一个爬虫。<br>
  @param site
  @param pageModelPipeline
diff --git a/zh_docs/us/codecraft/webmagic/model/PageModelPipeline-cmnt.xml b/zh_docs/us/codecraft/webmagic/model/PageModelPipeline-cmnt.xml
index ac65729..64fb524 100644
--- a/zh_docs/us/codecraft/webmagic/model/PageModelPipeline-cmnt.xml
+++ b/zh_docs/us/codecraft/webmagic/model/PageModelPipeline-cmnt.xml
@@ -4,7 +4,7 @@
 <date-generated>Sat Aug 17 14:14:46 CST 2013</date-generated>
 </meta>
 <comment>
-<key><![CDATA[us.codecraft.webmagic.model.PageModelPipeline]]></key>
+<key><![CDATA[us.codecraft.webmagic.pipeline.PageModelPipeline]]></key>
 <data><![CDATA[ @author code4crafter@gmail.com <br>
  Date: 13-8-3 <br>
  Time: 上午9:34 <br>