Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

change author info

master
yihua.huang 2013-06-18 07:24:19 +08:00
parent 672db58db7
commit 8cef8774cb
49 changed files with 57 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
webmagic-core/src/main/java/us/codecraft/webmagic/Page.java
View File

@ -10,7 +10,7 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 上午11:22 * Time: 上午11:22
*/ */

3
webmagic-core/src/main/java/us/codecraft/webmagic/Request.java
View File

@ -1,7 +1,8 @@
package us.codecraft.webmagic; package us.codecraft.webmagic;
/** /**
* User: cairne * Request对象是
* Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 上午11:37 * Time: 上午11:37
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/Site.java
View File

@ -4,7 +4,7 @@ import java.util.HashSet;
import java.util.Set; import java.util.Set;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午12:13 * Time: 下午12:13
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
View File

@ -14,7 +14,7 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 上午6:53 * Time: 上午6:53
*/ */

9
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/Downloader.java
View File

@ -5,11 +5,18 @@ import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site; import us.codecraft.webmagic.Site;
/** /**
* User: cairne * Downloader是webmagic抓取页面的核心接口。
* Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午12:14 * Time: 下午12:14
*/ */
public interface Downloader { public interface Downloader {
/**
*
* @param request
* @param site
* @return
*/
public Page download(Request request,Site site); public Page download(Request request,Site site);
} }

2
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -14,7 +14,7 @@ import us.codecraft.webmagic.utils.UrlUtils;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午12:15 * Time: 下午12:15
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientPool.java
View File

@ -13,7 +13,7 @@ import org.apache.http.params.*;
import us.codecraft.webmagic.Site; import us.codecraft.webmagic.Site;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午12:29 * Time: 下午12:29
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/ConsolePipeline.java
View File

@ -7,7 +7,7 @@ import us.codecraft.webmagic.selector.Selectable;
import java.util.Map; import java.util.Map;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午1:45 * Time: 下午1:45
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/FilePipeline.java
View File

@ -13,7 +13,7 @@ import java.io.PrintWriter;
import java.util.Map; import java.util.Map;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午6:28 * Time: 下午6:28
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/Pipeline.java
View File

@ -4,7 +4,7 @@ import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site; import us.codecraft.webmagic.Site;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午1:39 * Time: 下午1:39
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/processor/PageProcessor.java
View File

@ -4,7 +4,7 @@ import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site; import us.codecraft.webmagic.Site;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 上午11:42 * Time: 上午11:42
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/processor/SimplePageProcessor.java
View File

@ -7,7 +7,7 @@ import us.codecraft.webmagic.utils.UrlUtils;
import java.util.List; import java.util.List;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-22 * Date: 13-4-22
* Time: 下午9:15 * Time: 下午9:15
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/schedular/FileCacheQueueSchedular.java
View File

@ -16,7 +16,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicInteger;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午1:13 * Time: 下午1:13
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/schedular/QueueSchedular.java
View File

@ -10,7 +10,7 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.LinkedBlockingQueue;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午1:13 * Time: 下午1:13
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/schedular/Schedular.java
View File

@ -4,7 +4,7 @@ import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site; import us.codecraft.webmagic.Site;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午1:12 * Time: 下午1:12
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/selector/Html.java
View File

@ -4,7 +4,7 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 上午7:54 * Time: 上午7:54
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/selector/PlainText.java
View File

@ -6,7 +6,7 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 上午7:54 * Time: 上午7:54
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/selector/RegexResult.java
View File

@ -1,7 +1,7 @@
package us.codecraft.webmagic.selector; package us.codecraft.webmagic.selector;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 上午7:39 * Time: 上午7:39
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/selector/RegexSelector.java
View File

@ -9,7 +9,7 @@ import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException; import java.util.regex.PatternSyntaxException;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 上午7:09 * Time: 上午7:09
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/selector/ReplaceSelector.java
View File

@ -6,7 +6,7 @@ import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException; import java.util.regex.PatternSyntaxException;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 上午7:09 * Time: 上午7:09
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/selector/Selectable.java
View File

@ -3,7 +3,7 @@ package us.codecraft.webmagic.selector;
import java.util.List; import java.util.List;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-20 * Date: 13-4-20
* Time: 下午7:51 * Time: 下午7:51
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/selector/Selector.java
View File

@ -3,7 +3,7 @@ package us.codecraft.webmagic.selector;
import java.util.List; import java.util.List;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-20 * Date: 13-4-20
* Time: 下午8:02 * Time: 下午8:02
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/selector/SelectorFactory.java
View File

@ -7,7 +7,7 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 上午7:56 * Time: 上午7:56
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/selector/SmartContentSelector.java
View File

@ -10,7 +10,7 @@ import java.util.concurrent.atomic.AtomicInteger;
/** /**
* readability算法，基础是找到所有p标签的父节点 * readability算法，基础是找到所有p标签的父节点
* 写的比较乱，最终效果还在尝试中 * 写的比较乱，最终效果还在尝试中
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午4:42 * Time: 下午4:42
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/selector/XpathSelector.java
View File

@ -6,7 +6,7 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 上午9:39 * Time: 上午9:39
*/ */

2
webmagic-core/src/main/java/us/codecraft/webmagic/utils/UrlUtils.java
View File

@ -6,7 +6,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午1:52 * Time: 下午1:52
*/ */

2
webmagic-core/src/test/java/us/codecraft/webmagic/HtmlTest.java
View File

@ -5,7 +5,7 @@ import org.junit.Test;
import us.codecraft.webmagic.selector.Html; import us.codecraft.webmagic.selector.Html;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 上午8:42 * Time: 上午8:42
*/ */

2
webmagic-core/src/test/java/us/codecraft/webmagic/selector/RegexSelectorTest.java
View File

@ -4,7 +4,7 @@ import junit.framework.Assert;
import org.junit.Test; import org.junit.Test;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 上午7:13 * Time: 上午7:13
*/ */

2
webmagic-core/src/test/java/us/codecraft/webmagic/selector/XpathSelectorTest.java
View File

@ -4,7 +4,7 @@ import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
/** /**
* User: cairne Date: 13-4-21 Time: 上午10:06 * Author: code4crafter@gmail.com Date: 13-4-21 Time: 上午10:06
*/ */
public class XpathSelectorTest { public class XpathSelectorTest {

2
webmagic-core/src/test/java/us/codecraft/webmagic/utils/UrlUtilsTest.java
View File

@ -4,7 +4,7 @@ import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午2:22 * Time: 下午2:22
*/ */

2
webmagic-plugin/src/main/java/us/codecraft/webmagic/pipeline/FreemarkerPipeline.java
View File

@ -11,7 +11,7 @@ import us.codecraft.webmagic.utils.UrlUtils;
import java.io.*; import java.io.*;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-6-8 * Date: 13-6-8
* Time: 下午9:00 * Time: 下午9:00
*/ */

2
webmagic-plugin/src/test/java/us/codecraft/webmagic/FreemarkerPipelineTest.java
View File

@ -6,7 +6,7 @@ import us.codecraft.webmagic.pipeline.FreemarkerPipeline;
import java.io.IOException; import java.io.IOException;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-6-9 * Date: 13-6-9
* Time: 上午7:14 * Time: 上午7:14
*/ */

2
webmagic-samples/src/main/java/us/codecraft/webmagic/samples/DiandianBlogProcessor.java
View File

@ -7,7 +7,7 @@ import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List; import java.util.List;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午8:08 * Time: 下午8:08
*/ */

2
webmagic-samples/src/main/java/us/codecraft/webmagic/samples/DianpingBlogProcessor.java
View File

@ -7,7 +7,7 @@ import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List; import java.util.List;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午8:08 * Time: 下午8:08
*/ */

2
webmagic-samples/src/main/java/us/codecraft/webmagic/samples/DiaoyuwengProcessor.java
View File

@ -8,7 +8,7 @@ import us.codecraft.webmagic.selector.PlainText;
import java.util.List; import java.util.List;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午8:08 * Time: 下午8:08
*/ */

2
webmagic-samples/src/main/java/us/codecraft/webmagic/samples/F58PageProcesser.java
View File

@ -7,7 +7,7 @@ import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List; import java.util.List;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午1:48 * Time: 下午1:48
*/ */

2
webmagic-samples/src/main/java/us/codecraft/webmagic/samples/HuxiuProcessor.java
View File

@ -7,7 +7,7 @@ import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List; import java.util.List;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午8:08 * Time: 下午8:08
*/ */

2
webmagic-samples/src/main/java/us/codecraft/webmagic/samples/KaichibaProcessor.java
View File

@ -5,7 +5,7 @@ import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor; import us.codecraft.webmagic.processor.PageProcessor;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-5-20 * Date: 13-5-20
* Time: 下午5:31 * Time: 下午5:31
*/ */

2
webmagic-samples/src/main/java/us/codecraft/webmagic/samples/MeicanProcessor.java
View File

@ -7,7 +7,7 @@ import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List; import java.util.List;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-5-20 * Date: 13-5-20
* Time: 下午5:31 * Time: 下午5:31
*/ */

2
webmagic-samples/src/main/java/us/codecraft/webmagic/samples/NjuBBSProcessor.java
View File

@ -7,7 +7,7 @@ import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List; import java.util.List;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午8:08 * Time: 下午8:08
*/ */

2
webmagic-samples/src/main/java/us/codecraft/webmagic/samples/OschinaBlogPageProcesser.java
View File

@ -7,7 +7,7 @@ import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List; import java.util.List;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午1:48 * Time: 下午1:48
*/ */

2
webmagic-samples/src/main/java/us/codecraft/webmagic/samples/OschinaPageProcesser.java
View File

@ -7,7 +7,7 @@ import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List; import java.util.List;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午1:48 * Time: 下午1:48
*/ */

2
webmagic-samples/src/main/java/us/codecraft/webmagic/samples/QzoneBlogProcessor.java
View File

@ -7,7 +7,7 @@ import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List; import java.util.List;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午8:08 * Time: 下午8:08
*/ */

2
webmagic-samples/src/main/java/us/codecraft/webmagic/samples/SinaBlogProcesser.java
View File

@ -5,7 +5,7 @@ import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.processor.PageProcessor; import us.codecraft.webmagic.processor.PageProcessor;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午1:48 * Time: 下午1:48
*/ */

2
webmagic-samples/src/main/java/us/codecraft/webmagic/samples/TianyaPageProcesser.java
View File

@ -7,7 +7,7 @@ import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List; import java.util.List;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-21 * Date: 13-4-21
* Time: 下午1:48 * Time: 下午1:48
*/ */

2
webmagic-samples/src/test/java/us/codecraft/webmagic/SpiderTest.java
View File

@ -8,7 +8,7 @@ import us.codecraft.webmagic.samples.HuxiuProcessor;
import us.codecraft.webmagic.schedular.FileCacheQueueSchedular; import us.codecraft.webmagic.schedular.FileCacheQueueSchedular;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-4-20 * Date: 13-4-20
* Time: 下午7:46 * Time: 下午7:46
*/ */

2
webmagic-samples/src/test/java/us/codecraft/webmagic/processor/DiandianProcessorTest.java
View File

@ -11,7 +11,7 @@ import us.codecraft.webmagic.schedular.FileCacheQueueSchedular;
import java.io.IOException; import java.io.IOException;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-6-9 * Date: 13-6-9
* Time: 上午8:02 * Time: 上午8:02
*/ */

2
webmagic-samples/src/test/java/us/codecraft/webmagic/processor/DiaoyuwengProcessorTest.java
View File

@ -11,7 +11,7 @@ import us.codecraft.webmagic.schedular.FileCacheQueueSchedular;
import java.io.IOException; import java.io.IOException;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-6-9 * Date: 13-6-9
* Time: 上午8:02 * Time: 上午8:02
*/ */

2
webmagic-samples/src/test/java/us/codecraft/webmagic/processor/SinablogProcessorTest.java
View File

@ -11,7 +11,7 @@ import us.codecraft.webmagic.schedular.FileCacheQueueSchedular;
import java.io.IOException; import java.io.IOException;
/** /**
* User: cairne * Author: code4crafter@gmail.com
* Date: 13-6-9 * Date: 13-6-9
* Time: 上午8:02 * Time: 上午8:02
*/ */