update docs
parent
8f954c7997
commit
fb0797b65c
|
@ -10,10 +10,16 @@ import java.util.Map;
|
|||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* Page保存了抓取的结果,并可定义下一次抓取的链接内容。
|
||||
* Author: code4crafter@gmail.com
|
||||
* Date: 13-4-21
|
||||
* Time: 上午11:22
|
||||
* <pre>
|
||||
*Page保存了上一次抓取的结果,并可定义待抓取的链接内容。
|
||||
*
|
||||
* 主要方法:
|
||||
* {@link #getUrl()} 获取页面的Url
|
||||
* {@link #getHtml()} 获取页面的html内容
|
||||
* {@link #addTargetRequests(java.util.List)} {@link #addTargetRequest(String)} 添加待抓取的链接
|
||||
*
|
||||
* </pre>
|
||||
* @author code4crafter@gmail.com <br>
|
||||
*/
|
||||
public class Page {
|
||||
|
||||
|
@ -34,6 +40,10 @@ public class Page {
|
|||
public Page() {
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @return fields
|
||||
*/
|
||||
public Map<String, Selectable> getFields() {
|
||||
return fields;
|
||||
}
|
||||
|
|
|
@ -17,7 +17,7 @@ package us.codecraft.webmagic;
|
|||
* String linktext = (String)page.getRequest().getExtra()[0];
|
||||
* }
|
||||
* </pre>
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 上午11:37
|
||||
*/
|
||||
|
|
|
@ -4,7 +4,7 @@ import java.util.*;
|
|||
|
||||
/**
|
||||
* Site定义一个待抓取的站点的各种信息。
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午12:13
|
||||
*/
|
||||
|
|
|
@ -14,7 +14,7 @@ import java.util.ArrayList;
|
|||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 上午6:53
|
||||
*/
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
package us.codecraft.webmagic;
|
||||
|
||||
/**
|
||||
* Author: code4crafer@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-6-18
|
||||
* Time: 下午2:57
|
||||
*/
|
||||
|
|
|
@ -6,7 +6,7 @@ import us.codecraft.webmagic.Site;
|
|||
|
||||
/**
|
||||
* Downloader是webmagic下载页面的接口。webmagic默认使用了HttpComponent作为下载器,一般情况,你无需自己实现这个接口。
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午12:14
|
||||
*/
|
||||
|
@ -17,7 +17,7 @@ public interface Downloader {
|
|||
*
|
||||
* @param request
|
||||
* @param site
|
||||
* @return
|
||||
* @return page
|
||||
*/
|
||||
public Page download(Request request, Site site);
|
||||
}
|
||||
|
|
|
@ -14,7 +14,7 @@ import us.codecraft.webmagic.utils.UrlUtils;
|
|||
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午12:15
|
||||
*/
|
||||
|
|
|
@ -18,7 +18,7 @@ import us.codecraft.webmagic.Site;
|
|||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午12:29
|
||||
*/
|
||||
|
|
|
@ -7,7 +7,7 @@ import us.codecraft.webmagic.selector.Selectable;
|
|||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午1:45
|
||||
*/
|
||||
|
|
|
@ -12,7 +12,7 @@ import java.io.PrintWriter;
|
|||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午6:28
|
||||
*/
|
||||
|
|
|
@ -4,7 +4,7 @@ import us.codecraft.webmagic.Page;
|
|||
import us.codecraft.webmagic.Task;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午1:39
|
||||
*/
|
||||
|
|
|
@ -4,7 +4,7 @@ import us.codecraft.webmagic.Page;
|
|||
import us.codecraft.webmagic.Site;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 上午11:42
|
||||
*/
|
||||
|
|
|
@ -7,7 +7,7 @@ import us.codecraft.webmagic.utils.UrlUtils;
|
|||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-22
|
||||
* Time: 下午9:15
|
||||
*/
|
||||
|
|
|
@ -16,7 +16,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
|
|||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午1:13
|
||||
*/
|
||||
|
|
|
@ -10,7 +10,7 @@ import java.util.concurrent.BlockingQueue;
|
|||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午1:13
|
||||
*/
|
||||
|
|
|
@ -4,7 +4,7 @@ import us.codecraft.webmagic.Request;
|
|||
import us.codecraft.webmagic.Task;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午1:12
|
||||
*/
|
||||
|
|
|
@ -4,7 +4,7 @@ import java.util.ArrayList;
|
|||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 上午7:54
|
||||
*/
|
||||
|
|
|
@ -6,7 +6,7 @@ import java.util.ArrayList;
|
|||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 上午7:54
|
||||
*/
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
package us.codecraft.webmagic.selector;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 上午7:39
|
||||
*/
|
||||
|
|
|
@ -9,7 +9,7 @@ import java.util.regex.Pattern;
|
|||
import java.util.regex.PatternSyntaxException;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 上午7:09
|
||||
*/
|
||||
|
|
|
@ -6,7 +6,7 @@ import java.util.regex.Pattern;
|
|||
import java.util.regex.PatternSyntaxException;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 上午7:09
|
||||
*/
|
||||
|
|
|
@ -3,7 +3,7 @@ package us.codecraft.webmagic.selector;
|
|||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-20
|
||||
* Time: 下午7:51
|
||||
*/
|
||||
|
|
|
@ -3,7 +3,7 @@ package us.codecraft.webmagic.selector;
|
|||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-20
|
||||
* Time: 下午8:02
|
||||
*/
|
||||
|
|
|
@ -7,7 +7,7 @@ import java.util.Map;
|
|||
import java.util.concurrent.ConcurrentHashMap;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 上午7:56
|
||||
*/
|
||||
|
|
|
@ -10,7 +10,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
|||
/**
|
||||
* readability算法,基础是找到所有p标签的父节点
|
||||
* 写的比较乱,最终效果还在尝试中
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午4:42
|
||||
*/
|
||||
|
|
|
@ -6,7 +6,7 @@ import java.util.ArrayList;
|
|||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 上午9:39
|
||||
*/
|
||||
|
|
|
@ -6,7 +6,7 @@ import java.util.regex.Matcher;
|
|||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午1:52
|
||||
*/
|
||||
|
|
|
@ -5,7 +5,7 @@ import org.junit.Test;
|
|||
import us.codecraft.webmagic.selector.Html;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 上午8:42
|
||||
*/
|
||||
|
|
|
@ -4,7 +4,7 @@ import junit.framework.Assert;
|
|||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 上午7:13
|
||||
*/
|
||||
|
|
|
@ -4,7 +4,7 @@ import org.junit.Assert;
|
|||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com Date: 13-4-21 Time: 上午10:06
|
||||
* @author code4crafter@gmail.com <br> Date: 13-4-21 Time: 上午10:06
|
||||
*/
|
||||
public class XpathSelectorTest {
|
||||
|
||||
|
|
|
@ -4,7 +4,7 @@ import org.junit.Assert;
|
|||
import org.junit.Test;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午2:22
|
||||
*/
|
||||
|
|
|
@ -13,7 +13,7 @@ import java.io.IOException;
|
|||
import java.io.PrintWriter;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-6-8
|
||||
* Time: 下午9:00
|
||||
*/
|
||||
|
|
|
@ -6,7 +6,7 @@ import us.codecraft.webmagic.pipeline.FreemarkerPipeline;
|
|||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-6-9
|
||||
* Time: 上午7:14
|
||||
*/
|
||||
|
|
|
@ -7,7 +7,7 @@ import us.codecraft.webmagic.processor.PageProcessor;
|
|||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午8:08
|
||||
*/
|
||||
|
|
|
@ -8,7 +8,7 @@ import us.codecraft.webmagic.processor.PageProcessor;
|
|||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午8:08
|
||||
*/
|
||||
|
|
|
@ -8,7 +8,7 @@ import us.codecraft.webmagic.selector.PlainText;
|
|||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午8:08
|
||||
*/
|
||||
|
|
|
@ -7,7 +7,7 @@ import us.codecraft.webmagic.processor.PageProcessor;
|
|||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午1:48
|
||||
*/
|
||||
|
|
|
@ -7,7 +7,7 @@ import us.codecraft.webmagic.processor.PageProcessor;
|
|||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午8:08
|
||||
*/
|
||||
|
|
|
@ -5,7 +5,7 @@ import us.codecraft.webmagic.Site;
|
|||
import us.codecraft.webmagic.processor.PageProcessor;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-5-20
|
||||
* Time: 下午5:31
|
||||
*/
|
||||
|
|
|
@ -7,7 +7,7 @@ import us.codecraft.webmagic.processor.PageProcessor;
|
|||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-5-20
|
||||
* Time: 下午5:31
|
||||
*/
|
||||
|
|
|
@ -7,7 +7,7 @@ import us.codecraft.webmagic.processor.PageProcessor;
|
|||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午8:08
|
||||
*/
|
||||
|
|
|
@ -7,7 +7,7 @@ import us.codecraft.webmagic.processor.PageProcessor;
|
|||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午1:48
|
||||
*/
|
||||
|
|
|
@ -7,7 +7,7 @@ import us.codecraft.webmagic.processor.PageProcessor;
|
|||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午1:48
|
||||
*/
|
||||
|
|
|
@ -7,7 +7,7 @@ import us.codecraft.webmagic.processor.PageProcessor;
|
|||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午8:08
|
||||
*/
|
||||
|
|
|
@ -5,7 +5,7 @@ import us.codecraft.webmagic.Page;
|
|||
import us.codecraft.webmagic.processor.PageProcessor;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午1:48
|
||||
*/
|
||||
|
|
|
@ -7,7 +7,7 @@ import us.codecraft.webmagic.processor.PageProcessor;
|
|||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-21
|
||||
* Time: 下午1:48
|
||||
*/
|
||||
|
|
|
@ -8,7 +8,7 @@ import us.codecraft.webmagic.samples.HuxiuProcessor;
|
|||
import us.codecraft.webmagic.schedular.FileCacheQueueSchedular;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-4-20
|
||||
* Time: 下午7:46
|
||||
*/
|
||||
|
|
|
@ -11,7 +11,7 @@ import us.codecraft.webmagic.schedular.FileCacheQueueSchedular;
|
|||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-6-9
|
||||
* Time: 上午8:02
|
||||
*/
|
||||
|
|
|
@ -11,7 +11,7 @@ import us.codecraft.webmagic.schedular.FileCacheQueueSchedular;
|
|||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-6-9
|
||||
* Time: 上午8:02
|
||||
*/
|
||||
|
|
|
@ -11,7 +11,7 @@ import us.codecraft.webmagic.schedular.FileCacheQueueSchedular;
|
|||
import java.io.IOException;
|
||||
|
||||
/**
|
||||
* Author: code4crafter@gmail.com
|
||||
* @author code4crafter@gmail.com <br>
|
||||
* Date: 13-6-9
|
||||
* Time: 上午8:02
|
||||
*/
|
||||
|
|
Loading…
Reference in New Issue