webMagic-Mobai
/
magic-Dependency
mirror of https://gitea.qinmian.online/Mobai/magic-Dependency.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update package

master
yihua.huang 2013-08-04 07:53:28 +08:00
parent cfb8990453
commit 21cae2ff2e
21 changed files with 45 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
webmagic-core/src/main/java/us/codecraft/webmagic/Page.java
View File

@ -9,7 +9,7 @@ import java.util.List;
/**
* <pre>
*Page保存了上一次抓取的结果，并可定义待抓取的链接内容。
* Page保存了上一次抓取的结果，并可定义待抓取的链接内容。
*
* 主要方法：
* {@link #getUrl()} 获取页面的Url
@ -19,6 +19,7 @@ import java.util.List;
* {@link #addTargetRequests(java.util.List)} {@link #addTargetRequest(String)} 添加待抓取的链接
*
* </pre>
*
* @author code4crafter@gmail.com <br>
*/
public class Page {
@ -36,9 +37,16 @@ public class Page {
public Page() {
}
public Page setSkip(boolean skip) {
resultItems.setSkip(skip);
return this;
}
/**
* 保存抽取的结果
* @param key 结果的key
*
* @param key 结果的key
* @param field 结果的value
*/
public void putField(String key, Object field) {
@ -47,6 +55,7 @@ public class Page {
/**
* 获取页面的html内容
*
* @return html 页面的html内容
*/
public Selectable getHtml() {
@ -63,6 +72,7 @@ public class Page {
/**
* 添加待抓取的链接
*
* @param requests 待抓取的链接
*/
public void addTargetRequests(List<String> requests) {
@ -79,6 +89,7 @@ public class Page {
/**
* 添加待抓取的链接
*
* @param requestString 待抓取的链接
*/
public void addTargetRequest(String requestString) {
@ -93,6 +104,7 @@ public class Page {
/**
* 添加待抓取的页面，在需要传递附加信息时使用
*
* @param request 待抓取的页面
*/
public void addTargetRequest(Request request) {
@ -103,6 +115,7 @@ public class Page {
/**
* 获取页面的Url
*
* @return url 当前页面的url，可用于抽取
*/
public Selectable getUrl() {
@ -111,6 +124,7 @@ public class Page {
/**
* 设置url
*
* @param url
*/
public void setUrl(Selectable url) {
@ -119,6 +133,7 @@ public class Page {
/**
* 获取抓取请求
*
* @return request 抓取请求
*/
public Request getRequest() {

2
webmagic-core/src/main/java/us/codecraft/webmagic/oo/AfterExtractor.java → webmagic-core/src/main/java/us/codecraft/webmagic/model/AfterExtractor.java
View File

@ -1,4 +1,4 @@
package us.codecraft.webmagic.oo;
package us.codecraft.webmagic.model;
import us.codecraft.webmagic.Page;

2
webmagic-core/src/main/java/us/codecraft/webmagic/oo/ConsolePageModelPipeline.java → webmagic-core/src/main/java/us/codecraft/webmagic/model/ConsolePageModelPipeline.java
View File

@ -1,4 +1,4 @@
package us.codecraft.webmagic.oo;
package us.codecraft.webmagic.model;
import org.apache.commons.lang3.builder.ToStringBuilder;
import us.codecraft.webmagic.Task;

2
webmagic-core/src/main/java/us/codecraft/webmagic/oo/ExtractBy.java → webmagic-core/src/main/java/us/codecraft/webmagic/model/ExtractBy.java
View File

@ -1,4 +1,4 @@
package us.codecraft.webmagic.oo;
package us.codecraft.webmagic.model;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;

2
webmagic-core/src/main/java/us/codecraft/webmagic/oo/ExtractByUrl.java → webmagic-core/src/main/java/us/codecraft/webmagic/model/ExtractByUrl.java
View File

@ -1,4 +1,4 @@
package us.codecraft.webmagic.oo;
package us.codecraft.webmagic.model;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;

2
webmagic-core/src/main/java/us/codecraft/webmagic/oo/Extractor.java → webmagic-core/src/main/java/us/codecraft/webmagic/model/Extractor.java
View File

@ -1,4 +1,4 @@
package us.codecraft.webmagic.oo;
package us.codecraft.webmagic.model;
import us.codecraft.webmagic.selector.Selector;

2
webmagic-core/src/main/java/us/codecraft/webmagic/oo/FieldExtractor.java → webmagic-core/src/main/java/us/codecraft/webmagic/model/FieldExtractor.java
View File

@ -1,4 +1,4 @@
package us.codecraft.webmagic.oo;
package us.codecraft.webmagic.model;
import us.codecraft.webmagic.selector.Selector;

2
webmagic-core/src/main/java/us/codecraft/webmagic/oo/HelpUrl.java → webmagic-core/src/main/java/us/codecraft/webmagic/model/HelpUrl.java
View File

@ -1,4 +1,4 @@
package us.codecraft.webmagic.oo;
package us.codecraft.webmagic.model;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;

2
webmagic-core/src/main/java/us/codecraft/webmagic/oo/OOSpider.java → webmagic-core/src/main/java/us/codecraft/webmagic/model/OOSpider.java
View File

@ -1,4 +1,4 @@
package us.codecraft.webmagic.oo;
package us.codecraft.webmagic.model;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;

2
webmagic-core/src/main/java/us/codecraft/webmagic/oo/ObjectPageProcessor.java → webmagic-core/src/main/java/us/codecraft/webmagic/model/ObjectPageProcessor.java
View File

@ -1,4 +1,4 @@
package us.codecraft.webmagic.oo;
package us.codecraft.webmagic.model;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;

2
webmagic-core/src/main/java/us/codecraft/webmagic/oo/ObjectPipeline.java → webmagic-core/src/main/java/us/codecraft/webmagic/model/ObjectPipeline.java
View File

@ -1,4 +1,4 @@
package us.codecraft.webmagic.oo;
package us.codecraft.webmagic.model;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;

2
webmagic-core/src/main/java/us/codecraft/webmagic/oo/PageModelExtractor.java → webmagic-core/src/main/java/us/codecraft/webmagic/model/PageModelExtractor.java
View File

@ -1,4 +1,4 @@
package us.codecraft.webmagic.oo;
package us.codecraft.webmagic.model;
import org.apache.commons.lang3.StringUtils;
import us.codecraft.webmagic.Page;

2
webmagic-core/src/main/java/us/codecraft/webmagic/oo/PageModelPipeline.java → webmagic-core/src/main/java/us/codecraft/webmagic/model/PageModelPipeline.java
View File

@ -1,4 +1,4 @@
package us.codecraft.webmagic.oo;
package us.codecraft.webmagic.model;
import us.codecraft.webmagic.Task;

2
webmagic-core/src/main/java/us/codecraft/webmagic/oo/TargetUrl.java → webmagic-core/src/main/java/us/codecraft/webmagic/model/TargetUrl.java
View File

@ -1,4 +1,4 @@
package us.codecraft.webmagic.oo;
package us.codecraft.webmagic.model;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;

0
webmagic-core/src/main/java/us/codecraft/webmagic/oo/package.html → webmagic-core/src/main/java/us/codecraft/webmagic/model/package.html
View File

5
webmagic-core/src/test/java/us/codecraft/webmagic/oo/OschinaBlog.java → webmagic-core/src/test/java/us/codecraft/webmagic/model/OschinaBlog.java
View File

@ -1,4 +1,4 @@
package us.codecraft.webmagic.oo;
package us.codecraft.webmagic.model;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
@ -10,7 +10,7 @@ import java.util.List;
* @date: 13-8-1 <br>
* Time: 下午10:18 <br>
*/
@TargetUrl(value="http://my.oschina.net/flashsword/blog/*",sourceRegion = "//div[@class='BlogLinks']//a/@href")
@TargetUrl("http://my.oschina.net/flashsword/blog/*")
public class OschinaBlog implements AfterExtractor {
@ExtractBy("//title")
@ -27,6 +27,7 @@ public class OschinaBlog implements AfterExtractor {
System.out.println("title:\t"+title);
System.out.println("content:\t"+content);
System.out.println("tags:\t" + tags);
page.setSkip(true);
}
public static void main(String[] args) {

2
webmagic-core/src/test/java/us/codecraft/webmagic/oo/TestFetcher.java → webmagic-core/src/test/java/us/codecraft/webmagic/model/TestFetcher.java
View File

@ -1,4 +1,4 @@
package us.codecraft.webmagic.oo;
package us.codecraft.webmagic.model;
import org.junit.Ignore;
import org.junit.Test;

2
webmagic-samples/src/main/java/us/codecraft/webmagic/oo/samples/Blog.java → webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/Blog.java
View File

@ -1,4 +1,4 @@
package us.codecraft.webmagic.oo.samples;
package us.codecraft.webmagic.model.samples;
/**
* @author code4crafter@gmail.com <br>

8
webmagic-samples/src/main/java/us/codecraft/webmagic/oo/samples/IteyeBlog.java → webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/IteyeBlog.java
View File

@ -1,9 +1,9 @@
package us.codecraft.webmagic.oo.samples;
package us.codecraft.webmagic.model.samples;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.oo.ExtractBy;
import us.codecraft.webmagic.oo.OOSpider;
import us.codecraft.webmagic.oo.TargetUrl;
import us.codecraft.webmagic.model.ExtractBy;
import us.codecraft.webmagic.model.OOSpider;
import us.codecraft.webmagic.model.TargetUrl;
/**
* @author code4crafter@gmail.com <br>

4
webmagic-samples/src/main/java/us/codecraft/webmagic/oo/samples/OschinaAnswer.java → webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/OschinaAnswer.java
View File

@ -1,8 +1,8 @@
package us.codecraft.webmagic.oo.samples;
package us.codecraft.webmagic.model.samples;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.oo.*;
import us.codecraft.webmagic.model.*;
/**
* @author code4crafter@gmail.com <br>

8
webmagic-samples/src/main/java/us/codecraft/webmagic/oo/samples/OschinaBlog.java → webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/OschinaBlog.java
View File

@ -1,9 +1,9 @@
package us.codecraft.webmagic.oo.samples;
package us.codecraft.webmagic.model.samples;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.oo.ExtractBy;
import us.codecraft.webmagic.oo.OOSpider;
import us.codecraft.webmagic.oo.TargetUrl;
import us.codecraft.webmagic.model.ExtractBy;
import us.codecraft.webmagic.model.OOSpider;
import us.codecraft.webmagic.model.TargetUrl;
/**
* @author code4crafter@gmail.com <br>