webMagic-Mobai
/
magic-Dependency
mirror of https://gitea.qinmian.online/Mobai/magic-Dependency.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update api for direct download

master
yihua.huang 2013-11-06 12:46:50 +08:00
parent 807aefe9df
commit 6e32a19f80
22 changed files with 167 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
View File

@ -6,6 +6,7 @@ import org.apache.log4j.Logger;
import us.codecraft.webmagic.downloader.Downloader;
import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.pipeline.CollectorPipeline;
import us.codecraft.webmagic.pipeline.ResultItemsCollectorPipeline;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;
@ -424,21 +425,25 @@ public class Spider implements Runnable, Task {
* @param urls
* @return
*/
public List<ResultItems> getAll(Collection<String> urls) {
public <T> List<T> getAll(Collection<String> urls) {
destroyWhenExit = false;
spawnUrl = false;
startRequests = UrlUtils.convertToRequests(urls);
CollectorPipeline collectorPipeline = new CollectorPipeline();
CollectorPipeline collectorPipeline = getCollectorPipeline();
pipelines.add(collectorPipeline);
run();
spawnUrl = true;
destroyWhenExit = true;
return collectorPipeline.getCollector();
return collectorPipeline.getCollected();
}
public ResultItems get(String url) {
protected CollectorPipeline getCollectorPipeline() {
return new ResultItemsCollectorPipeline();
}
public <T> T get(String url) {
List<String> urls = Lists.newArrayList(url);
List<ResultItems> resultItemses = getAll(urls);
List<T> resultItemses = getAll(urls);
if (resultItemses != null && resultItemses.size() > 0) {
return resultItemses.get(0);
} else {

25
webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/CollectorPipeline.java
View File

@ -1,25 +1,20 @@
package us.codecraft.webmagic.pipeline;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import java.util.ArrayList;
import java.util.List;
/**
* Pipeline that can collect and store results. <br>
* Used for {@link us.codecraft.webmagic.Spider#getAll(java.util.Collection)}
*
* @author code4crafter@gmail.com
* @since 0.4.0
*/
public class CollectorPipeline implements Pipeline{
public interface CollectorPipeline<T> extends Pipeline {
private List<ResultItems> collector = new ArrayList<ResultItems>();
@Override
public void process(ResultItems resultItems, Task task) {
collector.add(resultItems);
}
public List<ResultItems> getCollector() {
return collector;
}
/**
* Get all results collected.
*
* @return collected results
*/
public List<T> getCollected();
}

26
webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/ResultItemsCollectorPipeline.java 100644
View File

@ -0,0 +1,26 @@
package us.codecraft.webmagic.pipeline;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import java.util.ArrayList;
import java.util.List;
/**
* @author code4crafter@gmail.com
* @since 0.4.0
*/
public class ResultItemsCollectorPipeline implements CollectorPipeline<ResultItems> {
private List<ResultItems> collector = new ArrayList<ResultItems>();
@Override
public void process(ResultItems resultItems, Task task) {
collector.add(resultItems);
}
@Override
public List<ResultItems> getCollected() {
return collector;
}
}

2
webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/BaiduBaikePageProcesser.java
View File

@ -38,7 +38,7 @@ public class BaiduBaikePageProcesser implements PageProcessor {
list.add(String.format(urlTemplate,"太阳能"));
list.add(String.format(urlTemplate,"地热发电"));
list.add(String.format(urlTemplate,"地热发电"));
List<ResultItems> resultItemses = spider.getAll(list);
List<ResultItems> resultItemses = spider.<ResultItems>getAll(list);
for (ResultItems resultItemse : resultItemses) {
System.out.println(resultItemse.getAll());
}

28
webmagic-extension/src/main/java/us/codecraft/webmagic/example/BaiduBaike.java
View File

@ -1,10 +1,8 @@
package us.codecraft.webmagic.example;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.model.AfterExtractor;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.model.OOSpider;
import us.codecraft.webmagic.model.annotation.ExtractBy;
import us.codecraft.webmagic.model.direct.Param;
import java.util.ArrayList;
import java.util.List;
@ -13,21 +11,31 @@ import java.util.List;
* @since 0.4.0
* @author code4crafter@gmail.com
*/
public class BaiduBaike implements AfterExtractor{
public class BaiduBaike{
private String word;
@ExtractBy("//h1[@class=title]/div[@class=lemmaTitleH1]/text()")
private String name;
@ExtractBy("//div[@id='lemmaContent-0']//div[@class='para']/allText()")
private String description;
@Override
public void afterProcess(Page page) {
public String toString() {
return "BaiduBaike{" +
"name='" + name + '\'' +
", description='" + description + '\'' +
'}';
}
public static void main(String[] args) {
List<Param> words = new ArrayList<Param>();
words.add(new Param().put("word","红烧肉"));
OOSpider.direct(words, BaiduBaike.class).thread(10).run();
List<String> list = new ArrayList<String>();
String urlTemplate = "http://baike.baidu.com/search/word?word=%s&pic=1&sug=1&enc=utf8";
list.add(String.format(urlTemplate,"水力发电"));
list.add(String.format(urlTemplate,"风力发电"));
list.add(String.format(urlTemplate,"太阳能"));
list.add(String.format(urlTemplate,"地热发电"));
list.add(String.format(urlTemplate, "地热发电"));
List<BaiduBaike> baiduBaikes = OOSpider.create(Site.me().setSleepTime(100), BaiduBaike.class).<BaiduBaike>getAll(list);
System.out.println(baiduBaikes);
}
}

5
webmagic-extension/src/main/java/us/codecraft/webmagic/example/GithubRepo.java
View File

@ -41,8 +41,9 @@ public class GithubRepo implements HasKey {
private String url;
public static void main(String[] args) {
OOSpider.create(Site.me().addStartUrl("https://github.com/code4craft").setSleepTime(100)
, new ConsolePageModelPipeline(), GithubRepo.class).thread(10).run();
OOSpider.create(Site.me().setSleepTime(100)
, new ConsolePageModelPipeline(), GithubRepo.class)
.addUrl("https://github.com/code4craft").thread(10).run();
}
@Override

5
webmagic-extension/src/main/java/us/codecraft/webmagic/example/OschinaBlog.java
View File

@ -31,8 +31,9 @@ public class OschinaBlog {
private Date date;
public static void main(String[] args) {
OOSpider.create(Site.me().addStartUrl("http://my.oschina.net/flashsword/blog")
, new JsonFilePageModelPipeline("/data/webmagic/"), OschinaBlog.class).run();
OOSpider.create(Site.me()
, new JsonFilePageModelPipeline("/data/webmagic/"), OschinaBlog.class)
.addUrl("http://my.oschina.net/flashsword/blog").run();
}
public String getTitle() {

1
webmagic-extension/src/main/java/us/codecraft/webmagic/model/ConsolePageModelPipeline.java
View File

@ -2,6 +2,7 @@ package us.codecraft.webmagic.model;
import org.apache.commons.lang3.builder.ToStringBuilder;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.pipeline.PageModelPipeline;
/**
* Print page model in console.<br>

1
webmagic-extension/src/main/java/us/codecraft/webmagic/model/ModelPipeline.java
View File

@ -3,6 +3,7 @@ package us.codecraft.webmagic.model;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.model.annotation.ExtractBy;
import us.codecraft.webmagic.pipeline.PageModelPipeline;
import us.codecraft.webmagic.pipeline.Pipeline;
import java.lang.annotation.Annotation;

50
webmagic-extension/src/main/java/us/codecraft/webmagic/model/OOSpider.java
View File

@ -2,10 +2,12 @@ package us.codecraft.webmagic.model;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.model.direct.Param;
import us.codecraft.webmagic.pipeline.CollectorPipeline;
import us.codecraft.webmagic.pipeline.PageModelPipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import java.util.Collection;
import java.util.ArrayList;
import java.util.List;
/**
* The spider for page model extractor.<br>
@ -36,12 +38,16 @@ import java.util.Collection;
* @author code4crafter@gmail.com <br>
* @since 0.2.0
*/
public class OOSpider extends Spider {
public class OOSpider<T> extends Spider {
private ModelPageProcessor modelPageProcessor;
private ModelPipeline modelPipeline;
private PageModelPipeline pageModelPipeline;
private List<Class> pageModelClasses = new ArrayList<Class>();
protected OOSpider(ModelPageProcessor modelPageProcessor) {
super(modelPageProcessor);
this.modelPageProcessor = modelPageProcessor;
@ -62,13 +68,19 @@ public class OOSpider extends Spider {
this(ModelPageProcessor.create(site, pageModels));
this.modelPipeline = new ModelPipeline();
super.addPipeline(modelPipeline);
if (pageModelPipeline != null) {
for (Class pageModel : pageModels) {
for (Class pageModel : pageModels) {
if (pageModelPipeline != null) {
this.modelPipeline.put(pageModel, pageModelPipeline);
}
pageModelClasses.add(pageModel);
}
}
@Override
protected CollectorPipeline getCollectorPipeline() {
return new PageModelCollectorPipeline<T>(pageModelClasses.get(0));
}
public static OOSpider create(Site site, Class... pageModels) {
return new OOSpider(site, null, pageModels);
}
@ -77,34 +89,6 @@ public class OOSpider extends Spider {
return new OOSpider(site, pageModelPipeline, pageModels);
}
/**
* @since 0.3.3
* NO implement yet!
*/
public static OOSpider direct(Site site, PageModelPipeline pageModelPipeline, Class... pageModels) {
return new OOSpider(site, pageModelPipeline, pageModels);
}
/**
* @since 0.3.3
* NO implement yet!
*/
public static OOSpider direct(PageModelPipeline pageModelPipeline, Class... pageModels) {
return new OOSpider(null, pageModelPipeline, pageModels);
}
/**
* @since 0.3.3
* NO implement yet!
*/
public static OOSpider direct(Class... pageModels) {
return new OOSpider(null, null, pageModels);
}
public static OOSpider direct(Collection<Param> params, Class... pageModels) {
return new OOSpider(null, null, pageModels);
}
public OOSpider addPageModel(PageModelPipeline pageModelPipeline, Class... pageModels) {
for (Class pageModel : pageModels) {
modelPageProcessor.addPageModel(pageModel);

46
webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelCollectorPipeline.java 100644
View File

@ -0,0 +1,46 @@
package us.codecraft.webmagic.model;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.model.annotation.ExtractBy;
import us.codecraft.webmagic.pipeline.CollectorPageModelPipeline;
import us.codecraft.webmagic.pipeline.CollectorPipeline;
import java.lang.annotation.Annotation;
import java.util.List;
/**
* @author code4crafter@gmail.com
* @since 0.4.0
*/
class PageModelCollectorPipeline<T> implements CollectorPipeline<T> {
private final CollectorPageModelPipeline<T> classPipeline = new CollectorPageModelPipeline<T>();
private final Class<?> clazz;
PageModelCollectorPipeline(Class<?> clazz) {
this.clazz = clazz;
}
@Override
public List<T> getCollected() {
return classPipeline.getCollected();
}
@Override
public synchronized void process(ResultItems resultItems, Task task) {
Object o = resultItems.get(clazz.getCanonicalName());
if (o != null) {
Annotation annotation = clazz.getAnnotation(ExtractBy.class);
if (annotation == null || !((ExtractBy) annotation).multi()) {
classPipeline.process((T) o, task);
} else {
List<Object> list = (List<Object>) o;
for (Object o1 : list) {
classPipeline.process((T) o1, task);
}
}
}
}
}

2
webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelExtractor.java
View File

@ -195,7 +195,7 @@ class PageModelExtractor {
private void initClassExtractors() {
Annotation annotation = clazz.getAnnotation(TargetUrl.class);
if (annotation == null) {
targetUrlPatterns.add(Pattern.compile(".*"));
targetUrlPatterns.add(Pattern.compile("(.*)"));
} else {
TargetUrl targetUrl = (TargetUrl) annotation;
String[] value = targetUrl.value();

23
webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/CollectorPageModelPipeline.java 100644
View File

@ -0,0 +1,23 @@
package us.codecraft.webmagic.pipeline;
import us.codecraft.webmagic.Task;
import java.util.ArrayList;
import java.util.List;
/**
* @author code4crafter@gmail.com
*/
public class CollectorPageModelPipeline<T> implements PageModelPipeline<T> {
private List<T> collected = new ArrayList<T>();
@Override
public synchronized void process(T t, Task task) {
collected.add(t);
}
public List<T> getCollected() {
return collected;
}
}

1
webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/FilePageModelPipeline.java
View File

@ -5,7 +5,6 @@ import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.log4j.Logger;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.model.HasKey;
import us.codecraft.webmagic.model.PageModelPipeline;
import us.codecraft.webmagic.utils.FilePersistentBase;
import java.io.FileWriter;

1
webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/JsonFilePageModelPipeline.java
View File

@ -6,7 +6,6 @@ import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.log4j.Logger;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.model.HasKey;
import us.codecraft.webmagic.model.PageModelPipeline;
import us.codecraft.webmagic.utils.FilePersistentBase;
import java.io.FileWriter;

2
webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelPipeline.java → webmagic-extension/src/main/java/us/codecraft/webmagic/pipeline/PageModelPipeline.java
View File

@ -1,4 +1,4 @@
package us.codecraft.webmagic.model;
package us.codecraft.webmagic.pipeline;
import us.codecraft.webmagic.Task;

2
webmagic-extension/src/test/java/us/codecraft/webmagic/MockPageModelPipeline.java
View File

@ -1,7 +1,7 @@
package us.codecraft.webmagic;
import junit.framework.Assert;
import us.codecraft.webmagic.model.PageModelPipeline;
import us.codecraft.webmagic.pipeline.PageModelPipeline;
/**
* @author code4crafter@gmail.com

1
webmagic-extension/src/test/java/us/codecraft/webmagic/model/GithubRepoTest.java
View File

@ -6,6 +6,7 @@ import us.codecraft.webmagic.MockDownloader;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.example.GithubRepo;
import us.codecraft.webmagic.pipeline.PageModelPipeline;
/**
* @author code4crafter@gmail.com <br>

2
webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/Kr36NewsModel.java
View File

@ -3,7 +3,7 @@ package us.codecraft.webmagic.model.samples;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.model.OOSpider;
import us.codecraft.webmagic.model.PageModelPipeline;
import us.codecraft.webmagic.pipeline.PageModelPipeline;
import us.codecraft.webmagic.model.annotation.ExtractBy;
import us.codecraft.webmagic.model.annotation.ExtractByUrl;
import us.codecraft.webmagic.model.annotation.HelpUrl;

2
webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/OschinaBlog.java
View File

@ -3,7 +3,7 @@ package us.codecraft.webmagic.model.samples;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.model.OOSpider;
import us.codecraft.webmagic.model.PageModelPipeline;
import us.codecraft.webmagic.pipeline.PageModelPipeline;
import us.codecraft.webmagic.model.annotation.ExtractBy;
import us.codecraft.webmagic.model.annotation.TargetUrl;

2
zh_docs/us/codecraft/webmagic/model/OOSpider-cmnt.xml
View File

@ -12,7 +12,7 @@
]]></data>
</comment>
<comment>
<key><![CDATA[us.codecraft.webmagic.model.OOSpider(us.codecraft.webmagic.Site, us.codecraft.webmagic.model.PageModelPipeline, java.lang.Class...)]]></key>
<key><![CDATA[us.codecraft.webmagic.model.OOSpider(us.codecraft.webmagic.Site, us.codecraft.webmagic.pipeline.PageModelPipeline, java.lang.Class...)]]></key>
<data><![CDATA[ 创建一个爬虫。<br>
@param site
@param pageModelPipeline

2
zh_docs/us/codecraft/webmagic/model/PageModelPipeline-cmnt.xml
View File

@ -4,7 +4,7 @@
<date-generated>Sat Aug 17 14:14:46 CST 2013</date-generated>
</meta>
<comment>
<key><![CDATA[us.codecraft.webmagic.model.PageModelPipeline]]></key>
<key><![CDATA[us.codecraft.webmagic.pipeline.PageModelPipeline]]></key>
<data><![CDATA[ @author code4crafter@gmail.com <br>
Date: 13-8-3 <br>
Time: 上午9:34 <br>