Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

split modules

master
yihua.huang 2013-06-08 20:48:27 +08:00
parent 3c3f001186
commit 6dc88fa111
131 changed files with 967 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
webmagic-core/pom.xml 100644
View File

@ -0,0 +1,105 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<groupId>us.codecraft</groupId>
<version>0.0.1-SNAPSHOT</version>
<modelVersion>4.0.0</modelVersion>
<artifactId>webmagic-core</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.2.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.7</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>13.0.1</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.1</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
<dependency>
<groupId>commons-collections</groupId>
<artifactId>commons-collections</artifactId>
<version>3.2.1</version>
</dependency>
<dependency>
<groupId>net.sourceforge.htmlcleaner</groupId>
<artifactId>htmlcleaner</artifactId>
<version>2.4</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-io</artifactId>
<version>1.3.2</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<configuration>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<executions>
<execution>
<id>attach-javadocs</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-release-plugin</artifactId>
<version>2.0-beta-7</version>
</plugin>
</plugins>
</build>
</project>

6
src/main/java/us/codecraft/spider/Page.java → webmagic-core/src/main/java/us/codecraft/webmagic/Page.java
View File

@ -1,8 +1,8 @@
package us.codecraft.spider; package us.codecraft.webmagic;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import us.codecraft.spider.selector.Selectable; import us.codecraft.webmagic.selector.Selectable;
import us.codecraft.spider.utils.UrlUtils; import us.codecraft.webmagic.utils.UrlUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;

6
src/main/java/us/codecraft/spider/Request.java → webmagic-core/src/main/java/us/codecraft/webmagic/Request.java
View File

@ -1,8 +1,4 @@
package us.codecraft.spider; package us.codecraft.webmagic;
import us.codecraft.spider.Site;
import java.util.List;
/** /**
* User: cairne * User: cairne

2
src/main/java/us/codecraft/spider/Site.java → webmagic-core/src/main/java/us/codecraft/webmagic/Site.java
View File

@ -1,4 +1,4 @@
package us.codecraft.spider; package us.codecraft.webmagic;
import java.util.HashSet; import java.util.HashSet;
import java.util.Set; import java.util.Set;

19
src/main/java/us/codecraft/spider/Spider.java → webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
View File

@ -1,17 +1,14 @@
package us.codecraft.spider; package us.codecraft.webmagic;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import us.codecraft.spider.downloader.Downloader; import us.codecraft.webmagic.downloader.Downloader;
import us.codecraft.spider.downloader.HttpClientDownloader; import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.spider.pipeline.ConsolePipeline; import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.spider.pipeline.Pipeline; import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.spider.processor.PageProcessor; import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.spider.schedular.QueueSchedular; import us.codecraft.webmagic.schedular.QueueSchedular;
import us.codecraft.spider.schedular.Schedular; import us.codecraft.webmagic.schedular.Schedular;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/** /**
* User: cairne * User: cairne

8
src/main/java/us/codecraft/spider/downloader/Downloader.java → webmagic-core/src/main/java/us/codecraft/webmagic/downloader/Downloader.java
View File

@ -1,8 +1,8 @@
package us.codecraft.spider.downloader; package us.codecraft.webmagic.downloader;
import us.codecraft.spider.Page; import us.codecraft.webmagic.Page;
import us.codecraft.spider.Request; import us.codecraft.webmagic.Request;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
/** /**
* User: cairne * User: cairne

14
src/main/java/us/codecraft/spider/downloader/HttpClientDownloader.java → webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -1,16 +1,16 @@
package us.codecraft.spider.downloader; package us.codecraft.webmagic.downloader;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.apache.http.HttpResponse; import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient; import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpGet;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import us.codecraft.spider.Page; import us.codecraft.webmagic.Page;
import us.codecraft.spider.Request; import us.codecraft.webmagic.Request;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
import us.codecraft.spider.selector.Html; import us.codecraft.webmagic.selector.Html;
import us.codecraft.spider.selector.PlainText; import us.codecraft.webmagic.selector.PlainText;
import us.codecraft.spider.utils.UrlUtils; import us.codecraft.webmagic.utils.UrlUtils;
/** /**

4
src/main/java/us/codecraft/spider/downloader/HttpClientPool.java → webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientPool.java
View File

@ -1,4 +1,4 @@
package us.codecraft.spider.downloader; package us.codecraft.webmagic.downloader;
import org.apache.http.HttpVersion; import org.apache.http.HttpVersion;
import org.apache.http.client.HttpClient; import org.apache.http.client.HttpClient;
@ -10,7 +10,7 @@ import org.apache.http.conn.scheme.SchemeRegistry;
import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.PoolingClientConnectionManager; import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.http.params.*; import org.apache.http.params.*;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
/** /**
* User: cairne * User: cairne

8
src/main/java/us/codecraft/spider/pipeline/ConsolePipeline.java → webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/ConsolePipeline.java
View File

@ -1,8 +1,8 @@
package us.codecraft.spider.pipeline; package us.codecraft.webmagic.pipeline;
import us.codecraft.spider.Page; import us.codecraft.webmagic.Page;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
import us.codecraft.spider.selector.Selectable; import us.codecraft.webmagic.selector.Selectable;
import java.util.Map; import java.util.Map;

10
src/main/java/us/codecraft/spider/pipeline/FilePipeline.java → webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/FilePipeline.java
View File

@ -1,10 +1,10 @@
package us.codecraft.spider.pipeline; package us.codecraft.webmagic.pipeline;
import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import us.codecraft.spider.Page; import us.codecraft.webmagic.Page;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
import us.codecraft.spider.selector.Selectable; import us.codecraft.webmagic.selector.Selectable;
import java.io.File; import java.io.File;
import java.io.FileWriter; import java.io.FileWriter;
@ -19,7 +19,7 @@ import java.util.Map;
*/ */
public class FilePipeline implements Pipeline { public class FilePipeline implements Pipeline {
private String path = "/data/temp/spider/"; private String path = "/data/temp/webmagic/";
public FilePipeline(){ public FilePipeline(){

6
src/main/java/us/codecraft/spider/pipeline/Pipeline.java → webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/Pipeline.java
View File

@ -1,7 +1,7 @@
package us.codecraft.spider.pipeline; package us.codecraft.webmagic.pipeline;
import us.codecraft.spider.Page; import us.codecraft.webmagic.Page;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
/** /**
* User: cairne * User: cairne

6
src/main/java/us/codecraft/spider/processor/PageProcessor.java → webmagic-core/src/main/java/us/codecraft/webmagic/processor/PageProcessor.java
View File

@ -1,7 +1,7 @@
package us.codecraft.spider.processor; package us.codecraft.webmagic.processor;
import us.codecraft.spider.Page; import us.codecraft.webmagic.Page;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
/** /**
* User: cairne * User: cairne

8
src/main/java/us/codecraft/spider/processor/SimplePageProcessor.java → webmagic-core/src/main/java/us/codecraft/webmagic/processor/SimplePageProcessor.java
View File

@ -1,8 +1,8 @@
package us.codecraft.spider.processor; package us.codecraft.webmagic.processor;
import us.codecraft.spider.Page; import us.codecraft.webmagic.Page;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
import us.codecraft.spider.utils.UrlUtils; import us.codecraft.webmagic.utils.UrlUtils;
import java.util.List; import java.util.List;

6
src/main/java/us/codecraft/spider/schedular/FileCacheQueueSchedular.java → webmagic-core/src/main/java/us/codecraft/webmagic/schedular/FileCacheQueueSchedular.java
View File

@ -1,9 +1,9 @@
package us.codecraft.spider.schedular; package us.codecraft.webmagic.schedular;
import org.apache.commons.lang3.math.NumberUtils; import org.apache.commons.lang3.math.NumberUtils;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
import us.codecraft.spider.Request; import us.codecraft.webmagic.Request;
import java.io.*; import java.io.*;
import java.util.LinkedHashSet; import java.util.LinkedHashSet;

6
src/main/java/us/codecraft/spider/schedular/QueueSchedular.java → webmagic-core/src/main/java/us/codecraft/webmagic/schedular/QueueSchedular.java
View File

@ -1,8 +1,8 @@
package us.codecraft.spider.schedular; package us.codecraft.webmagic.schedular;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import us.codecraft.spider.Request; import us.codecraft.webmagic.Request;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
import java.util.HashSet; import java.util.HashSet;
import java.util.Set; import java.util.Set;

6
src/main/java/us/codecraft/spider/schedular/Schedular.java → webmagic-core/src/main/java/us/codecraft/webmagic/schedular/Schedular.java
View File

@ -1,7 +1,7 @@
package us.codecraft.spider.schedular; package us.codecraft.webmagic.schedular;
import us.codecraft.spider.Request; import us.codecraft.webmagic.Request;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
/** /**
* User: cairne * User: cairne

4
src/main/java/us/codecraft/spider/selector/Html.java → webmagic-core/src/main/java/us/codecraft/webmagic/selector/Html.java
View File

@ -1,6 +1,4 @@
package us.codecraft.spider.selector; package us.codecraft.webmagic.selector;
import org.apache.commons.collections.CollectionUtils;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;

2
src/main/java/us/codecraft/spider/selector/PlainText.java → webmagic-core/src/main/java/us/codecraft/webmagic/selector/PlainText.java
View File

@ -1,4 +1,4 @@
package us.codecraft.spider.selector; package us.codecraft.webmagic.selector;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;

2
src/main/java/us/codecraft/spider/selector/RegexResult.java → webmagic-core/src/main/java/us/codecraft/webmagic/selector/RegexResult.java
View File

@ -1,4 +1,4 @@
package us.codecraft.spider.selector; package us.codecraft.webmagic.selector;
/** /**
* User: cairne * User: cairne

2
src/main/java/us/codecraft/spider/selector/RegexSelector.java → webmagic-core/src/main/java/us/codecraft/webmagic/selector/RegexSelector.java
View File

@ -1,4 +1,4 @@
package us.codecraft.spider.selector; package us.codecraft.webmagic.selector;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;

2
src/main/java/us/codecraft/spider/selector/ReplaceSelector.java → webmagic-core/src/main/java/us/codecraft/webmagic/selector/ReplaceSelector.java
View File

@ -1,4 +1,4 @@
package us.codecraft.spider.selector; package us.codecraft.webmagic.selector;
import java.util.List; import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;

2
src/main/java/us/codecraft/spider/selector/Selectable.java → webmagic-core/src/main/java/us/codecraft/webmagic/selector/Selectable.java
View File

@ -1,4 +1,4 @@
package us.codecraft.spider.selector; package us.codecraft.webmagic.selector;
import java.util.List; import java.util.List;

2
src/main/java/us/codecraft/spider/selector/Selector.java → webmagic-core/src/main/java/us/codecraft/webmagic/selector/Selector.java
View File

@ -1,4 +1,4 @@
package us.codecraft.spider.selector; package us.codecraft.webmagic.selector;
import java.util.List; import java.util.List;

2
src/main/java/us/codecraft/spider/selector/SelectorFactory.java → webmagic-core/src/main/java/us/codecraft/webmagic/selector/SelectorFactory.java
View File

@ -1,4 +1,4 @@
package us.codecraft.spider.selector; package us.codecraft.webmagic.selector;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;

2
src/main/java/us/codecraft/spider/selector/SmartContentSelector.java → webmagic-core/src/main/java/us/codecraft/webmagic/selector/SmartContentSelector.java
View File

@ -1,4 +1,4 @@
package us.codecraft.spider.selector; package us.codecraft.webmagic.selector;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import org.htmlcleaner.HtmlCleaner; import org.htmlcleaner.HtmlCleaner;

2
src/main/java/us/codecraft/spider/selector/XpathSelector.java → webmagic-core/src/main/java/us/codecraft/webmagic/selector/XpathSelector.java
View File

@ -1,4 +1,4 @@
package us.codecraft.spider.selector; package us.codecraft.webmagic.selector;
import org.htmlcleaner.*; import org.htmlcleaner.*;

2
src/main/java/us/codecraft/spider/utils/UrlUtils.java → webmagic-core/src/main/java/us/codecraft/webmagic/utils/UrlUtils.java
View File

@ -1,4 +1,4 @@
package us.codecraft.spider.utils; package us.codecraft.webmagic.utils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;

0
src/main/resources/log4j.xml → webmagic-core/src/main/resources/log4j.xml
View File

4
src/test/java/us/codecraft/spider/HtmlTest.java → webmagic-core/src/test/java/us/codecraft/webmagic/HtmlTest.java
View File

@ -1,8 +1,8 @@
package us.codecraft.spider; package us.codecraft.webmagic;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import us.codecraft.spider.selector.Html; import us.codecraft.webmagic.selector.Html;
/** /**
* User: cairne * User: cairne

14
src/test/java/us/codecraft/spider/SpiderTest.java → webmagic-core/src/test/java/us/codecraft/webmagic/SpiderTest.java
View File

@ -1,11 +1,11 @@
package us.codecraft.spider; package us.codecraft.webmagic;
import org.junit.Ignore; import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import us.codecraft.spider.pipeline.FilePipeline; import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.spider.processor.SimplePageProcessor; import us.codecraft.webmagic.processor.SimplePageProcessor;
import us.codecraft.spider.samples.HuxiuProcessor; import us.codecraft.webmagic.samples.HuxiuProcessor;
import us.codecraft.spider.schedular.FileCacheQueueSchedular; import us.codecraft.webmagic.schedular.FileCacheQueueSchedular;
/** /**
* User: cairne * User: cairne
@ -24,12 +24,12 @@ public class SpiderTest {
@Test @Test
public void testGlobalSpider(){ public void testGlobalSpider(){
// PageProcessor pageProcessor = new MeicanProcessor(); // PageProcessor pageProcessor = new MeicanProcessor();
// Spider.me().pipeline(new FilePipeline()).schedular(new FileCacheQueueSchedular(pageProcessor.getSite(),"/data/temp/spider/cache/")). // Spider.me().pipeline(new FilePipeline()).schedular(new FileCacheQueueSchedular(pageProcessor.getSite(),"/data/temp/webmagic/cache/")).
// processor(pageProcessor).run(); // processor(pageProcessor).run();
SimplePageProcessor pageProcessor2 = new SimplePageProcessor("http://www.diaoyuweng.com/home.php?mod=space&uid=88304&do=thread&view=me&type=thread&from=space", "http://www.diaoyuweng.com/thread-*-1-1.html"); SimplePageProcessor pageProcessor2 = new SimplePageProcessor("http://www.diaoyuweng.com/home.php?mod=space&uid=88304&do=thread&view=me&type=thread&from=space", "http://www.diaoyuweng.com/thread-*-1-1.html");
pageProcessor2.getSite().setEncoding("GBK"); pageProcessor2.getSite().setEncoding("GBK");
System.out.println(pageProcessor2.getSite().getEncoding()); System.out.println(pageProcessor2.getSite().getEncoding());
Spider.me().pipeline(new FilePipeline()).schedular(new FileCacheQueueSchedular(pageProcessor2.getSite(),"/data/temp/spider/cache/")). Spider.me().pipeline(new FilePipeline()).schedular(new FileCacheQueueSchedular(pageProcessor2.getSite(),"/data/temp/webmagic/cache/")).
processor(pageProcessor2).run(); processor(pageProcessor2).run();

8
src/test/java/us/codecraft/spider/samples/DiandianBlogProcessor.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/DiandianBlogProcessor.java
View File

@ -1,8 +1,8 @@
package us.codecraft.spider.samples; package us.codecraft.webmagic.samples;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
import us.codecraft.spider.Page; import us.codecraft.webmagic.Page;
import us.codecraft.spider.processor.PageProcessor; import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List; import java.util.List;

8
src/test/java/us/codecraft/spider/samples/DianpingBlogProcessor.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/DianpingBlogProcessor.java
View File

@ -1,8 +1,8 @@
package us.codecraft.spider.samples; package us.codecraft.webmagic.samples;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
import us.codecraft.spider.Page; import us.codecraft.webmagic.Page;
import us.codecraft.spider.processor.PageProcessor; import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List; import java.util.List;

8
src/test/java/us/codecraft/spider/samples/DiaoyuwengProcessor.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/DiaoyuwengProcessor.java
View File

@ -1,8 +1,8 @@
package us.codecraft.spider.samples; package us.codecraft.webmagic.samples;
import us.codecraft.spider.Page; import us.codecraft.webmagic.Page;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
import us.codecraft.spider.processor.PageProcessor; import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List; import java.util.List;

8
src/test/java/us/codecraft/spider/samples/F58PageProcesser.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/F58PageProcesser.java
View File

@ -1,8 +1,8 @@
package us.codecraft.spider.samples; package us.codecraft.webmagic.samples;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
import us.codecraft.spider.Page; import us.codecraft.webmagic.Page;
import us.codecraft.spider.processor.PageProcessor; import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List; import java.util.List;

8
src/test/java/us/codecraft/spider/samples/HuxiuProcessor.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/HuxiuProcessor.java
View File

@ -1,8 +1,8 @@
package us.codecraft.spider.samples; package us.codecraft.webmagic.samples;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
import us.codecraft.spider.Page; import us.codecraft.webmagic.Page;
import us.codecraft.spider.processor.PageProcessor; import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List; import java.util.List;

8
src/test/java/us/codecraft/spider/samples/KaichibaProcessor.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/KaichibaProcessor.java
View File

@ -1,8 +1,8 @@
package us.codecraft.spider.samples; package us.codecraft.webmagic.samples;
import us.codecraft.spider.Page; import us.codecraft.webmagic.Page;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
import us.codecraft.spider.processor.PageProcessor; import us.codecraft.webmagic.processor.PageProcessor;
/** /**
* User: cairne * User: cairne

8
src/test/java/us/codecraft/spider/samples/MeicanProcessor.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/MeicanProcessor.java
View File

@ -1,8 +1,8 @@
package us.codecraft.spider.samples; package us.codecraft.webmagic.samples;
import us.codecraft.spider.Page; import us.codecraft.webmagic.Page;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
import us.codecraft.spider.processor.PageProcessor; import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List; import java.util.List;

8
src/test/java/us/codecraft/spider/samples/NjuBBSProcessor.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/NjuBBSProcessor.java
View File

@ -1,8 +1,8 @@
package us.codecraft.spider.samples; package us.codecraft.webmagic.samples;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
import us.codecraft.spider.Page; import us.codecraft.webmagic.Page;
import us.codecraft.spider.processor.PageProcessor; import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List; import java.util.List;

8
src/test/java/us/codecraft/spider/samples/OschinaBlogPageProcesser.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/OschinaBlogPageProcesser.java
View File

@ -1,8 +1,8 @@
package us.codecraft.spider.samples; package us.codecraft.webmagic.samples;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
import us.codecraft.spider.Page; import us.codecraft.webmagic.Page;
import us.codecraft.spider.processor.PageProcessor; import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List; import java.util.List;

8
src/test/java/us/codecraft/spider/samples/OschinaPageProcesser.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/OschinaPageProcesser.java
View File

@ -1,8 +1,8 @@
package us.codecraft.spider.samples; package us.codecraft.webmagic.samples;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
import us.codecraft.spider.Page; import us.codecraft.webmagic.Page;
import us.codecraft.spider.processor.PageProcessor; import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List; import java.util.List;

8
src/test/java/us/codecraft/spider/samples/QzoneBlogProcessor.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/QzoneBlogProcessor.java
View File

@ -1,8 +1,8 @@
package us.codecraft.spider.samples; package us.codecraft.webmagic.samples;
import us.codecraft.spider.Page; import us.codecraft.webmagic.Page;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
import us.codecraft.spider.processor.PageProcessor; import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List; import java.util.List;

8
src/test/java/us/codecraft/spider/samples/SinaBlogProcesser.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/SinaBlogProcesser.java
View File

@ -1,8 +1,8 @@
package us.codecraft.spider.samples; package us.codecraft.webmagic.samples;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
import us.codecraft.spider.Page; import us.codecraft.webmagic.Page;
import us.codecraft.spider.processor.PageProcessor; import us.codecraft.webmagic.processor.PageProcessor;
/** /**
* User: cairne * User: cairne

8
src/test/java/us/codecraft/spider/samples/TianyaPageProcesser.java → webmagic-core/src/test/java/us/codecraft/webmagic/samples/TianyaPageProcesser.java
View File

@ -1,8 +1,8 @@
package us.codecraft.spider.samples; package us.codecraft.webmagic.samples;
import us.codecraft.spider.Site; import us.codecraft.webmagic.Site;
import us.codecraft.spider.Page; import us.codecraft.webmagic.Page;
import us.codecraft.spider.processor.PageProcessor; import us.codecraft.webmagic.processor.PageProcessor;
import java.util.List; import java.util.List;

3
src/test/java/us/codecraft/spider/selector/HtmlCleanerTest.java → webmagic-core/src/test/java/us/codecraft/webmagic/selector/HtmlCleanerTest.java
View File

@ -1,4 +1,4 @@
package us.codecraft.spider.selector; package us.codecraft.webmagic.selector;
import org.htmlcleaner.CleanerProperties; import org.htmlcleaner.CleanerProperties;
import org.htmlcleaner.HtmlCleaner; import org.htmlcleaner.HtmlCleaner;
@ -6,7 +6,6 @@ import org.htmlcleaner.TagNode;
import org.junit.Test; import org.junit.Test;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
/** /**

2
src/test/java/us/codecraft/spider/selector/RegexSelectorTest.java → webmagic-core/src/test/java/us/codecraft/webmagic/selector/RegexSelectorTest.java
View File

@ -1,4 +1,4 @@
package us.codecraft.spider.selector; package us.codecraft.webmagic.selector;
import junit.framework.Assert; import junit.framework.Assert;
import org.junit.Test; import org.junit.Test;

6
src/test/java/us/codecraft/spider/selector/SmartConentSelectorTest.java → webmagic-core/src/test/java/us/codecraft/webmagic/selector/SmartConentSelectorTest.java
View File

@ -1,12 +1,8 @@
package us.codecraft.spider.selector; package us.codecraft.webmagic.selector;
import org.htmlcleaner.CleanerProperties;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import org.junit.Test; import org.junit.Test;
import java.io.IOException; import java.io.IOException;
import java.net.URL;
/** /**
* User: cairne * User: cairne

2
src/test/java/us/codecraft/spider/selector/XpathSelectorTest.java → webmagic-core/src/test/java/us/codecraft/webmagic/selector/XpathSelectorTest.java
View File

@ -1,4 +1,4 @@
package us.codecraft.spider.selector; package us.codecraft.webmagic.selector;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;

2
src/test/java/us/codecraft/spider/utils/UrlUtilsTest.java → webmagic-core/src/test/java/us/codecraft/webmagic/utils/UrlUtilsTest.java
View File

@ -1,4 +1,4 @@
package us.codecraft.spider.utils; package us.codecraft.webmagic.utils;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;

0
src/test/resources/log4j.xml → webmagic-core/src/test/resources/log4j.xml
View File

26
webmagic-core/target/classes/log4j.xml 100644
View File

@ -0,0 +1,26 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
<appender name="stdout" class="org.apache.log4j.ConsoleAppender">
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
</layout>
</appender>
<logger name="org.springframework" additivity="false">
<level value="warn" />
<appender-ref ref="stdout" />
</logger>
<logger name="net.sf.ehcache" additivity="false">
<level value="warn" />
<appender-ref ref="stdout" />
</logger>
<root>
<level value="info" />
<appender-ref ref="stdout" />
</root>
</log4j:configuration>

BIN
webmagic-core/target/classes/us/codecraft/webmagic/Page.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/Request.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/Site.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/Spider.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/downloader/Downloader.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/downloader/HttpClientDownloader.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/downloader/HttpClientPool.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/pipeline/ConsolePipeline.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/pipeline/FilePipeline.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/pipeline/Pipeline.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/processor/PageProcessor.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/processor/SimplePageProcessor.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/schedular/FileCacheQueueSchedular$1.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/schedular/FileCacheQueueSchedular.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/schedular/QueueSchedular.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/schedular/Schedular.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/selector/Html.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/selector/PlainText.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/selector/RegexResult.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/selector/RegexSelector.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/selector/ReplaceSelector.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/selector/Selectable.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/selector/Selector.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/selector/SelectorFactory.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/selector/SmartContentSelector$1.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/selector/SmartContentSelector.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/selector/XpathSelector.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/classes/us/codecraft/webmagic/utils/UrlUtils.class 100644
View File

Binary file not shown.

31
webmagic-core/target/test-classes/log4j.xml 100644
View File

@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
<appender name="stdout" class="org.apache.log4j.ConsoleAppender">
<layout class="org.apache.log4j.PatternLayout">
<param name="ConversionPattern" value="%d{yy-MM-dd HH:mm:ss,SSS} %-5p %c(%F:%L) ## %m%n" />
</layout>
</appender>
<logger name="org.springframework" additivity="false">
<level value="warn" />
<appender-ref ref="stdout" />
</logger>
<logger name="org.apache" additivity="false">
<level value="warn" />
<appender-ref ref="stdout" />
</logger>
<logger name="net.sf.ehcache" additivity="false">
<level value="warn" />
<appender-ref ref="stdout" />
</logger>
<root>
<level value="debug" />
<appender-ref ref="stdout" />
</root>
</log4j:configuration>

BIN
webmagic-core/target/test-classes/us/codecraft/webmagic/HtmlTest.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/test-classes/us/codecraft/webmagic/SpiderTest.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/test-classes/us/codecraft/webmagic/samples/DiandianBlogProcessor.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/test-classes/us/codecraft/webmagic/samples/DianpingBlogProcessor.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/test-classes/us/codecraft/webmagic/samples/DiaoyuwengProcessor.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/test-classes/us/codecraft/webmagic/samples/F58PageProcesser.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/test-classes/us/codecraft/webmagic/samples/HuxiuProcessor.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/test-classes/us/codecraft/webmagic/samples/KaichibaProcessor.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/test-classes/us/codecraft/webmagic/samples/MeicanProcessor.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/test-classes/us/codecraft/webmagic/samples/NjuBBSProcessor.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/test-classes/us/codecraft/webmagic/samples/OschinaBlogPageProcesser.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/test-classes/us/codecraft/webmagic/samples/OschinaPageProcesser.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/test-classes/us/codecraft/webmagic/samples/QzoneBlogProcessor.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/test-classes/us/codecraft/webmagic/samples/SinaBlogProcesser.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/test-classes/us/codecraft/webmagic/samples/TianyaPageProcesser.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/test-classes/us/codecraft/webmagic/selector/HtmlCleanerTest.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/test-classes/us/codecraft/webmagic/selector/RegexSelectorTest.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/test-classes/us/codecraft/webmagic/selector/SmartConentSelectorTest.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/test-classes/us/codecraft/webmagic/selector/XpathSelectorTest.class 100644
View File

Binary file not shown.

BIN
webmagic-core/target/test-classes/us/codecraft/webmagic/utils/UrlUtilsTest.class 100644
View File

Binary file not shown.

67
webmagic-plugin/pom.xml 100644
View File

@ -0,0 +1,67 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<groupId>us.codecraft</groupId>
<version>0.0.1-SNAPSHOT</version>
<modelVersion>4.0.0</modelVersion>
<artifactId>webmagic-plugin</artifactId>
<dependencies>
<dependency>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-core</artifactId>
<version>0.0.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.7</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<configuration>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<executions>
<execution>
<id>attach-javadocs</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-release-plugin</artifactId>
<version>2.0-beta-7</version>
</plugin>
</plugins>
</build>
</project>

Some files were not shown because too many files have changed in this diff Show More