diff --git a/src/main/java/us/codecraft/spider/downloader/HttpClientDownloader.java b/src/main/java/us/codecraft/spider/downloader/HttpClientDownloader.java index 6167665..fc6a107 100644 --- a/src/main/java/us/codecraft/spider/downloader/HttpClientDownloader.java +++ b/src/main/java/us/codecraft/spider/downloader/HttpClientDownloader.java @@ -32,7 +32,7 @@ public class HttpClientDownloader implements Downloader { int statusCode = httpResponse.getStatusLine().getStatusCode(); if (site.getAcceptStatCode().contains(statusCode)) { String content = IOUtils.toString(httpResponse.getEntity().getContent(), - site.getEncoding() == null ? site.getEncoding() : httpResponse.getEntity().getContentType().getValue()); + site.getEncoding() == null ? httpResponse.getEntity().getContentType().getValue() : site.getEncoding()); Page page = new Page(); page.setHtml(new Html(UrlUtils.fixAllRelativeHrefs(content, request.getUrl()))); page.setUrl(new PlainText(request.getUrl())); diff --git a/src/main/java/us/codecraft/spider/utils/UrlUtils.java b/src/main/java/us/codecraft/spider/utils/UrlUtils.java index f2fb036..fdaa419 100644 --- a/src/main/java/us/codecraft/spider/utils/UrlUtils.java +++ b/src/main/java/us/codecraft/spider/utils/UrlUtils.java @@ -91,4 +91,15 @@ public class UrlUtils { return stringBuilder.toString(); } + private static final Pattern patternForCharset = Pattern.compile("charset=([^\\s;]*)"); + + public static String getCharset(String contentType) { + Matcher matcher = patternForCharset.matcher(contentType); + if (matcher.find()) { + return matcher.group(1); + } else { + return null; + } + } + } diff --git a/src/test/java/us/codecraft/spider/SpiderTest.java b/src/test/java/us/codecraft/spider/SpiderTest.java index c23b6ce..cbc84a2 100644 --- a/src/test/java/us/codecraft/spider/SpiderTest.java +++ b/src/test/java/us/codecraft/spider/SpiderTest.java @@ -25,14 +25,14 @@ public class SpiderTest { @Test public void testGlobalSpider(){ - SimplePageProcessor pageProcessor = new SimplePageProcessor("http://2012guang.diandian.com/", "http://2012guang.diandian.com/post/*"); + SimplePageProcessor pageProcessor = new SimplePageProcessor("http://blog.163.com/", "http://blog.163.com/*/blog/static/*"); + pageProcessor.getSite().setEncoding("gbk"); Spider.me().pipeline(new FilePipeline()).schedular(new FileCacheQueueSchedular(pageProcessor.getSite(),"/data/temp/spider/cache/")). - processor(pageProcessor).thread().start(); - SimplePageProcessor pageProcessor2 = new SimplePageProcessor("http://lol.duowan.com/", "http://lol.duowan.com/*.html"); - Spider.me().pipeline(new FilePipeline()).schedular(new FileCacheQueueSchedular(pageProcessor2.getSite(),"/data/temp/spider/cache/")). - processor(pageProcessor2).run(); + processor(pageProcessor).run(); +// SimplePageProcessor pageProcessor2 = new SimplePageProcessor("http://lol.duowan.com/", "http://lol.duowan.com/*.html"); +// Spider.me().pipeline(new FilePipeline()).schedular(new FileCacheQueueSchedular(pageProcessor2.getSite(),"/data/temp/spider/cache/")). +// processor(pageProcessor2).run(); - Spider.me().processor(new SimplePageProcessor("http://my.oschina.net/", "http://my.oschina.net/*/blog/*")).run(); } diff --git a/src/test/java/us/codecraft/spider/samples/QzoneBlogProcessor.java b/src/test/java/us/codecraft/spider/samples/QzoneBlogProcessor.java new file mode 100644 index 0000000..320494d --- /dev/null +++ b/src/test/java/us/codecraft/spider/samples/QzoneBlogProcessor.java @@ -0,0 +1,32 @@ +package us.codecraft.spider.samples; + +import us.codecraft.spider.Page; +import us.codecraft.spider.Site; +import us.codecraft.spider.processor.PageProcessor; + +import java.util.List; + +/** + * User: cairne + * Date: 13-4-21 + * Time: 下午8:08 + */ +public class QzoneBlogProcessor implements PageProcessor { + @Override + public void process(Page page) { + //http://progressdaily.diandian.com/post/2013-01-24/40046867275 + + //http://b1.cnc.qzone.qq.com/cgi-bin/blognew/get_abs?hostUin=233017404&uin=233017404&blogType=0&statYear=2013&source=0&statYear=2013&g_tk=291639571&g_tk=291639571&reqInfo=7&pos=0&num=15&source=0&rand=0.46480297949165106 + // &cateName=&cateHex=&statYear=2013&reqInfo=7&pos=0&num=15&sortType=0&source=0&rand=0.46480297949165106&g_tk=291639571&verbose=1&ref=qzone + List requests = page.getHtml().rs("]*href=[\"']{1}(http://17dujingdian\\.com/post/[^#]*?)[\"']{1}").toStrings(); + page.addTargetRequests(requests); + page.putField("title",page.getHtml().x("//div[@id='content']//h2/a")); + page.putField("content",page.getHtml().sc()); + } + + @Override + public Site getSite() { + return Site.me().setDomain("www.diandian.com").setStartUrl("http://17dujingdian.com/"). + setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31"); + } +}