Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix encoding

master
yihua.huang 2013-07-30 15:24:59 +08:00
parent 78cfb4d554
commit 52fd5cfc1c
3 changed files with 44 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
pom.xml
View File

@ -69,6 +69,7 @@
<configuration> <configuration>
<source>1.6</source> <source>1.6</source>
<target>1.6</target> <target>1.6</target>
<encoding>UTF-8</encoding>
</configuration> </configuration>
</plugin> </plugin>
<plugin> <plugin>

5
webmagic-core/src/test/java/us/codecraft/webmagic/utils/UrlUtilsTest.java
View File

@ -25,10 +25,7 @@ public class UrlUtilsTest {
Assert.assertEquals("http://www.dianping.com/aa", fixrelativeurl); Assert.assertEquals("http://www.dianping.com/aa", fixrelativeurl);
fixrelativeurl = UrlUtils.canonicalizeUrl("..aa", "http://www.dianping.com/sh/ss/com"); fixrelativeurl = UrlUtils.canonicalizeUrl("..aa", "http://www.dianping.com/sh/ss/com");
Assert.assertEquals("http://www.dianping.com/sh/ss/..aa", fixrelativeurl); Assert.assertEquals("http://www.dianping.com/sh/ss/..aa", fixrelativeurl);
// fixrelativeurl = fixrelativeurl("/aa", "http://www.dianping.com"); fixrelativeurl = UrlUtils.canonicalizeUrl("../../aa", "http://www.dianping.com/sh/ss/com");
// System.out.println("fix: " + fixrelativeurl);
// fixrelativeurl = fixrelativeurl("/aa", "http://www.dianping.com/");
// System.out.println("fix: " + fixrelativeurl);
} }
@Test @Test

20
webmagic-plugin/webmagic-selenium/src/test/java/us/codecraft/webmagic/selenium/downloader/SeleniumDownloaderTest.java
View File

@ -38,4 +38,24 @@ public class SeleniumDownloaderTest {
System.out.println(System.currentTimeMillis() - time1); System.out.println(System.currentTimeMillis() - time1);
} }
@Ignore
@Test
public void testBaiduWenku() {
SeleniumDownloader seleniumDownloader = new SeleniumDownloader(chromeDriverPath);
seleniumDownloader.setSleepTime(10000);
long time1 = System.currentTimeMillis();
Page page = seleniumDownloader.download(new Request("http://wenku.baidu.com/view/462933ff04a1b0717fd5ddc2.html"), new Task() {
@Override
public String getUUID() {
return "huaban.com";
}
@Override
public Site getSite() {
return Site.me();
}
});
System.out.println(page.getHtml().$("div.inner").replace("<[^<>]+>","").replace("&nsbp;","").all());
}
} }