Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:code4craft/webmagic

master
yihua.huang 2014-02-11 22:23:56 +08:00
parent af809c4d55 7e8a5c7dd2
commit 362fdd0662
4 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -1,4 +1,5 @@
target/* target
*.iml *.iml
out/ out/
.idea .idea

4
README.md
View File

@ -127,3 +127,7 @@ To write webmagic, I refered to the projects below :
### Mail-list: ### Mail-list:
[https://groups.google.com/forum/#!forum/webmagic-java](https://groups.google.com/forum/#!forum/webmagic-java) [https://groups.google.com/forum/#!forum/webmagic-java](https://groups.google.com/forum/#!forum/webmagic-java)
[![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/code4craft/webmagic/trend.png)](https://bitdeli.com/free "Bitdeli Badge")

2
webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/FileCacheQueueScheduler.java
View File

@ -126,7 +126,7 @@ public class FileCacheQueueScheduler implements Scheduler {
private void readCursorFile() throws IOException { private void readCursorFile() throws IOException {
BufferedReader fileCursorReader = null; BufferedReader fileCursorReader = null;
try { try {
new BufferedReader(new FileReader(getFileName(fileCursor))); fileCursorReader = new BufferedReader(new FileReader(getFileName(fileCursor)));
String line; String line;
//read the last number //read the last number
while ((line = fileCursorReader.readLine()) != null) { while ((line = fileCursorReader.readLine()) != null) {

1
webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java
View File

@ -84,6 +84,7 @@ public class SeleniumDownloader implements Downloader, Closeable {
WebElement webElement = webDriver.findElement(By.xpath("/html")); WebElement webElement = webDriver.findElement(By.xpath("/html"));
String content = webElement.getAttribute("outerHTML"); String content = webElement.getAttribute("outerHTML");
Page page = new Page(); Page page = new Page();
page.setRawText(content);
page.setHtml(new Html(UrlUtils.fixAllRelativeHrefs(content, request.getUrl()))); page.setHtml(new Html(UrlUtils.fixAllRelativeHrefs(content, request.getUrl())));
page.setUrl(new PlainText(request.getUrl())); page.setUrl(new PlainText(request.getUrl()));
page.setRequest(request); page.setRequest(request);