Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #528 from GZhY/master 

对 WebMagic-0.7.0 的小修补
master
Yihua Huang 2017-04-10 07:43:53 +08:00 committed by GitHub
parent 4a2c0f4f97 5f34adf938
commit 0cd2f6031a
5 changed files with 9 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
webmagic-core/src/main/java/us/codecraft/webmagic/selector/HtmlNode.java
View File

@ -90,7 +90,7 @@ public class HtmlNode extends AbstractSelectable {
* See: https://github.com/code4craft/webmagic/issues/113
*
* @param elementIterator elementIterator
* @param element element
* @return element element
*/
private Element checkElementAndConvert(ListIterator<Element> elementIterator) {
Element element = elementIterator.next();

35
webmagic-core/src/main/java/us/codecraft/webmagic/utils/UrlUtils.java
View File

@ -92,41 +92,6 @@ public class UrlUtils {
}
}
/**
* allow blank space in quote
*/
private static Pattern patternForHrefWithQuote = Pattern.compile("(<a[^<>]*href=)[\"']([^\"'<>]*)[\"']", Pattern.CASE_INSENSITIVE);
/**
* disallow blank space without quote
*/
private static Pattern patternForHrefWithoutQuote = Pattern.compile("(<a[^<>]*href=)([^\"'<>\\s]+)", Pattern.CASE_INSENSITIVE);
public static String fixAllRelativeHrefs(String html, String url) {
html = replaceByPattern(html, url, patternForHrefWithQuote);
html = replaceByPattern(html, url, patternForHrefWithoutQuote);
return html;
}
public static String replaceByPattern(String html, String url, Pattern pattern) {
StringBuilder stringBuilder = new StringBuilder();
Matcher matcher = pattern.matcher(html);
int lastEnd = 0;
boolean modified = false;
while (matcher.find()) {
modified = true;
stringBuilder.append(StringUtils.substring(html, lastEnd, matcher.start()));
stringBuilder.append(matcher.group(1));
stringBuilder.append("\"").append(canonicalizeUrl(matcher.group(2), url)).append("\"");
lastEnd = matcher.end();
}
if (!modified) {
return html;
}
stringBuilder.append(StringUtils.substring(html, lastEnd));
return stringBuilder.toString();
}
public static List<Request> convertToRequests(Collection<String> urls) {
List<Request> requestList = new ArrayList<Request>(urls.size());
for (String url : urls) {

8
webmagic-core/src/test/java/us/codecraft/webmagic/selector/LinksSelectorTest.java
View File

@ -1,5 +1,6 @@
package us.codecraft.webmagic.selector;
import org.jsoup.Jsoup;
import org.junit.Test;
import java.util.List;
@ -15,7 +16,12 @@ public class LinksSelectorTest {
@Test
public void testLinks() throws Exception {
List<String> links = new LinksSelector().selectList(html);
LinksSelector linksSelector = new LinksSelector();
List<String> links = linksSelector.selectList(html);
System.out.println(links);
html = "<div><a href='aaa'></a></div><div><a href='http://whatever.com/bbb'></a></div><div><a href='http://other.com/bbb'></a></div>";
links = linksSelector.selectList(Jsoup.parse(html, "http://whatever.com/"));
System.out.println(links);
}
}

19
webmagic-core/src/test/java/us/codecraft/webmagic/utils/UrlUtilsTest.java
View File

@ -33,25 +33,6 @@ public class UrlUtilsTest {
assertThat(absoluteUrl).isEqualTo("http://www.dianping.com/aa");
}
@Test
public void testFixAllRelativeHrefs() {
String originHtml = "<a href=\"/start\">";
String replacedHtml = UrlUtils.fixAllRelativeHrefs(originHtml, "http://www.dianping.com/");
assertThat(replacedHtml).isEqualTo("<a href=\"http://www.dianping.com/start\">");
originHtml = "<a href=\"/start a\">";
replacedHtml = UrlUtils.fixAllRelativeHrefs(originHtml, "http://www.dianping.com/");
assertThat(replacedHtml).isEqualTo("<a href=\"http://www.dianping.com/start%20a\">");
originHtml = "<a href='/start a'>";
replacedHtml = UrlUtils.fixAllRelativeHrefs(originHtml, "http://www.dianping.com/");
assertThat(replacedHtml).isEqualTo("<a href=\"http://www.dianping.com/start%20a\">");
originHtml = "<a href=/start tag>";
replacedHtml = UrlUtils.fixAllRelativeHrefs(originHtml, "http://www.dianping.com/");
assertThat(replacedHtml).isEqualTo("<a href=\"http://www.dianping.com/start\" tag>");
}
@Test
public void testGetDomain(){
String url = "http://www.dianping.com/aa/";

5
webmagic-selenium/src/main/java/us/codecraft/webmagic/downloader/selenium/SeleniumDownloader.java
View File

@ -5,7 +5,6 @@ import org.openqa.selenium.By;
import org.openqa.selenium.Cookie;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
@ -13,7 +12,6 @@ import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.downloader.Downloader;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.PlainText;
import us.codecraft.webmagic.utils.UrlUtils;
import java.io.Closeable;
import java.io.IOException;
@ -108,8 +106,7 @@ public class SeleniumDownloader implements Downloader, Closeable {
String content = webElement.getAttribute("outerHTML");
Page page = new Page();
page.setRawText(content);
page.setHtml(new Html(UrlUtils.fixAllRelativeHrefs(content,
request.getUrl())));
page.setHtml(new Html(content, request.getUrl()));
page.setUrl(new PlainText(request.getUrl()));
page.setRequest(request);
webDriverPool.returnToPool(webDriver);