Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

encode illegal charactors in url #80

master
yihua.huang 2014-04-01 22:14:30 +08:00
parent 2780423e60
commit 01848301d4
3 changed files with 29 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
webmagic-core/src/main/java/us/codecraft/webmagic/utils/UrlUtils.java
View File

@ -43,12 +43,22 @@ public class UrlUtils {
if (url.startsWith("?"))
url = base.getPath() + url;
URL abs = new URL(base, url);
return abs.toExternalForm();
return encodeIllegalCharacterInUrl(abs.toExternalForm());
} catch (MalformedURLException e) {
return "";
}
}
/**
*
* @param url
* @return
*/
public static String encodeIllegalCharacterInUrl(String url) {
//TODO more charator support
return url.replace(" ", "%20");
}
public static String getHost(String url) {
String host = url;
int i = StringUtils.ordinalIndexOf(url, "/", 3);
@ -101,9 +111,9 @@ public class UrlUtils {
stringBuilder.append("\"").append(canonicalizeUrl(matcher.group(2), url)).append("\"");
lastEnd = matcher.end();
}
if (!modified) {
return html;
}
if (!modified) {
return html;
}
stringBuilder.append(StringUtils.substring(html, lastEnd));
return stringBuilder.toString();
}

8
webmagic-core/src/test/java/us/codecraft/webmagic/downloader/HttpClientDownloaderTest.java
View File

@ -8,6 +8,8 @@ import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.selector.Html;
import java.io.UnsupportedEncodingException;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertTrue;
@ -32,6 +34,12 @@ public class HttpClientDownloaderTest {
assertTrue(!html.getText().isEmpty());
}
@Test(expected = IllegalArgumentException.class)
public void testDownloaderInIllegalUrl() throws UnsupportedEncodingException {
HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
httpClientDownloader.download("http://www.oschina.net/>");
}
@Test
public void testCycleTriedTimes() {
HttpClientDownloader httpClientDownloader = new HttpClientDownloader();

9
webmagic-core/src/test/java/us/codecraft/webmagic/utils/UrlUtilsTest.java
View File

@ -38,17 +38,22 @@ public class UrlUtilsTest {
originHtml = "<a href=\"/start a\">";
replacedHtml = UrlUtils.fixAllRelativeHrefs(originHtml, "http://www.dianping.com/");
assertThat(replacedHtml).isEqualTo("<a href=\"http://www.dianping.com/start a\">");
assertThat(replacedHtml).isEqualTo("<a href=\"http://www.dianping.com/start%20a\">");
originHtml = "<a href='/start a'>";
replacedHtml = UrlUtils.fixAllRelativeHrefs(originHtml, "http://www.dianping.com/");
assertThat(replacedHtml).isEqualTo("<a href=\"http://www.dianping.com/start a\">");
assertThat(replacedHtml).isEqualTo("<a href=\"http://www.dianping.com/start%20a\">");
originHtml = "<a href=/start tag>";
replacedHtml = UrlUtils.fixAllRelativeHrefs(originHtml, "http://www.dianping.com/");
assertThat(replacedHtml).isEqualTo("<a href=\"http://www.dianping.com/start\" tag>");
}
@Test
public void test(){
UrlUtils.canonicalizeUrl("start tag", "http://www.dianping.com/");
}
@Test
public void testGetDomain(){
String url = "http://www.dianping.com/aa/";