Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add test for cycleTriedTimes and fix cycleTriedTimes inc error #60

master
yihua.huang 2014-03-01 15:10:38 +08:00
parent bbd0d7e600
commit 2768a1cae4
5 changed files with 31 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
pom.xml
View File

@ -106,6 +106,11 @@
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<version>1.5.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>

2
webmagic-avalon/src/main/java/us/codecraft/webmagic/avalon/web/DashBoardController.java
View File

@ -12,7 +12,7 @@ import org.springframework.web.servlet.ModelAndView;
public class DashBoardController {
@RequestMapping
public ModelAndView create() {
public ModelAndView index() {
ModelAndView map = new ModelAndView("dashboard");
return map;
}

5
webmagic-core/pom.xml
View File

@ -55,6 +55,11 @@
<artifactId>htmlcleaner</artifactId>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
</dependency>
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>

2
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -156,7 +156,7 @@ public class HttpClientDownloader implements Downloader {
if (cycleTriedTimes >= site.getCycleRetryTimes()) {
return null;
}
page.addTargetRequest(request.setPriority(0).putExtra(Request.CYCLE_TRIED_TIMES, 1));
page.addTargetRequest(request.setPriority(0).putExtra(Request.CYCLE_TRIED_TIMES, cycleTriedTimes));
}
return page;
}

25
webmagic-core/src/test/java/us/codecraft/webmagic/downloader/HttpClientDownloaderTest.java
View File

@ -1,17 +1,18 @@
package us.codecraft.webmagic.downloader;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.selector.Html;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertTrue;
/**
* Author: code4crafer@gmail.com
* Date: 13-6-18
* Time: 上午8:22
* @author code4crafer@gmail.com
*/
public class HttpClientDownloaderTest {
@ -21,14 +22,26 @@ public class HttpClientDownloaderTest {
Site site = Site.me().setDomain("www.diandian.com").addCookie("t", "43ztv9srfszl99yxv2aumx3zr7el7ybb");
HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
Page download = httpClientDownloader.download(new Request("http://www.diandian.com"), site.toTask());
Assert.assertTrue(download.getHtml().toString().contains("flashsword30"));
assertTrue(download.getHtml().toString().contains("flashsword30"));
}
@Test
public void testDownloader() {
HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
Html html = httpClientDownloader.download("http://www.oschina.net");
Assert.assertTrue(!html.getText().isEmpty());
assertTrue(!html.getText().isEmpty());
}
@Test
public void testCycleTriedTimes() {
HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
Task task = Site.me().setDomain("localhost").setCycleRetryTimes(5).toTask();
Request request = new Request("http://localhost/404");
Page page = httpClientDownloader.download(request, task);
assertThat(page.getTargetRequests().size() > 0);
assertThat((Integer) page.getTargetRequests().get(0).getExtra(Request.CYCLE_TRIED_TIMES)).isEqualTo(1);
page = httpClientDownloader.download(page.getTargetRequests().get(0), task);
assertThat((Integer) page.getTargetRequests().get(0).getExtra(Request.CYCLE_TRIED_TIMES)).isEqualTo(2);
}
}