diff --git a/webmagic-extension/src/test/java/us/codecraft/webmagic/model/PageMapperTest.java b/webmagic-extension/src/test/java/us/codecraft/webmagic/model/PageMapperTest.java new file mode 100644 index 0000000..2e6f67b --- /dev/null +++ b/webmagic-extension/src/test/java/us/codecraft/webmagic/model/PageMapperTest.java @@ -0,0 +1,46 @@ +package us.codecraft.webmagic.model; + +import org.apache.commons.io.IOUtils; +import org.junit.Test; +import us.codecraft.webmagic.Page; +import us.codecraft.webmagic.Request; +import us.codecraft.webmagic.model.annotation.ExtractBy; +import us.codecraft.webmagic.selector.PlainText; + +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author code4crafter@gmail.com + * Date: 2017/6/3 + * Time: 下午3:23 + */ +public class PageMapperTest { + + public static class GithubRepo { + + @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$.name") + private String name; + + public String getName() { + return name; + } + } + + @Test + public void test_get() throws Exception { + PageMapper pageMapper = new PageMapper(GithubRepo.class); + GithubRepo githubRepo = pageMapper.get(getMockJsonPage()); + assertThat(githubRepo.getName()).isEqualTo("webmagic"); + } + + private Page getMockJsonPage() throws IOException { + Page page = new Page(); + page.setRawText(IOUtils.toString(getClass().getClassLoader().getResourceAsStream("json/mock-githubrepo.json"))); + page.setRequest(new Request("https://api.github.com/repos/code4craft/webmagic")); + page.setUrl(new PlainText("https://api.github.com/repos/code4craft/webmagic")); + return page; + } + +} diff --git a/webmagic-extension/src/test/resources/json/mock-githubrepo.json b/webmagic-extension/src/test/resources/json/mock-githubrepo.json new file mode 100644 index 0000000..a5037d3 --- /dev/null +++ b/webmagic-extension/src/test/resources/json/mock-githubrepo.json @@ -0,0 +1,91 @@ +{ + "id": 9623064, + "name": "webmagic", + "full_name": "code4craft/webmagic", + "owner": { + "login": "code4craft", + "id": 1351884, + "avatar_url": "https://avatars0.githubusercontent.com/u/1351884?v=3", + "gravatar_id": "", + "url": "https://api.github.com/users/code4craft", + "html_url": "https://github.com/code4craft", + "followers_url": "https://api.github.com/users/code4craft/followers", + "following_url": "https://api.github.com/users/code4craft/following{/other_user}", + "gists_url": "https://api.github.com/users/code4craft/gists{/gist_id}", + "starred_url": "https://api.github.com/users/code4craft/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/code4craft/subscriptions", + "organizations_url": "https://api.github.com/users/code4craft/orgs", + "repos_url": "https://api.github.com/users/code4craft/repos", + "events_url": "https://api.github.com/users/code4craft/events{/privacy}", + "received_events_url": "https://api.github.com/users/code4craft/received_events", + "type": "User", + "site_admin": false + }, + "private": false, + "html_url": "https://github.com/code4craft/webmagic", + "description": "A scalable web crawler framework for Java.", + "fork": false, + "url": "https://api.github.com/repos/code4craft/webmagic", + "forks_url": "https://api.github.com/repos/code4craft/webmagic/forks", + "keys_url": "https://api.github.com/repos/code4craft/webmagic/keys{/key_id}", + "collaborators_url": "https://api.github.com/repos/code4craft/webmagic/collaborators{/collaborator}", + "teams_url": "https://api.github.com/repos/code4craft/webmagic/teams", + "hooks_url": "https://api.github.com/repos/code4craft/webmagic/hooks", + "issue_events_url": "https://api.github.com/repos/code4craft/webmagic/issues/events{/number}", + "events_url": "https://api.github.com/repos/code4craft/webmagic/events", + "assignees_url": "https://api.github.com/repos/code4craft/webmagic/assignees{/user}", + "branches_url": "https://api.github.com/repos/code4craft/webmagic/branches{/branch}", + "tags_url": "https://api.github.com/repos/code4craft/webmagic/tags", + "blobs_url": "https://api.github.com/repos/code4craft/webmagic/git/blobs{/sha}", + "git_tags_url": "https://api.github.com/repos/code4craft/webmagic/git/tags{/sha}", + "git_refs_url": "https://api.github.com/repos/code4craft/webmagic/git/refs{/sha}", + "trees_url": "https://api.github.com/repos/code4craft/webmagic/git/trees{/sha}", + "statuses_url": "https://api.github.com/repos/code4craft/webmagic/statuses/{sha}", + "languages_url": "https://api.github.com/repos/code4craft/webmagic/languages", + "stargazers_url": "https://api.github.com/repos/code4craft/webmagic/stargazers", + "contributors_url": "https://api.github.com/repos/code4craft/webmagic/contributors", + "subscribers_url": "https://api.github.com/repos/code4craft/webmagic/subscribers", + "subscription_url": "https://api.github.com/repos/code4craft/webmagic/subscription", + "commits_url": "https://api.github.com/repos/code4craft/webmagic/commits{/sha}", + "git_commits_url": "https://api.github.com/repos/code4craft/webmagic/git/commits{/sha}", + "comments_url": "https://api.github.com/repos/code4craft/webmagic/comments{/number}", + "issue_comment_url": "https://api.github.com/repos/code4craft/webmagic/issues/comments{/number}", + "contents_url": "https://api.github.com/repos/code4craft/webmagic/contents/{+path}", + "compare_url": "https://api.github.com/repos/code4craft/webmagic/compare/{base}...{head}", + "merges_url": "https://api.github.com/repos/code4craft/webmagic/merges", + "archive_url": "https://api.github.com/repos/code4craft/webmagic/{archive_format}{/ref}", + "downloads_url": "https://api.github.com/repos/code4craft/webmagic/downloads", + "issues_url": "https://api.github.com/repos/code4craft/webmagic/issues{/number}", + "pulls_url": "https://api.github.com/repos/code4craft/webmagic/pulls{/number}", + "milestones_url": "https://api.github.com/repos/code4craft/webmagic/milestones{/number}", + "notifications_url": "https://api.github.com/repos/code4craft/webmagic/notifications{?since,all,participating}", + "labels_url": "https://api.github.com/repos/code4craft/webmagic/labels{/name}", + "releases_url": "https://api.github.com/repos/code4craft/webmagic/releases{/id}", + "deployments_url": "https://api.github.com/repos/code4craft/webmagic/deployments", + "created_at": "2013-04-23T12:57:36Z", + "updated_at": "2017-06-03T03:58:13Z", + "pushed_at": "2017-06-03T07:10:15Z", + "git_url": "git://github.com/code4craft/webmagic.git", + "ssh_url": "git@github.com:code4craft/webmagic.git", + "clone_url": "https://github.com/code4craft/webmagic.git", + "svn_url": "https://github.com/code4craft/webmagic", + "homepage": "http://webmagic.io/", + "size": 16982, + "stargazers_count": 4566, + "watchers_count": 4566, + "language": "Java", + "has_issues": true, + "has_projects": true, + "has_downloads": true, + "has_wiki": true, + "has_pages": true, + "forks_count": 2432, + "mirror_url": null, + "open_issues_count": 96, + "forks": 2432, + "open_issues": 96, + "watchers": 4566, + "default_branch": "master", + "network_count": 2432, + "subscribers_count": 618 +}