From e9341d0291101f24af259f62a15d26427a2df853 Mon Sep 17 00:00:00 2001 From: "yihua.huang" Date: Fri, 17 Mar 2017 07:54:28 +0800 Subject: [PATCH] complete test #447 --- .../downloader/HttpClientDownloader.java | 40 +++++++++--------- .../downloader/HttpClientDownloaderTest.java | 42 +++++++++++++++++++ 2 files changed, 63 insertions(+), 19 deletions(-) diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java index 0c11149..9e77ef5 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java @@ -162,34 +162,36 @@ public class HttpClientDownloader extends AbstractDownloader { String method = request.getMethod(); if (method == null || method.equalsIgnoreCase(HttpConstant.Method.GET)) { //default get - return addParams(RequestBuilder.get(),request.getParams()); + return addQueryParams(RequestBuilder.get(),request.getParams()); } else if (method.equalsIgnoreCase(HttpConstant.Method.POST)) { - RequestBuilder requestBuilder = RequestBuilder.post(); - NameValuePair[] nameValuePair = (NameValuePair[]) request.getExtra("nameValuePair"); - List allNameValuePair=new ArrayList(); - if (nameValuePair != null && nameValuePair.length > 0) { - allNameValuePair= Arrays.asList(nameValuePair); - } - if (request.getParams() != null) { - for (String key : request.getParams().keySet()) { - allNameValuePair.add(new BasicNameValuePair(key, request.getParams().get(key))); - } - } - requestBuilder.setEntity(new UrlEncodedFormEntity(allNameValuePair, Charset.forName("utf8"))); - return requestBuilder; + return addFormParams(RequestBuilder.post(), (NameValuePair[]) request.getExtra("nameValuePair"), request.getParams()); } else if (method.equalsIgnoreCase(HttpConstant.Method.HEAD)) { - return addParams(RequestBuilder.head(),request.getParams()); + return addQueryParams(RequestBuilder.head(),request.getParams()); } else if (method.equalsIgnoreCase(HttpConstant.Method.PUT)) { - return addParams(RequestBuilder.put(),request.getParams()); + return addFormParams(RequestBuilder.put(), (NameValuePair[]) request.getExtra("nameValuePair"), request.getParams()); } else if (method.equalsIgnoreCase(HttpConstant.Method.DELETE)) { - return addParams(RequestBuilder.delete(),request.getParams()); + return addQueryParams(RequestBuilder.delete(),request.getParams()); } else if (method.equalsIgnoreCase(HttpConstant.Method.TRACE)) { - return addParams(RequestBuilder.trace(),request.getParams()); + return addQueryParams(RequestBuilder.trace(),request.getParams()); } throw new IllegalArgumentException("Illegal HTTP Method " + method); } - private RequestBuilder addParams(RequestBuilder requestBuilder, Map params) { + private RequestBuilder addFormParams(RequestBuilder requestBuilder, NameValuePair[] nameValuePair, Map params) { + List allNameValuePair=new ArrayList(); + if (nameValuePair != null && nameValuePair.length > 0) { + allNameValuePair= Arrays.asList(nameValuePair); + } + if (params != null) { + for (String key : params.keySet()) { + allNameValuePair.add(new BasicNameValuePair(key, params.get(key))); + } + } + requestBuilder.setEntity(new UrlEncodedFormEntity(allNameValuePair, Charset.forName("utf8"))); + return requestBuilder; + } + + private RequestBuilder addQueryParams(RequestBuilder requestBuilder, Map params) { if (params != null) { for (Map.Entry entry : params.entrySet()) { requestBuilder.addParameter(entry.getKey(), entry.getValue()); diff --git a/webmagic-core/src/test/java/us/codecraft/webmagic/downloader/HttpClientDownloaderTest.java b/webmagic-core/src/test/java/us/codecraft/webmagic/downloader/HttpClientDownloaderTest.java index 1735e00..0e442a8 100644 --- a/webmagic-core/src/test/java/us/codecraft/webmagic/downloader/HttpClientDownloaderTest.java +++ b/webmagic-core/src/test/java/us/codecraft/webmagic/downloader/HttpClientDownloaderTest.java @@ -5,13 +5,17 @@ import com.github.dreamhead.moco.Runnable; import com.github.dreamhead.moco.Runner; import org.apache.commons.io.IOUtils; import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.RequestBuilder; import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; import org.junit.Test; import us.codecraft.webmagic.Page; import us.codecraft.webmagic.Request; import us.codecraft.webmagic.Site; import us.codecraft.webmagic.Task; import us.codecraft.webmagic.selector.Html; +import us.codecraft.webmagic.utils.HttpConstant; import java.io.IOException; import java.io.UnsupportedEncodingException; @@ -103,4 +107,42 @@ public class HttpClientDownloaderTest { } }); } + + @Test + public void test_selectRequestMethod() throws Exception { + HttpServer server = httpserver(12306); + server.get(eq(query("q"), "webmagic")).response("get"); + server.post(eq(form("q"), "webmagic")).response("post"); + server.put(eq(form("q"), "webmagic")).response("put"); + server.delete(eq(query("q"), "webmagic")).response("delete"); + server.request(and(by(method("HEAD")),eq(query("q"), "webmagic"))).response(header("method","head")); + server.request(and(by(method("TRACE")),eq(query("q"), "webmagic"))).response("trace"); + Runner.running(server, new Runnable() { + @Override + public void run() throws Exception { + HttpClientDownloader httpClientDownloader = new HttpClientDownloader(); + Request request = new Request(); + request.setUrl("http://127.0.0.1:12306/search"); + request.putParams("q", "webmagic"); + request.setMethod(HttpConstant.Method.GET); + RequestBuilder requestBuilder = httpClientDownloader.selectRequestMethod(request).setUri(request.getUrl()); + assertThat(EntityUtils.toString(HttpClients.custom().build().execute(requestBuilder.build()).getEntity())).isEqualTo("get"); + request.setMethod(HttpConstant.Method.POST); + requestBuilder = httpClientDownloader.selectRequestMethod(request).setUri(request.getUrl()); + assertThat(EntityUtils.toString(HttpClients.custom().build().execute(requestBuilder.build()).getEntity())).isEqualTo("post"); + request.setMethod(HttpConstant.Method.PUT); + requestBuilder = httpClientDownloader.selectRequestMethod(request).setUri(request.getUrl()); + assertThat(EntityUtils.toString(HttpClients.custom().build().execute(requestBuilder.build()).getEntity())).isEqualTo("put"); + request.setMethod(HttpConstant.Method.DELETE); + requestBuilder = httpClientDownloader.selectRequestMethod(request).setUri(request.getUrl()); + assertThat(EntityUtils.toString(HttpClients.custom().build().execute(requestBuilder.build()).getEntity())).isEqualTo("delete"); + request.setMethod(HttpConstant.Method.HEAD); + requestBuilder = httpClientDownloader.selectRequestMethod(request).setUri(request.getUrl()); + assertThat(HttpClients.custom().build().execute(requestBuilder.build()).getFirstHeader("method").getValue()).isEqualTo("head"); + request.setMethod(HttpConstant.Method.TRACE); + requestBuilder = httpClientDownloader.selectRequestMethod(request).setUri(request.getUrl()); + assertThat(EntityUtils.toString(HttpClients.custom().build().execute(requestBuilder.build()).getEntity())).isEqualTo("trace"); + } + }); + } }