complete test #447

webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java

@@ -162,34 +162,36 @@ public class HttpClientDownloader extends AbstractDownloader {
         String method = request.getMethod();
         if (method == null || method.equalsIgnoreCase(HttpConstant.Method.GET)) {
             //default get
-            return addParams(RequestBuilder.get(),request.getParams());
+            return addQueryParams(RequestBuilder.get(),request.getParams());
         } else if (method.equalsIgnoreCase(HttpConstant.Method.POST)) {
-            RequestBuilder requestBuilder = RequestBuilder.post();
+            return addFormParams(RequestBuilder.post(), (NameValuePair[]) request.getExtra("nameValuePair"), request.getParams());
-            NameValuePair[] nameValuePair = (NameValuePair[]) request.getExtra("nameValuePair");
-            List<NameValuePair> allNameValuePair=new ArrayList<NameValuePair>();
-            if (nameValuePair != null && nameValuePair.length > 0) {
-                allNameValuePair= Arrays.asList(nameValuePair);
-            }
-            if (request.getParams() != null) {
-                for (String key : request.getParams().keySet()) {
-                    allNameValuePair.add(new BasicNameValuePair(key, request.getParams().get(key)));
-                }
-            }
-            requestBuilder.setEntity(new UrlEncodedFormEntity(allNameValuePair, Charset.forName("utf8")));
-            return requestBuilder;
         } else if (method.equalsIgnoreCase(HttpConstant.Method.HEAD)) {
-            return addParams(RequestBuilder.head(),request.getParams());
+            return addQueryParams(RequestBuilder.head(),request.getParams());
         } else if (method.equalsIgnoreCase(HttpConstant.Method.PUT)) {
-            return addParams(RequestBuilder.put(),request.getParams());
+            return addFormParams(RequestBuilder.put(), (NameValuePair[]) request.getExtra("nameValuePair"), request.getParams());
         } else if (method.equalsIgnoreCase(HttpConstant.Method.DELETE)) {
-            return addParams(RequestBuilder.delete(),request.getParams());
+            return addQueryParams(RequestBuilder.delete(),request.getParams());
         } else if (method.equalsIgnoreCase(HttpConstant.Method.TRACE)) {
-            return addParams(RequestBuilder.trace(),request.getParams());
+            return addQueryParams(RequestBuilder.trace(),request.getParams());
         }
         throw new IllegalArgumentException("Illegal HTTP Method " + method);
     }
 
-    private RequestBuilder addParams(RequestBuilder requestBuilder, Map<String, String> params) {
+    private RequestBuilder addFormParams(RequestBuilder requestBuilder, NameValuePair[] nameValuePair, Map<String, String> params) {
+        List<NameValuePair> allNameValuePair=new ArrayList<NameValuePair>();
+        if (nameValuePair != null && nameValuePair.length > 0) {
+            allNameValuePair= Arrays.asList(nameValuePair);
+        }
+        if (params != null) {
+            for (String key : params.keySet()) {
+                allNameValuePair.add(new BasicNameValuePair(key, params.get(key)));
+            }
+        }
+        requestBuilder.setEntity(new UrlEncodedFormEntity(allNameValuePair, Charset.forName("utf8")));
+        return requestBuilder;
+    }
+
+    private RequestBuilder addQueryParams(RequestBuilder requestBuilder, Map<String, String> params) {
         if (params != null) {
             for (Map.Entry<String, String> entry : params.entrySet()) {
                 requestBuilder.addParameter(entry.getKey(), entry.getValue());

webmagic-core/src/test/java/us/codecraft/webmagic/downloader/HttpClientDownloaderTest.java

@@ -5,13 +5,17 @@ import com.github.dreamhead.moco.Runnable;
 import com.github.dreamhead.moco.Runner;
 import org.apache.commons.io.IOUtils;
 import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.RequestBuilder;
 import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.util.EntityUtils;
 import org.junit.Test;
 import us.codecraft.webmagic.Page;
 import us.codecraft.webmagic.Request;
 import us.codecraft.webmagic.Site;
 import us.codecraft.webmagic.Task;
 import us.codecraft.webmagic.selector.Html;
+import us.codecraft.webmagic.utils.HttpConstant;
 
 import java.io.IOException;
 import java.io.UnsupportedEncodingException;
@@ -103,4 +107,42 @@ public class HttpClientDownloaderTest {
             }
         });
     }
+
+    @Test
+    public void test_selectRequestMethod() throws Exception {
+        HttpServer server = httpserver(12306);
+        server.get(eq(query("q"), "webmagic")).response("get");
+        server.post(eq(form("q"), "webmagic")).response("post");
+        server.put(eq(form("q"), "webmagic")).response("put");
+        server.delete(eq(query("q"), "webmagic")).response("delete");
+        server.request(and(by(method("HEAD")),eq(query("q"), "webmagic"))).response(header("method","head"));
+        server.request(and(by(method("TRACE")),eq(query("q"), "webmagic"))).response("trace");
+        Runner.running(server, new Runnable() {
+            @Override
+            public void run() throws Exception {
+                HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
+                Request request = new Request();
+                request.setUrl("http://127.0.0.1:12306/search");
+                request.putParams("q", "webmagic");
+                request.setMethod(HttpConstant.Method.GET);
+                RequestBuilder requestBuilder = httpClientDownloader.selectRequestMethod(request).setUri(request.getUrl());
+                assertThat(EntityUtils.toString(HttpClients.custom().build().execute(requestBuilder.build()).getEntity())).isEqualTo("get");
+                request.setMethod(HttpConstant.Method.POST);
+                requestBuilder = httpClientDownloader.selectRequestMethod(request).setUri(request.getUrl());
+                assertThat(EntityUtils.toString(HttpClients.custom().build().execute(requestBuilder.build()).getEntity())).isEqualTo("post");
+                request.setMethod(HttpConstant.Method.PUT);
+                requestBuilder = httpClientDownloader.selectRequestMethod(request).setUri(request.getUrl());
+                assertThat(EntityUtils.toString(HttpClients.custom().build().execute(requestBuilder.build()).getEntity())).isEqualTo("put");
+                request.setMethod(HttpConstant.Method.DELETE);
+                requestBuilder = httpClientDownloader.selectRequestMethod(request).setUri(request.getUrl());
+                assertThat(EntityUtils.toString(HttpClients.custom().build().execute(requestBuilder.build()).getEntity())).isEqualTo("delete");
+                request.setMethod(HttpConstant.Method.HEAD);
+                requestBuilder = httpClientDownloader.selectRequestMethod(request).setUri(request.getUrl());
+                assertThat(HttpClients.custom().build().execute(requestBuilder.build()).getFirstHeader("method").getValue()).isEqualTo("head");
+                request.setMethod(HttpConstant.Method.TRACE);
+                requestBuilder = httpClientDownloader.selectRequestMethod(request).setUri(request.getUrl());
+                assertThat(EntityUtils.toString(HttpClients.custom().build().execute(requestBuilder.build()).getEntity())).isEqualTo("trace");
+            }
+        });
+    }
 }