diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/Request.java b/webmagic-core/src/main/java/us/codecraft/webmagic/Request.java
index 1978792..c8c5978 100644
--- a/webmagic-core/src/main/java/us/codecraft/webmagic/Request.java
+++ b/webmagic-core/src/main/java/us/codecraft/webmagic/Request.java
@@ -115,14 +115,18 @@ public class Request implements Serializable {
return params;
}
/**
- * POST/GET参数设置
+ * set params for request
+ *
+ * DO NOT set this for request already has params, like 'https://github.com/search?q=webmagic'
* @param params params
* */
public void setParams(Map params) {
this.params = params;
}
/**
- * POST/GET参数设置
+ * set params for request
+ *
+ * DO NOT set this for request already has params, like 'https://github.com/search?q=webmagic'
* @param key key
* @param value value
* */
diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
index 26a7288..0c11149 100644
--- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
+++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
@@ -162,13 +162,7 @@ public class HttpClientDownloader extends AbstractDownloader {
String method = request.getMethod();
if (method == null || method.equalsIgnoreCase(HttpConstant.Method.GET)) {
//default get
- RequestBuilder requestBuilder=RequestBuilder.get();
- if (request.getParams() != null) {
- for (Map.Entry entry : request.getParams().entrySet()) {
- requestBuilder.addParameter(entry.getKey(), entry.getValue());
- }
- }
- return requestBuilder;
+ return addParams(RequestBuilder.get(),request.getParams());
} else if (method.equalsIgnoreCase(HttpConstant.Method.POST)) {
RequestBuilder requestBuilder = RequestBuilder.post();
NameValuePair[] nameValuePair = (NameValuePair[]) request.getExtra("nameValuePair");
@@ -184,17 +178,26 @@ public class HttpClientDownloader extends AbstractDownloader {
requestBuilder.setEntity(new UrlEncodedFormEntity(allNameValuePair, Charset.forName("utf8")));
return requestBuilder;
} else if (method.equalsIgnoreCase(HttpConstant.Method.HEAD)) {
- return RequestBuilder.head();
+ return addParams(RequestBuilder.head(),request.getParams());
} else if (method.equalsIgnoreCase(HttpConstant.Method.PUT)) {
- return RequestBuilder.put();
+ return addParams(RequestBuilder.put(),request.getParams());
} else if (method.equalsIgnoreCase(HttpConstant.Method.DELETE)) {
- return RequestBuilder.delete();
+ return addParams(RequestBuilder.delete(),request.getParams());
} else if (method.equalsIgnoreCase(HttpConstant.Method.TRACE)) {
- return RequestBuilder.trace();
+ return addParams(RequestBuilder.trace(),request.getParams());
}
throw new IllegalArgumentException("Illegal HTTP Method " + method);
}
+ private RequestBuilder addParams(RequestBuilder requestBuilder, Map params) {
+ if (params != null) {
+ for (Map.Entry entry : params.entrySet()) {
+ requestBuilder.addParameter(entry.getKey(), entry.getValue());
+ }
+ }
+ return requestBuilder;
+ }
+
protected Page handleResponse(Request request, String charset, HttpResponse httpResponse, Task task) throws IOException {
String content = getContent(charset, httpResponse);
Page page = new Page();