diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/Request.java b/webmagic-core/src/main/java/us/codecraft/webmagic/Request.java index 1978792..c8c5978 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/Request.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/Request.java @@ -115,14 +115,18 @@ public class Request implements Serializable { return params; } /** - * POST/GET参数设置 + * set params for request + *
+ * DO NOT set this for request already has params, like 'https://github.com/search?q=webmagic' * @param params params * */ public void setParams(Map params) { this.params = params; } /** - * POST/GET参数设置 + * set params for request + *
+ * DO NOT set this for request already has params, like 'https://github.com/search?q=webmagic' * @param key key * @param value value * */ diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java index 26a7288..0c11149 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java @@ -162,13 +162,7 @@ public class HttpClientDownloader extends AbstractDownloader { String method = request.getMethod(); if (method == null || method.equalsIgnoreCase(HttpConstant.Method.GET)) { //default get - RequestBuilder requestBuilder=RequestBuilder.get(); - if (request.getParams() != null) { - for (Map.Entry entry : request.getParams().entrySet()) { - requestBuilder.addParameter(entry.getKey(), entry.getValue()); - } - } - return requestBuilder; + return addParams(RequestBuilder.get(),request.getParams()); } else if (method.equalsIgnoreCase(HttpConstant.Method.POST)) { RequestBuilder requestBuilder = RequestBuilder.post(); NameValuePair[] nameValuePair = (NameValuePair[]) request.getExtra("nameValuePair"); @@ -184,17 +178,26 @@ public class HttpClientDownloader extends AbstractDownloader { requestBuilder.setEntity(new UrlEncodedFormEntity(allNameValuePair, Charset.forName("utf8"))); return requestBuilder; } else if (method.equalsIgnoreCase(HttpConstant.Method.HEAD)) { - return RequestBuilder.head(); + return addParams(RequestBuilder.head(),request.getParams()); } else if (method.equalsIgnoreCase(HttpConstant.Method.PUT)) { - return RequestBuilder.put(); + return addParams(RequestBuilder.put(),request.getParams()); } else if (method.equalsIgnoreCase(HttpConstant.Method.DELETE)) { - return RequestBuilder.delete(); + return addParams(RequestBuilder.delete(),request.getParams()); } else if (method.equalsIgnoreCase(HttpConstant.Method.TRACE)) { - return RequestBuilder.trace(); + return addParams(RequestBuilder.trace(),request.getParams()); } throw new IllegalArgumentException("Illegal HTTP Method " + method); } + private RequestBuilder addParams(RequestBuilder requestBuilder, Map params) { + if (params != null) { + for (Map.Entry entry : params.entrySet()) { + requestBuilder.addParameter(entry.getKey(), entry.getValue()); + } + } + return requestBuilder; + } + protected Page handleResponse(Request request, String charset, HttpResponse httpResponse, Task task) throws IOException { String content = getContent(charset, httpResponse); Page page = new Page();