diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/Request.java b/webmagic-core/src/main/java/us/codecraft/webmagic/Request.java index 3649d32..bb0d485 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/Request.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/Request.java @@ -29,6 +29,10 @@ public class Request implements Serializable { * Store additional information in extras. */ private Map extras; + /** + * POST/GET param set + * */ + private Map params=new HashMap(); /** * Priority of the request.
@@ -107,6 +111,21 @@ public class Request implements Serializable { this.method = method; } + public Map getParams() { + return params; + } + /** + * POST/GET参数设置 + * */ + public void setParams(Map params) { + this.params = params; + } + /** + * POST/GET参数设置 + * */ + public void putParams(String key,String value) { + params.put(key,value); + } @Override public String toString() { return "Request{" + diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java index ca35867..26a7288 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java @@ -7,10 +7,12 @@ import org.apache.http.NameValuePair; import org.apache.http.annotation.ThreadSafe; import org.apache.http.client.config.CookieSpecs; import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.methods.RequestBuilder; import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.message.BasicNameValuePair; import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,9 +28,7 @@ import us.codecraft.webmagic.utils.WMCollections; import java.io.IOException; import java.nio.charset.Charset; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; +import java.util.*; /** @@ -162,13 +162,26 @@ public class HttpClientDownloader extends AbstractDownloader { String method = request.getMethod(); if (method == null || method.equalsIgnoreCase(HttpConstant.Method.GET)) { //default get - return RequestBuilder.get(); + RequestBuilder requestBuilder=RequestBuilder.get(); + if (request.getParams() != null) { + for (Map.Entry entry : request.getParams().entrySet()) { + requestBuilder.addParameter(entry.getKey(), entry.getValue()); + } + } + return requestBuilder; } else if (method.equalsIgnoreCase(HttpConstant.Method.POST)) { RequestBuilder requestBuilder = RequestBuilder.post(); NameValuePair[] nameValuePair = (NameValuePair[]) request.getExtra("nameValuePair"); + List allNameValuePair=new ArrayList(); if (nameValuePair != null && nameValuePair.length > 0) { - requestBuilder.addParameters(nameValuePair); + allNameValuePair= Arrays.asList(nameValuePair); } + if (request.getParams() != null) { + for (String key : request.getParams().keySet()) { + allNameValuePair.add(new BasicNameValuePair(key, request.getParams().get(key))); + } + } + requestBuilder.setEntity(new UrlEncodedFormEntity(allNameValuePair, Charset.forName("utf8"))); return requestBuilder; } else if (method.equalsIgnoreCase(HttpConstant.Method.HEAD)) { return RequestBuilder.head();