Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

some fix for tests #130

master
yihua.huang 2014-06-10 00:05:30 +08:00
parent b75e64a61b
commit b3a282e58d
4 changed files with 273 additions and 288 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
webmagic-core/src/main/java/us/codecraft/webmagic/Site.java
View File

@ -49,7 +49,7 @@ public class Site {
private HttpHost httpProxy; private HttpHost httpProxy;
private ProxyPool httpProxyPool=new ProxyPool(); private ProxyPool httpProxyPool;
private boolean useGzip = true; private boolean useGzip = true;
@ -453,6 +453,11 @@ public class Site {
return this; return this;
} }
public Site enableHttpProxyPool() {
this.httpProxyPool=new ProxyPool();
return this;
}
public ProxyPool getHttpProxyPool() { public ProxyPool getHttpProxyPool() {
return httpProxyPool; return httpProxyPool;
} }

2
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -141,7 +141,7 @@ public class HttpClientDownloader extends AbstractDownloader {
.setSocketTimeout(site.getTimeOut()) .setSocketTimeout(site.getTimeOut())
.setConnectTimeout(site.getTimeOut()) .setConnectTimeout(site.getTimeOut())
.setCookieSpec(CookieSpecs.BEST_MATCH); .setCookieSpec(CookieSpecs.BEST_MATCH);
if (site.getHttpProxyPool().isEnable()) { if (site.getHttpProxyPool() != null && site.getHttpProxyPool().isEnable()) {
HttpHost host = site.getHttpProxyFromPool(); HttpHost host = site.getHttpProxyFromPool();
requestConfigBuilder.setProxy(host); requestConfigBuilder.setProxy(host);
request.putExtra(Request.PROXY, host); request.putExtra(Request.PROXY, host);

35
webmagic-core/src/main/java/us/codecraft/webmagic/proxy/ProxyPool.java
View File

@ -1,39 +1,26 @@
package us.codecraft.webmagic.proxy; package us.codecraft.webmagic.proxy;
import java.io.File; import org.apache.http.HttpHost;
import java.io.FileInputStream; import org.slf4j.Logger;
import java.io.FileNotFoundException; import org.slf4j.LoggerFactory;
import java.io.FileOutputStream; import us.codecraft.webmagic.utils.FilePersistentBase;
import java.io.IOException; import us.codecraft.webmagic.utils.ProxyUtils;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream; import java.io.*;
import java.net.InetAddress; import java.net.InetAddress;
import java.net.UnknownHostException; import java.net.UnknownHostException;
import java.util.ArrayList; import java.util.*;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Timer;
import java.util.TimerTask;
import java.util.Map.Entry; import java.util.Map.Entry;
import java.util.concurrent.BlockingQueue; import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.DelayQueue; import java.util.concurrent.DelayQueue;
import org.apache.http.HttpHost;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.utils.FilePersistentBase;
import us.codecraft.webmagic.utils.ProxyUtils;
/** /**
* Pooled Proxy Object * Pooled Proxy Object
* *
* @author yxssfxwzy@sina.com <br> * @author yxssfxwzy@sina.com <br>
* @since 0.5.1
* @see Proxy * @see Proxy
* @since 0.5.1
*/ */
public class ProxyPool { public class ProxyPool {
@ -87,9 +74,9 @@ public class ProxyPool {
private void setFilePath() { private void setFilePath() {
String tmpDir = System.getProperty("java.io.tmpdir"); String tmpDir = System.getProperty("java.io.tmpdir");
String path = tmpDir + "webmagic\\lastUse.proxy"; String path = tmpDir + FilePersistentBase.PATH_SEPERATOR + "webmagic" + FilePersistentBase.PATH_SEPERATOR + "lastUse.proxy";
if (tmpDir != null && new File(tmpDir).isDirectory()) { if (tmpDir != null && new File(tmpDir).isDirectory()) {
fBase.setPath(tmpDir + "webmagic"); fBase.setPath(tmpDir + FilePersistentBase.PATH_SEPERATOR + "webmagic");
File f = fBase.getFile(path); File f = fBase.getFile(path);
if (!f.exists()) { if (!f.exists()) {
try { try {

17
webmagic-core/src/test/java/us/codecraft/webmagic/proxy/ProxyTest.java
View File

@ -1,16 +1,14 @@
package us.codecraft.webmagic.proxy; package us.codecraft.webmagic.proxy;
import static org.assertj.core.api.Assertions.assertThat;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import org.apache.http.HttpHost; import org.apache.http.HttpHost;
import org.junit.BeforeClass; import org.junit.BeforeClass;
import org.junit.Test; import org.junit.Test;
import us.codecraft.webmagic.Request; import java.io.File;
import java.util.ArrayList;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
/** /**
* @author yxssfxwzy@sina.com May 30, 2014 * @author yxssfxwzy@sina.com May 30, 2014
@ -30,11 +28,6 @@ public class ProxyTest {
} }
} }
@Test
public void testAddProxy() {
}
@Test @Test
public void testProxy() { public void testProxy() {
ProxyPool proxyPool = new ProxyPool(httpProxyList); ProxyPool proxyPool = new ProxyPool(httpProxyList);