Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

New method Proxy#create.

master
Sutra Zhou 2020-06-24 13:43:16 +08:00
parent 6d3f2d9b64
commit 48bc73fbff
2 changed files with 88 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
webmagic-core/src/main/java/us/codecraft/webmagic/proxy/Proxy.java
View File

@ -20,6 +20,21 @@ public class Proxy {
private String password; private String password;
public static Proxy create(final URI uri) {
Proxy proxy = new Proxy(uri.getHost(), uri.getPort(), uri.getScheme());
String userInfo = uri.getUserInfo();
if (userInfo != null) {
String[] up = userInfo.split(":");
if (up.length == 1) {
proxy.username = up[0].isEmpty() ? null : up[0];
} else {
proxy.username = up[0].isEmpty() ? null : up[0];
proxy.password = up[1].isEmpty() ? null : up[1];
}
}
return proxy;
}
public Proxy(String host, int port) { public Proxy(String host, int port) {
this(host, port, null); this(host, port, null);
} }

106
webmagic-core/src/test/java/us/codecraft/webmagic/proxy/ProxyTest.java
View File

@ -1,7 +1,9 @@
package us.codecraft.webmagic.proxy; package us.codecraft.webmagic.proxy;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import java.net.URI;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
@ -15,43 +17,81 @@ import org.junit.Test;
*/ */
public class ProxyTest { public class ProxyTest {
private static List<String[]> httpProxyList = new ArrayList<String[]>(); private static List<String[]> httpProxyList = new ArrayList<String[]>();
@BeforeClass @BeforeClass
public static void before() { public static void before() {
// String[] source = { "0.0.0.1:0", "0.0.0.2:0", "0.0.0.3:0", // String[] source = { "0.0.0.1:0", "0.0.0.2:0", "0.0.0.3:0",
// "0.0.0.4:0" }; // "0.0.0.4:0" };
String[] source = { "::0.0.0.1:0", "::0.0.0.2:0", "::0.0.0.3:0", "::0.0.0.4:0" }; String[] source = { "::0.0.0.1:0", "::0.0.0.2:0", "::0.0.0.3:0", "::0.0.0.4:0" };
for (String line : source) { for (String line : source) {
httpProxyList.add(new String[] {line.split(":")[0], line.split(":")[1], line.split(":")[2], line.split(":")[3] }); httpProxyList.add(new String[] {line.split(":")[0], line.split(":")[1], line.split(":")[2], line.split(":")[3] });
} }
} }
class Fetch extends Thread { class Fetch extends Thread {
HttpHost hp; HttpHost hp;
public Fetch(HttpHost hp) { public Fetch(HttpHost hp) {
this.hp = hp; this.hp = hp;
} }
@Override @Override
public void run() { public void run() {
try { try {
System.out.println("fetch web page use proxy: " + hp.getHostName() + ":" + hp.getPort()); System.out.println("fetch web page use proxy: " + hp.getHostName() + ":" + hp.getPort());
sleep(500); sleep(500);
} catch (InterruptedException e) { } catch (InterruptedException e) {
e.printStackTrace(); e.printStackTrace();
} }
} }
} }
@Test @Test
public void testToString() { public void testCreate() {
assertEquals("//127.0.0.1:8080", new Proxy("127.0.0.1", 8080).toString()); Proxy proxy = Proxy.create(URI.create("//127.0.0.1:8080"));
assertEquals("http://127.0.0.1:8080", new Proxy("127.0.0.1", 8080, "http").toString()); assertNull(proxy.getScheme());
assertEquals("//username:password@127.0.0.1:8080", new Proxy("127.0.0.1", 8080, "username", "password").toString()); assertNull(proxy.getUsername());
assertEquals("//username@127.0.0.1:8080", new Proxy("127.0.0.1", 8080, "username", null).toString()); assertNull(proxy.getPassword());
assertEquals("//:password@127.0.0.1:8080", new Proxy("127.0.0.1", 8080, null, "password").toString()); assertEquals("127.0.0.1", proxy.getHost());
} assertEquals(8080, proxy.getPort());
proxy = Proxy.create(URI.create("http://127.0.0.1:8080"));
assertEquals("http", proxy.getScheme());
assertNull(proxy.getUsername());
assertNull(proxy.getPassword());
assertEquals("127.0.0.1", proxy.getHost());
assertEquals(8080, proxy.getPort());
proxy = Proxy.create(URI.create("//username:password@127.0.0.1:8080"));
assertNull(proxy.getScheme());
assertEquals("username", proxy.getUsername());
assertEquals("password", proxy.getPassword());
assertEquals("127.0.0.1", proxy.getHost());
assertEquals(8080, proxy.getPort());
proxy = Proxy.create(URI.create("//username@127.0.0.1:8080"));
assertNull(proxy.getScheme());
assertEquals("username", proxy.getUsername());
assertNull(proxy.getPassword());
assertEquals("127.0.0.1", proxy.getHost());
assertEquals(8080, proxy.getPort());
proxy = Proxy.create(URI.create("//:password@127.0.0.1:8080"));
assertNull(proxy.getScheme());
assertNull(proxy.getUsername());
assertEquals("password", proxy.getPassword());
assertEquals("127.0.0.1", proxy.getHost());
assertEquals(8080, proxy.getPort());
}
@Test
public void testToString() {
assertEquals("//127.0.0.1:8080", new Proxy("127.0.0.1", 8080).toString());
assertEquals("http://127.0.0.1:8080", new Proxy("127.0.0.1", 8080, "http").toString());
assertEquals("//username:password@127.0.0.1:8080", new Proxy("127.0.0.1", 8080, "username", "password").toString());
assertEquals("//username@127.0.0.1:8080", new Proxy("127.0.0.1", 8080, "username", null).toString());
assertEquals("//:password@127.0.0.1:8080", new Proxy("127.0.0.1", 8080, null, "password").toString());
}
} }