Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

extension point and sample for JMX support #98

master
yihua.huang 2014-04-17 08:12:37 +08:00
parent a5db6cf292
commit 27b37e8164
7 changed files with 149 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderMonitor.java
View File

@ -21,31 +21,39 @@ import java.util.concurrent.atomic.AtomicInteger;
* @author code4crafer@gmail.com
* @since 0.5.0
*/
public class SpiderMonitor implements SpiderMonitorMBean {
public class SpiderMonitor {
private List<SpiderStatus> spiderStatuses = new ArrayList<SpiderStatus>();
private List<SpiderStatusMBean> spiderStatuses = new ArrayList<SpiderStatusMBean>();
@Override
public List<SpiderStatus> getSpiders() {
public List<SpiderStatusMBean> getSpiders() {
return spiderStatuses;
}
@Override
public SpiderStatus getSpider() {
public SpiderStatusMBean getSpider() {
return spiderStatuses.get(0);
}
public void register(Spider spider) {
MonitorSpiderListener monitorSpiderListener = new MonitorSpiderListener();
if (spider.getSpiderListeners() == null) {
List<SpiderListener> spiderListeners = new ArrayList<SpiderListener>();
spiderListeners.add(monitorSpiderListener);
spider.setSpiderListeners(spiderListeners);
} else {
spider.getSpiderListeners().add(monitorSpiderListener);
public SpiderMonitor register(Spider... spiders) {
for (Spider spider : spiders) {
MonitorSpiderListener monitorSpiderListener = new MonitorSpiderListener();
if (spider.getSpiderListeners() == null) {
List<SpiderListener> spiderListeners = new ArrayList<SpiderListener>();
spiderListeners.add(monitorSpiderListener);
spider.setSpiderListeners(spiderListeners);
} else {
spider.getSpiderListeners().add(monitorSpiderListener);
}
spiderStatuses.add(getSpiderStatusMBean(spider, monitorSpiderListener));
}
spiderStatuses.add(new SpiderStatus(spider, monitorSpiderListener));
return this;
}
protected SpiderStatusMBean getSpiderStatusMBean(Spider spider, MonitorSpiderListener monitorSpiderListener) {
return new SpiderStatus(spider, monitorSpiderListener);
}
public static SpiderMonitor create(){
return new SpiderMonitor();
}
public class MonitorSpiderListener implements SpiderListener {
@ -81,42 +89,45 @@ public class SpiderMonitor implements SpiderMonitorMBean {
}
public static void main(String[] args) throws MalformedObjectNameException,
NullPointerException, InstanceAlreadyExistsException,
MBeanRegistrationException, NotCompliantMBeanException, IOException {
int rmiPort = 1099;
SpiderMonitor spiderMonitor = new SpiderMonitor();
String jmxServerName = "TestJMXServer";
Spider oschinaSpider = Spider.create(new OschinaBlogPageProcessor()).addUrl("http://my.oschina.net/flashsword/blog").thread(2);
spiderMonitor.register(oschinaSpider);
Spider githubSpider = Spider.create(new GithubRepoPageProcessor()).addUrl("https://github.com/code4craft");
spiderMonitor.register(githubSpider);
public void jmxStart() throws IOException, JMException {
jmxStart(14721);
}
public void jmxStart(int rmiPort) throws IOException, JMException {
String jmxServerName = "WebMagic";
// jdkfolder/bin/rmiregistry.exe 9999
Registry registry = LocateRegistry.createRegistry(rmiPort);
MBeanServer mbs = MBeanServerFactory.createMBeanServer(jmxServerName);
//MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
ObjectName objName = new ObjectName(jmxServerName + ":name=" + "HelloWorld");
mbs.registerMBean(spiderMonitor, objName);
ObjectName objName;
JMXServiceURL url = new JMXServiceURL("service:jmx:rmi:///jndi/rmi://localhost:" + rmiPort + "/" + jmxServerName);
System.out.println("JMXServiceURL: " + url.toString());
JMXConnectorServer jmxConnServer = JMXConnectorServerFactory.newJMXConnectorServer(url, null, mbs);
jmxConnServer.start();
for (SpiderStatus spiderStatuse : spiderMonitor.spiderStatuses) {
objName = new ObjectName(jmxServerName + ":name=" + spiderStatuse.getName());
mbs.registerMBean(spiderStatuse, objName);
for (SpiderStatusMBean spiderStatus : spiderStatuses) {
objName = new ObjectName(jmxServerName + ":name=" + spiderStatus.getName());
mbs.registerMBean(spiderStatus, objName);
}
}
public static void main(String[] args) throws JMException,
NullPointerException,
IOException {
Spider oschinaSpider = Spider.create(new OschinaBlogPageProcessor())
.addUrl("http://my.oschina.net/flashsword/blog").thread(2);
Spider githubSpider = Spider.create(new GithubRepoPageProcessor())
.addUrl("https://github.com/code4craft");
SpiderMonitor spiderMonitor = new SpiderMonitor();
spiderMonitor.register(oschinaSpider, githubSpider);
spiderMonitor.jmxStart();
}
}

14
webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderMonitorMBean.java
View File

@ -1,14 +0,0 @@
package us.codecraft.webmagic.monitor;
import java.util.List;
/**
* @author code4crafer@gmail.com
*/
public interface SpiderMonitorMBean {
public List<SpiderStatus> getSpiders();
public SpiderStatus getSpider();
}

36
webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderStatus.java
View File

@ -1,5 +1,7 @@
package us.codecraft.webmagic.monitor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.Spider;
import java.util.List;
@ -8,11 +10,13 @@ import java.util.List;
* @author code4crafer@gmail.com
* @since 0.5.0
*/
public class SpiderStatus implements SpiderStatusMBean{
public class SpiderStatus implements SpiderStatusMBean {
private final Spider spider;
protected final Spider spider;
private final SpiderMonitor.MonitorSpiderListener monitorSpiderListener;
protected Logger logger = LoggerFactory.getLogger(getClass());
protected final SpiderMonitor.MonitorSpiderListener monitorSpiderListener;
public SpiderStatus(Spider spider, SpiderMonitor.MonitorSpiderListener monitorSpiderListener) {
this.spider = spider;
@ -23,24 +27,46 @@ public class SpiderStatus implements SpiderStatusMBean{
return spider.getUUID();
}
public int getLeftPages() {
public int getLeftPageCount() {
if (spider.getScheduler() instanceof MonitorableScheduler) {
return ((MonitorableScheduler) spider.getScheduler()).getLeftRequestsCount(spider);
}
logger.warn("Get leftPageCount fail, try to use a Scheduler implement MonitorableScheduler for monitor count!");
return -1;
}
public int getTotalPages() {
public int getTotalPageCount() {
if (spider.getScheduler() instanceof MonitorableScheduler) {
return ((MonitorableScheduler) spider.getScheduler()).getTotalRequestsCount(spider);
}
logger.warn("Get totalPageCount fail, try to use a Scheduler implement MonitorableScheduler for monitor count!");
return -1;
}
@Override
public int getSuccessPageCount() {
return monitorSpiderListener.getSuccessCount().get();
}
@Override
public int getErrorPageCount() {
return monitorSpiderListener.getErrorCount().get();
}
public List<String> getErrorPages() {
return monitorSpiderListener.getErrorUrls();
}
@Override
public String getStatus() {
return spider.getStatus().name();
}
@Override
public int getThread() {
return spider.getThreadAlive();
}
public void start() {
spider.start();
}

14
webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderStatusMBean.java
View File

@ -10,13 +10,21 @@ public interface SpiderStatusMBean {
public String getName();
public int getLeftPages();
public String getStatus();
public int getThread();
public int getTotalPageCount();
public int getLeftPageCount();
public int getSuccessPageCount();
public int getErrorPageCount();
public int getTotalPages();
public List<String> getErrorPages();
public void start();
public void stop();
}

19
webmagic-core/src/test/java/us/codecraft/webmagic/monitor/CustomSpiderStatus.java 100644
View File

@ -0,0 +1,19 @@
package us.codecraft.webmagic.monitor;
import us.codecraft.webmagic.Spider;
/**
* @author code4crafer@gmail.com
*/
public class CustomSpiderStatus extends SpiderStatus implements CustomSpiderStatusMBean {
public CustomSpiderStatus(Spider spider, SpiderMonitor.MonitorSpiderListener monitorSpiderListener) {
super(spider, monitorSpiderListener);
}
@Override
public String getSchedulerName() {
return spider.getScheduler().getClass().getName();
}
}

10
webmagic-core/src/test/java/us/codecraft/webmagic/monitor/CustomSpiderStatusMBean.java 100644
View File

@ -0,0 +1,10 @@
package us.codecraft.webmagic.monitor;
/**
* @author code4crafer@gmail.com
*/
public interface CustomSpiderStatusMBean extends SpiderStatusMBean {
public String getSchedulerName();
}

32
webmagic-core/src/test/java/us/codecraft/webmagic/monitor/SpiderMonitorTest.java 100644
View File

@ -0,0 +1,32 @@
package us.codecraft.webmagic.monitor;
import org.junit.Test;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.example.GithubRepoPageProcessor;
import us.codecraft.webmagic.processor.example.OschinaBlogPageProcessor;
/**
* @author code4crafer@gmail.com
* @since 0.5.0
*/
public class SpiderMonitorTest {
@Test
public void testInherit() throws Exception {
SpiderMonitor spiderMonitor = new SpiderMonitor(){
@Override
protected SpiderStatusMBean getSpiderStatusMBean(Spider spider, MonitorSpiderListener monitorSpiderListener) {
return new CustomSpiderStatus(spider, monitorSpiderListener);
}
};
Spider oschinaSpider = Spider.create(new OschinaBlogPageProcessor())
.addUrl("http://my.oschina.net/flashsword/blog").thread(2);
Spider githubSpider = Spider.create(new GithubRepoPageProcessor())
.addUrl("https://github.com/code4craft");
spiderMonitor.register(oschinaSpider, githubSpider);
spiderMonitor.jmxStart();
}
}