diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderMonitor.java b/webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderMonitor.java index ccf498d..2c11d71 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderMonitor.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderMonitor.java @@ -21,31 +21,39 @@ import java.util.concurrent.atomic.AtomicInteger; * @author code4crafer@gmail.com * @since 0.5.0 */ -public class SpiderMonitor implements SpiderMonitorMBean { +public class SpiderMonitor { - private List spiderStatuses = new ArrayList(); + private List spiderStatuses = new ArrayList(); - @Override - public List getSpiders() { + public List getSpiders() { return spiderStatuses; } - @Override - public SpiderStatus getSpider() { + public SpiderStatusMBean getSpider() { return spiderStatuses.get(0); } - public void register(Spider spider) { - MonitorSpiderListener monitorSpiderListener = new MonitorSpiderListener(); - if (spider.getSpiderListeners() == null) { - List spiderListeners = new ArrayList(); - spiderListeners.add(monitorSpiderListener); - spider.setSpiderListeners(spiderListeners); - } else { - spider.getSpiderListeners().add(monitorSpiderListener); + public SpiderMonitor register(Spider... spiders) { + for (Spider spider : spiders) { + MonitorSpiderListener monitorSpiderListener = new MonitorSpiderListener(); + if (spider.getSpiderListeners() == null) { + List spiderListeners = new ArrayList(); + spiderListeners.add(monitorSpiderListener); + spider.setSpiderListeners(spiderListeners); + } else { + spider.getSpiderListeners().add(monitorSpiderListener); + } + spiderStatuses.add(getSpiderStatusMBean(spider, monitorSpiderListener)); } - spiderStatuses.add(new SpiderStatus(spider, monitorSpiderListener)); + return this; + } + protected SpiderStatusMBean getSpiderStatusMBean(Spider spider, MonitorSpiderListener monitorSpiderListener) { + return new SpiderStatus(spider, monitorSpiderListener); + } + + public static SpiderMonitor create(){ + return new SpiderMonitor(); } public class MonitorSpiderListener implements SpiderListener { @@ -81,42 +89,45 @@ public class SpiderMonitor implements SpiderMonitorMBean { } - public static void main(String[] args) throws MalformedObjectNameException, - NullPointerException, InstanceAlreadyExistsException, - MBeanRegistrationException, NotCompliantMBeanException, IOException { - - int rmiPort = 1099; - SpiderMonitor spiderMonitor = new SpiderMonitor(); - String jmxServerName = "TestJMXServer"; - - Spider oschinaSpider = Spider.create(new OschinaBlogPageProcessor()).addUrl("http://my.oschina.net/flashsword/blog").thread(2); - - spiderMonitor.register(oschinaSpider); - - Spider githubSpider = Spider.create(new GithubRepoPageProcessor()).addUrl("https://github.com/code4craft"); - - spiderMonitor.register(githubSpider); + public void jmxStart() throws IOException, JMException { + jmxStart(14721); + } + public void jmxStart(int rmiPort) throws IOException, JMException { + String jmxServerName = "WebMagic"; // jdkfolder/bin/rmiregistry.exe 9999 Registry registry = LocateRegistry.createRegistry(rmiPort); MBeanServer mbs = MBeanServerFactory.createMBeanServer(jmxServerName); //MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); - ObjectName objName = new ObjectName(jmxServerName + ":name=" + "HelloWorld"); - mbs.registerMBean(spiderMonitor, objName); + ObjectName objName; JMXServiceURL url = new JMXServiceURL("service:jmx:rmi:///jndi/rmi://localhost:" + rmiPort + "/" + jmxServerName); System.out.println("JMXServiceURL: " + url.toString()); JMXConnectorServer jmxConnServer = JMXConnectorServerFactory.newJMXConnectorServer(url, null, mbs); jmxConnServer.start(); - for (SpiderStatus spiderStatuse : spiderMonitor.spiderStatuses) { - objName = new ObjectName(jmxServerName + ":name=" + spiderStatuse.getName()); - mbs.registerMBean(spiderStatuse, objName); + for (SpiderStatusMBean spiderStatus : spiderStatuses) { + objName = new ObjectName(jmxServerName + ":name=" + spiderStatus.getName()); + mbs.registerMBean(spiderStatus, objName); } + } + public static void main(String[] args) throws JMException, + NullPointerException, + IOException { + + Spider oschinaSpider = Spider.create(new OschinaBlogPageProcessor()) + .addUrl("http://my.oschina.net/flashsword/blog").thread(2); + Spider githubSpider = Spider.create(new GithubRepoPageProcessor()) + .addUrl("https://github.com/code4craft"); + + SpiderMonitor spiderMonitor = new SpiderMonitor(); + spiderMonitor.register(oschinaSpider, githubSpider); + spiderMonitor.jmxStart(); + } } diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderMonitorMBean.java b/webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderMonitorMBean.java deleted file mode 100644 index 8b77b33..0000000 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderMonitorMBean.java +++ /dev/null @@ -1,14 +0,0 @@ -package us.codecraft.webmagic.monitor; - -import java.util.List; - -/** - * @author code4crafer@gmail.com - */ -public interface SpiderMonitorMBean { - - public List getSpiders(); - - public SpiderStatus getSpider(); - -} diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderStatus.java b/webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderStatus.java index 84d8603..6b49cbb 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderStatus.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderStatus.java @@ -1,5 +1,7 @@ package us.codecraft.webmagic.monitor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import us.codecraft.webmagic.Spider; import java.util.List; @@ -8,11 +10,13 @@ import java.util.List; * @author code4crafer@gmail.com * @since 0.5.0 */ -public class SpiderStatus implements SpiderStatusMBean{ +public class SpiderStatus implements SpiderStatusMBean { - private final Spider spider; + protected final Spider spider; - private final SpiderMonitor.MonitorSpiderListener monitorSpiderListener; + protected Logger logger = LoggerFactory.getLogger(getClass()); + + protected final SpiderMonitor.MonitorSpiderListener monitorSpiderListener; public SpiderStatus(Spider spider, SpiderMonitor.MonitorSpiderListener monitorSpiderListener) { this.spider = spider; @@ -23,24 +27,46 @@ public class SpiderStatus implements SpiderStatusMBean{ return spider.getUUID(); } - public int getLeftPages() { + public int getLeftPageCount() { if (spider.getScheduler() instanceof MonitorableScheduler) { return ((MonitorableScheduler) spider.getScheduler()).getLeftRequestsCount(spider); } + logger.warn("Get leftPageCount fail, try to use a Scheduler implement MonitorableScheduler for monitor count!"); return -1; } - public int getTotalPages() { + public int getTotalPageCount() { if (spider.getScheduler() instanceof MonitorableScheduler) { return ((MonitorableScheduler) spider.getScheduler()).getTotalRequestsCount(spider); } + logger.warn("Get totalPageCount fail, try to use a Scheduler implement MonitorableScheduler for monitor count!"); return -1; } + @Override + public int getSuccessPageCount() { + return monitorSpiderListener.getSuccessCount().get(); + } + + @Override + public int getErrorPageCount() { + return monitorSpiderListener.getErrorCount().get(); + } + public List getErrorPages() { return monitorSpiderListener.getErrorUrls(); } + @Override + public String getStatus() { + return spider.getStatus().name(); + } + + @Override + public int getThread() { + return spider.getThreadAlive(); + } + public void start() { spider.start(); } diff --git a/webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderStatusMBean.java b/webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderStatusMBean.java index cd884a5..156b653 100644 --- a/webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderStatusMBean.java +++ b/webmagic-core/src/main/java/us/codecraft/webmagic/monitor/SpiderStatusMBean.java @@ -10,13 +10,21 @@ public interface SpiderStatusMBean { public String getName(); - public int getLeftPages(); + public String getStatus(); + + public int getThread(); + + public int getTotalPageCount(); + + public int getLeftPageCount(); + + public int getSuccessPageCount(); + + public int getErrorPageCount(); - public int getTotalPages(); public List getErrorPages(); public void start(); public void stop(); - } diff --git a/webmagic-core/src/test/java/us/codecraft/webmagic/monitor/CustomSpiderStatus.java b/webmagic-core/src/test/java/us/codecraft/webmagic/monitor/CustomSpiderStatus.java new file mode 100644 index 0000000..a0b7f77 --- /dev/null +++ b/webmagic-core/src/test/java/us/codecraft/webmagic/monitor/CustomSpiderStatus.java @@ -0,0 +1,19 @@ +package us.codecraft.webmagic.monitor; + +import us.codecraft.webmagic.Spider; + +/** + * @author code4crafer@gmail.com + */ +public class CustomSpiderStatus extends SpiderStatus implements CustomSpiderStatusMBean { + + public CustomSpiderStatus(Spider spider, SpiderMonitor.MonitorSpiderListener monitorSpiderListener) { + super(spider, monitorSpiderListener); + } + + + @Override + public String getSchedulerName() { + return spider.getScheduler().getClass().getName(); + } +} diff --git a/webmagic-core/src/test/java/us/codecraft/webmagic/monitor/CustomSpiderStatusMBean.java b/webmagic-core/src/test/java/us/codecraft/webmagic/monitor/CustomSpiderStatusMBean.java new file mode 100644 index 0000000..104b27d --- /dev/null +++ b/webmagic-core/src/test/java/us/codecraft/webmagic/monitor/CustomSpiderStatusMBean.java @@ -0,0 +1,10 @@ +package us.codecraft.webmagic.monitor; + +/** + * @author code4crafer@gmail.com + */ +public interface CustomSpiderStatusMBean extends SpiderStatusMBean { + + public String getSchedulerName(); + +} diff --git a/webmagic-core/src/test/java/us/codecraft/webmagic/monitor/SpiderMonitorTest.java b/webmagic-core/src/test/java/us/codecraft/webmagic/monitor/SpiderMonitorTest.java new file mode 100644 index 0000000..450f3fe --- /dev/null +++ b/webmagic-core/src/test/java/us/codecraft/webmagic/monitor/SpiderMonitorTest.java @@ -0,0 +1,32 @@ +package us.codecraft.webmagic.monitor; + +import org.junit.Test; +import us.codecraft.webmagic.Spider; +import us.codecraft.webmagic.processor.example.GithubRepoPageProcessor; +import us.codecraft.webmagic.processor.example.OschinaBlogPageProcessor; + +/** + * @author code4crafer@gmail.com + * @since 0.5.0 + */ +public class SpiderMonitorTest { + + @Test + public void testInherit() throws Exception { + SpiderMonitor spiderMonitor = new SpiderMonitor(){ + @Override + protected SpiderStatusMBean getSpiderStatusMBean(Spider spider, MonitorSpiderListener monitorSpiderListener) { + return new CustomSpiderStatus(spider, monitorSpiderListener); + } + }; + + Spider oschinaSpider = Spider.create(new OschinaBlogPageProcessor()) + .addUrl("http://my.oschina.net/flashsword/blog").thread(2); + Spider githubSpider = Spider.create(new GithubRepoPageProcessor()) + .addUrl("https://github.com/code4craft"); + + spiderMonitor.register(oschinaSpider, githubSpider); + spiderMonitor.jmxStart(); + + } +}