From 1b886d48a2de1f621305a802faca5d94f72f7f86 Mon Sep 17 00:00:00 2001 From: Jsbd Date: Thu, 8 Dec 2016 14:29:42 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9EPhantomJSDownloader=E6=9E=84?= =?UTF-8?q?=E9=80=A0=E5=87=BD=E6=95=B0=EF=BC=8C=E6=94=AF=E6=8C=81crawl.js?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E8=87=AA=E5=AE=9A=E4=B9=89=EF=BC=8C=E5=9B=A0?= =?UTF-8?q?=E4=B8=BA=E5=BD=93=E5=85=B6=E4=BB=96=E9=A1=B9=E7=9B=AE=E4=BE=9D?= =?UTF-8?q?=E8=B5=96=E6=AD=A4jar=E5=8C=85=E6=97=B6=EF=BC=8Cruntime.exec()?= =?UTF-8?q?=E6=89=A7=E8=A1=8Cphantomjs=E5=91=BD=E4=BB=A4=E6=97=B6=E6=97=A0?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E6=B3=95jar=E5=8C=85=E4=B8=AD=E7=9A=84crawl.?= =?UTF-8?q?js?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../downloader/PhantomJSDownloader.java | 51 +++++++++++++++---- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/PhantomJSDownloader.java b/webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/PhantomJSDownloader.java index bea44fd..a955e73 100644 --- a/webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/PhantomJSDownloader.java +++ b/webmagic-extension/src/main/java/us/codecraft/webmagic/downloader/PhantomJSDownloader.java @@ -29,21 +29,54 @@ public class PhantomJSDownloader extends AbstractDownloader { public PhantomJSDownloader() { this.initPhantomjsCrawlPath(); } + /** - * 添加新的构造函数,支持phantomjs自定义命令 - * - * example: - * phantomjs.exe 支持windows环境 - * phantomjs --ignore-ssl-errors=yes 忽略抓取地址是https时的一些错误 - * /usr/local/bin/phantomjs 命令的绝对路径,避免因系统环境变量引起的IOException - * - * @param phantomJsCommand - */ + * 添加新的构造函数,支持phantomjs自定义命令 + * + * example: + * phantomjs.exe 支持windows环境 + * phantomjs --ignore-ssl-errors=yes 忽略抓取地址是https时的一些错误 + * /usr/local/bin/phantomjs 命令的绝对路径,避免因系统环境变量引起的IOException + * + * @param phantomJsCommand + */ public PhantomJSDownloader(String phantomJsCommand) { this.initPhantomjsCrawlPath(); PhantomJSDownloader.phantomJsCommand = phantomJsCommand; } + /** + * 新增构造函数,支持crawl.js路径自定义,因为当其他项目依赖此jar包时,runtime.exec()执行phantomjs命令时无使用法jar包中的crawl.js + * + * crawl.js start -->> + * + * var system = require('system'); + * var url = system.args[1]; + * + * var page = require('webpage').create(); + * page.settings.loadImages = false; + * page.settings.resourceTimeout = 5000; + * + * page.open(url, function (status) { + * if (status != 'success') { + * console.log("HTTP request failed!"); + * } else { + * console.log(page.content); + * } + * + * page.close(); + * phantom.exit(); + * }); + * + * <<-- crawl.js end + * 具体项目时可以将以上js代码复制下来使用 + * + * example: + * new PhantomJSDownloader("/your/path/phantomjs", "/your/path/crawl.js"); + * + * @param phantomJsCommand + * @param crawlJsPath + */ public PhantomJSDownloader(String phantomJsCommand, String crawlJsPath) { PhantomJSDownloader.phantomJsCommand = phantomJsCommand; PhantomJSDownloader.crawlJsPath = crawlJsPath;