fix spell error=.=
parent
7bed01c9f2
commit
b1f023ead5
|
@ -66,7 +66,7 @@ public class Spider implements Runnable, Task {
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Spider schedular(Scheduler scheduler) {
|
public Spider scheduler(Scheduler scheduler) {
|
||||||
this.scheduler = scheduler;
|
this.scheduler = scheduler;
|
||||||
return this;
|
return this;
|
||||||
}
|
}
|
||||||
|
|
|
@ -63,7 +63,7 @@ public class FileCacheQueueScheduler implements Scheduler {
|
||||||
initWriter();
|
initWriter();
|
||||||
initFlushThread();
|
initFlushThread();
|
||||||
inited.set(true);
|
inited.set(true);
|
||||||
logger.info("init cache schedular success");
|
logger.info("init cache scheduler success");
|
||||||
}
|
}
|
||||||
|
|
||||||
private void initFlushThread() {
|
private void initFlushThread() {
|
||||||
|
@ -80,7 +80,7 @@ public class FileCacheQueueScheduler implements Scheduler {
|
||||||
fileUrlWriter = new PrintWriter(new FileWriter(getFileName(fileUrlAllName), true));
|
fileUrlWriter = new PrintWriter(new FileWriter(getFileName(fileUrlAllName), true));
|
||||||
fileCursorWriter = new PrintWriter(new FileWriter(getFileName(fileCursor), false));
|
fileCursorWriter = new PrintWriter(new FileWriter(getFileName(fileCursor), false));
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
throw new RuntimeException("init cache schedular error", e);
|
throw new RuntimeException("init cache scheduler error", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -26,12 +26,12 @@ public class SpiderTest {
|
||||||
@Test
|
@Test
|
||||||
public void testGlobalSpider(){
|
public void testGlobalSpider(){
|
||||||
// PageProcessor pageProcessor = new MeicanProcessor();
|
// PageProcessor pageProcessor = new MeicanProcessor();
|
||||||
// Spider.me().pipeline(new FilePipeline()).schedular(new FileCacheQueueScheduler(pageProcessor.getSite(),"/data/temp/webmagic/cache/")).
|
// Spider.me().pipeline(new FilePipeline()).scheduler(new FileCacheQueueScheduler(pageProcessor.getSite(),"/data/temp/webmagic/cache/")).
|
||||||
// processor(pageProcessor).run();
|
// processor(pageProcessor).run();
|
||||||
SimplePageProcessor pageProcessor2 = new SimplePageProcessor("http://www.diaoyuweng.com/home.php?mod=space&uid=88304&do=thread&view=me&type=thread&from=space", "http://www.diaoyuweng.com/thread-*-1-1.html");
|
SimplePageProcessor pageProcessor2 = new SimplePageProcessor("http://www.diaoyuweng.com/home.php?mod=space&uid=88304&do=thread&view=me&type=thread&from=space", "http://www.diaoyuweng.com/thread-*-1-1.html");
|
||||||
System.out.println(pageProcessor2.getSite().getEncoding());
|
System.out.println(pageProcessor2.getSite().getEncoding());
|
||||||
pageProcessor2.getSite().setSleepTime(500);
|
pageProcessor2.getSite().setSleepTime(500);
|
||||||
Spider.create(pageProcessor2).pipeline(new FilePipeline()).schedular(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
|
Spider.create(pageProcessor2).pipeline(new FilePipeline()).scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
|
||||||
run();
|
run();
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -30,7 +30,7 @@ public class DiandianProcessorTest {
|
||||||
//ConsolePipeline输出结果到控制台
|
//ConsolePipeline输出结果到控制台
|
||||||
//FileCacheQueueSchedular保存url,支持断点续传,临时文件输出到/data/temp/webmagic/cache目录
|
//FileCacheQueueSchedular保存url,支持断点续传,临时文件输出到/data/temp/webmagic/cache目录
|
||||||
//Spider.run()执行
|
//Spider.run()执行
|
||||||
Spider.create(diaoyuwengProcessor).pipeline(new ConsolePipeline()).pipeline(pipeline).schedular(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
|
Spider.create(diaoyuwengProcessor).pipeline(new ConsolePipeline()).pipeline(pipeline).scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
|
||||||
run();
|
run();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -22,7 +22,7 @@ public class DiaoyuwengProcessorTest {
|
||||||
public void test() throws IOException {
|
public void test() throws IOException {
|
||||||
DiaoyuwengProcessor diaoyuwengProcessor = new DiaoyuwengProcessor();
|
DiaoyuwengProcessor diaoyuwengProcessor = new DiaoyuwengProcessor();
|
||||||
FreemarkerPipeline pipeline = new FreemarkerPipeline("wordpress.ftl");
|
FreemarkerPipeline pipeline = new FreemarkerPipeline("wordpress.ftl");
|
||||||
Spider.create(diaoyuwengProcessor).pipeline(new FilePipeline()).pipeline(pipeline).schedular(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
|
Spider.create(diaoyuwengProcessor).pipeline(new FilePipeline()).pipeline(pipeline).scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
|
||||||
run();
|
run();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -30,7 +30,7 @@ public class SinablogProcessorTest {
|
||||||
//ConsolePipeline输出结果到控制台
|
//ConsolePipeline输出结果到控制台
|
||||||
//FileCacheQueueSchedular保存url,支持断点续传,临时文件输出到/data/temp/webmagic/cache目录
|
//FileCacheQueueSchedular保存url,支持断点续传,临时文件输出到/data/temp/webmagic/cache目录
|
||||||
//Spider.run()执行
|
//Spider.run()执行
|
||||||
Spider.create(sinaBlogProcesser).pipeline(new FilePipeline()).pipeline(pipeline).schedular(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
|
Spider.create(sinaBlogProcesser).pipeline(new FilePipeline()).pipeline(pipeline).scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
|
||||||
run();
|
run();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue