fix spell error=.=
parent
7bed01c9f2
commit
b1f023ead5
|
@ -66,7 +66,7 @@ public class Spider implements Runnable, Task {
|
|||
return this;
|
||||
}
|
||||
|
||||
public Spider schedular(Scheduler scheduler) {
|
||||
public Spider scheduler(Scheduler scheduler) {
|
||||
this.scheduler = scheduler;
|
||||
return this;
|
||||
}
|
||||
|
|
|
@ -63,7 +63,7 @@ public class FileCacheQueueScheduler implements Scheduler {
|
|||
initWriter();
|
||||
initFlushThread();
|
||||
inited.set(true);
|
||||
logger.info("init cache schedular success");
|
||||
logger.info("init cache scheduler success");
|
||||
}
|
||||
|
||||
private void initFlushThread() {
|
||||
|
@ -80,7 +80,7 @@ public class FileCacheQueueScheduler implements Scheduler {
|
|||
fileUrlWriter = new PrintWriter(new FileWriter(getFileName(fileUrlAllName), true));
|
||||
fileCursorWriter = new PrintWriter(new FileWriter(getFileName(fileCursor), false));
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("init cache schedular error", e);
|
||||
throw new RuntimeException("init cache scheduler error", e);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -26,12 +26,12 @@ public class SpiderTest {
|
|||
@Test
|
||||
public void testGlobalSpider(){
|
||||
// PageProcessor pageProcessor = new MeicanProcessor();
|
||||
// Spider.me().pipeline(new FilePipeline()).schedular(new FileCacheQueueScheduler(pageProcessor.getSite(),"/data/temp/webmagic/cache/")).
|
||||
// Spider.me().pipeline(new FilePipeline()).scheduler(new FileCacheQueueScheduler(pageProcessor.getSite(),"/data/temp/webmagic/cache/")).
|
||||
// processor(pageProcessor).run();
|
||||
SimplePageProcessor pageProcessor2 = new SimplePageProcessor("http://www.diaoyuweng.com/home.php?mod=space&uid=88304&do=thread&view=me&type=thread&from=space", "http://www.diaoyuweng.com/thread-*-1-1.html");
|
||||
System.out.println(pageProcessor2.getSite().getEncoding());
|
||||
pageProcessor2.getSite().setSleepTime(500);
|
||||
Spider.create(pageProcessor2).pipeline(new FilePipeline()).schedular(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
|
||||
Spider.create(pageProcessor2).pipeline(new FilePipeline()).scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
|
||||
run();
|
||||
|
||||
|
||||
|
|
|
@ -30,7 +30,7 @@ public class DiandianProcessorTest {
|
|||
//ConsolePipeline输出结果到控制台
|
||||
//FileCacheQueueSchedular保存url,支持断点续传,临时文件输出到/data/temp/webmagic/cache目录
|
||||
//Spider.run()执行
|
||||
Spider.create(diaoyuwengProcessor).pipeline(new ConsolePipeline()).pipeline(pipeline).schedular(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
|
||||
Spider.create(diaoyuwengProcessor).pipeline(new ConsolePipeline()).pipeline(pipeline).scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
|
||||
run();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -22,7 +22,7 @@ public class DiaoyuwengProcessorTest {
|
|||
public void test() throws IOException {
|
||||
DiaoyuwengProcessor diaoyuwengProcessor = new DiaoyuwengProcessor();
|
||||
FreemarkerPipeline pipeline = new FreemarkerPipeline("wordpress.ftl");
|
||||
Spider.create(diaoyuwengProcessor).pipeline(new FilePipeline()).pipeline(pipeline).schedular(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
|
||||
Spider.create(diaoyuwengProcessor).pipeline(new FilePipeline()).pipeline(pipeline).scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
|
||||
run();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -30,7 +30,7 @@ public class SinablogProcessorTest {
|
|||
//ConsolePipeline输出结果到控制台
|
||||
//FileCacheQueueSchedular保存url,支持断点续传,临时文件输出到/data/temp/webmagic/cache目录
|
||||
//Spider.run()执行
|
||||
Spider.create(sinaBlogProcesser).pipeline(new FilePipeline()).pipeline(pipeline).schedular(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
|
||||
Spider.create(sinaBlogProcesser).pipeline(new FilePipeline()).pipeline(pipeline).scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
|
||||
run();
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue