webMagic-Mobai
/
magic-Dependency
mirror of https://gitea.qinmian.online/Mobai/magic-Dependency.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update lucene pipeline

master
yihua.huang 2013-08-06 21:47:44 +08:00
parent 29f8cd2ec6
commit 7d277e84d4
6 changed files with 57 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
View File

@ -228,8 +228,10 @@ public class Spider implements Runnable, Task {
}
pageProcessor.process(page);
addRequest(page);
for (Pipeline pipeline : pipelines) {
pipeline.process(page.getResultItems(), this);
if (!page.getResultItems().isSkip()){
for (Pipeline pipeline : pipelines) {
pipeline.process(page.getResultItems(), this);
}
}
sleep(site.getSleepTime());
}

3
webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/ConsolePipeline.java
View File

@ -15,9 +15,6 @@ public class ConsolePipeline implements Pipeline{
@Override
public void process(ResultItems resultItems,Task task) {
if (resultItems.isSkip()){
return;
}
System.out.println("get page: "+resultItems.getRequest().getUrl());
for (Map.Entry<String, Object> entry : resultItems.getAll().entrySet()) {
if (entry.getValue() instanceof Iterable) {

3
webmagic-core/src/main/java/us/codecraft/webmagic/pipeline/FilePipeline.java
View File

@ -47,9 +47,6 @@ public class FilePipeline implements Pipeline {
if (!file.exists()) {
file.mkdirs();
}
if (resultItems.isSkip()) {
return;
}
try {
PrintWriter printWriter = new PrintWriter(new FileWriter(path + DigestUtils.md5Hex(resultItems.getRequest().getUrl()) + ".html"));
printWriter.println("url:\t" + resultItems.getRequest().getUrl());

77
webmagic-plugin/webmagic-lucene/src/main/java/us/codecraft/webmagic/pipeline/LucenePipeline.java
View File

@ -3,20 +3,26 @@ package us.codecraft.webmagic.pipeline;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Version;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* @author yihua.huang@dianping.com <br>
@ -24,41 +30,64 @@ import java.io.File;
* Time: 下午2:11 <br>
*/
public class LucenePipeline implements Pipeline {
@Override
public void process(ResultItems resultItems, Task task) {
private Directory directory;
private IndexWriter indexWriter;
private Analyzer analyzer;
private void init() throws IOException {
analyzer = new StandardAnalyzer(Version.LUCENE_44);
directory = new RAMDirectory();
IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_44, analyzer);
indexWriter = new IndexWriter(directory, config);
indexWriter.close();
}
public LucenePipeline() {
try {
} catch (Exception e) {
init();
} catch (IOException e) {
e.printStackTrace();
}
}
public static void main(String[] args) throws Exception {
Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_44);
// Directory directory = new RAMDirectory();
// To store an index on disk, use this instead:
Directory directory = FSDirectory.open(new File("/data/webmagic/www.guoxue123.cn/"));
IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_44, analyzer);
IndexWriter iwriter = new IndexWriter(directory, config);
Document doc = new Document();
// String text = "This is the text to be indexed.";
// doc.add(new Field("fieldname", text, TextField.TYPE_STORED));
// iwriter.addDocument(doc);
iwriter.close();
// Now search the index:
public List<Document> search(String fieldName, String value) throws IOException, ParseException {
List<Document> documents = new ArrayList<Document>();
DirectoryReader ireader = DirectoryReader.open(directory);
IndexSearcher isearcher = new IndexSearcher(ireader);
// Parse a simple query that searches for "text":
QueryParser parser = new QueryParser(Version.LUCENE_44, "fieldname", analyzer);
Query query = parser.parse("经典");
QueryParser parser = new QueryParser(Version.LUCENE_44, fieldName, analyzer);
Query query = parser.parse(value);
ScoreDoc[] hits = isearcher.search(query, null, 1000).scoreDocs;
// Iterate through the results:
for (int i = 0; i < hits.length; i++) {
Document hitDoc = isearcher.doc(hits[i].doc);
System.out.println(hitDoc);
documents.add(hitDoc);
}
ireader.close();
directory.close();
return documents;
}
@Override
public void process(ResultItems resultItems, Task task) {
if (resultItems.isSkip()){
return;
}
Document doc = new Document();
Map<String,Object> all = resultItems.getAll();
if (all==null){
return;
}
for (Map.Entry<String, Object> objectEntry : all.entrySet()) {
doc.add(new Field(objectEntry.getKey(), objectEntry.getValue().toString(), TextField.TYPE_STORED));
}
try {
indexWriter.addDocument(doc);
} catch (IOException e) {
e.printStackTrace();
}
}
}

3
webmagic-plugin/webmagic-misc/src/main/java/us/codecraft/webmagic/model/ModelPipeline.java
View File

@ -29,9 +29,6 @@ class ModelPipeline implements Pipeline {
@Override
public void process(ResultItems resultItems, Task task) {
if (resultItems.isSkip()) {
return;
}
for (Map.Entry<Class, PageModelPipeline> classPageModelPipelineEntry : pageModelPipelines.entrySet()) {
Object o = resultItems.get(classPageModelPipelineEntry.getKey().getCanonicalName());
if (o != null) {

3
webmagic-samples/src/main/java/us/codecraft/webmagic/pipeline/FreemarkerPipeline.java
View File

@ -40,9 +40,6 @@ public class FreemarkerPipeline implements Pipeline {
@Override
public void process(ResultItems resultItems, Task task) {
if (resultItems.isSkip()) {
return;
}
String path = this.path + "" + task.getUUID() + "/";
File file = new File(path);
if (!file.exists()) {