add priority to request
parent
971e7b6ce2
commit
d56c681be1
|
@ -28,6 +28,8 @@ public class Request {
|
|||
|
||||
private Object[] extra;
|
||||
|
||||
private double priority;
|
||||
|
||||
/**
|
||||
* 构建一个request对象
|
||||
* @param url 必须参数,待抓取的url
|
||||
|
@ -38,6 +40,15 @@ public class Request {
|
|||
this.extra = extra;
|
||||
}
|
||||
|
||||
public double getPriority() {
|
||||
return priority;
|
||||
}
|
||||
|
||||
public Request setPriority(double priority) {
|
||||
this.priority = priority;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取预存的对象
|
||||
* @return object[] 预存的对象数组
|
||||
|
@ -54,4 +65,20 @@ public class Request {
|
|||
return url;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) return true;
|
||||
if (o == null || getClass() != o.getClass()) return false;
|
||||
|
||||
Request request = (Request) o;
|
||||
|
||||
if (!url.equals(request.url)) return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return url.hashCode();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -12,6 +12,7 @@
|
|||
<modules>
|
||||
<module>webmagic-misc</module>
|
||||
<module>webmagic-selenium</module>
|
||||
<module>webmagic-lucene</module>
|
||||
</modules>
|
||||
|
||||
<artifactId>webmagic-plugin</artifactId>
|
||||
|
|
|
@ -0,0 +1,28 @@
|
|||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<artifactId>webmagic-plugin</artifactId>
|
||||
<groupId>us.codecraft</groupId>
|
||||
<version>0.1.0</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>webmagic-lucene</artifactId>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.lucene</groupId>
|
||||
<artifactId>lucene-analyzers-common</artifactId>
|
||||
<version>4.4.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.lucene</groupId>
|
||||
<artifactId>lucene-queryparser</artifactId>
|
||||
<version>4.4.0</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
|
||||
</project>
|
|
@ -0,0 +1,64 @@
|
|||
package us.codecraft.webmagic.pipeline;
|
||||
|
||||
import org.apache.lucene.analysis.Analyzer;
|
||||
import org.apache.lucene.analysis.standard.StandardAnalyzer;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.index.DirectoryReader;
|
||||
import org.apache.lucene.index.IndexWriter;
|
||||
import org.apache.lucene.index.IndexWriterConfig;
|
||||
import org.apache.lucene.queryparser.classic.QueryParser;
|
||||
import org.apache.lucene.search.IndexSearcher;
|
||||
import org.apache.lucene.search.Query;
|
||||
import org.apache.lucene.search.ScoreDoc;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.store.FSDirectory;
|
||||
import org.apache.lucene.util.Version;
|
||||
import us.codecraft.webmagic.ResultItems;
|
||||
import us.codecraft.webmagic.Task;
|
||||
|
||||
import java.io.File;
|
||||
|
||||
/**
|
||||
* @author yihua.huang@dianping.com <br>
|
||||
* @date: 13-8-5 <br>
|
||||
* Time: 下午2:11 <br>
|
||||
*/
|
||||
public class LucenePipeline implements Pipeline {
|
||||
@Override
|
||||
public void process(ResultItems resultItems, Task task) {
|
||||
try {
|
||||
|
||||
} catch (Exception e) {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_44);
|
||||
// Directory directory = new RAMDirectory();
|
||||
// To store an index on disk, use this instead:
|
||||
Directory directory = FSDirectory.open(new File("/data/webmagic/www.guoxue123.cn/"));
|
||||
IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_44, analyzer);
|
||||
IndexWriter iwriter = new IndexWriter(directory, config);
|
||||
Document doc = new Document();
|
||||
// String text = "This is the text to be indexed.";
|
||||
// doc.add(new Field("fieldname", text, TextField.TYPE_STORED));
|
||||
// iwriter.addDocument(doc);
|
||||
iwriter.close();
|
||||
|
||||
// Now search the index:
|
||||
DirectoryReader ireader = DirectoryReader.open(directory);
|
||||
IndexSearcher isearcher = new IndexSearcher(ireader);
|
||||
// Parse a simple query that searches for "text":
|
||||
QueryParser parser = new QueryParser(Version.LUCENE_44, "fieldname", analyzer);
|
||||
Query query = parser.parse("经典");
|
||||
ScoreDoc[] hits = isearcher.search(query, null, 1000).scoreDocs;
|
||||
// Iterate through the results:
|
||||
for (int i = 0; i < hits.length; i++) {
|
||||
Document hitDoc = isearcher.doc(hits[i].doc);
|
||||
System.out.println(hitDoc);
|
||||
}
|
||||
ireader.close();
|
||||
directory.close();
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue