Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

提交可恢复爬取内容例子

master
linweisen 2021-04-09 17:00:00 +08:00
parent be6f5ff771
commit 76f625c02e
4 changed files with 213 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
webmagic-samples/pom.xml
View File

@ -24,6 +24,26 @@
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
<dependency>
<groupId>org.mapdb</groupId>
<artifactId>mapdb</artifactId>
<version>3.0.7</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.9.5</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>2.9.5</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.9.5</version>
</dependency>
</dependencies>
</project>

82
webmagic-samples/src/main/java/us/codecraft/webmagic/recover/DuplicateStorageRemover.java 100644
View File

@ -0,0 +1,82 @@
package us.codecraft.webmagic.recover;
import com.google.common.base.Charsets;
import com.google.common.hash.BloomFilter;
import com.google.common.hash.Funnels;
import org.mapdb.DB;
import org.mapdb.DBMaker;
import org.mapdb.IndexTreeList;
import org.mapdb.Serializer;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.scheduler.component.DuplicateRemover;
import java.util.concurrent.atomic.AtomicInteger;
/**
* @author ：linweisen
* @date ：Created in 2021/4/9 14:46
* @description：${description}
* @modified By：
* @version: 1.0
*/
public class DuplicateStorageRemover implements DuplicateRemover {
private DB db;
private static String DATABASE_NAME = "duplicate";
private IndexTreeList<String> urlDuplicateQueue;
private BloomFilter<CharSequence> bloomFilter;
private AtomicInteger counter;
public DuplicateStorageRemover(String path) {
String duplicatStoragePath = path;
DB db = DBMaker.fileDB(duplicatStoragePath)
.fileMmapEnableIfSupported()
.fileMmapPreclearDisable()
.cleanerHackEnable()
.closeOnJvmShutdown()
.transactionEnable()
.concurrencyScale(128)
.make();
this.db = db;
this.urlDuplicateQueue = db.indexTreeList(DATABASE_NAME, Serializer.STRING).createOrOpen();
counter = new AtomicInteger(this.urlDuplicateQueue.size());
this.bloomFilter = BloomFilter.create(Funnels.stringFunnel(Charsets.UTF_8), 200000, 1E-7);
for (String url : this.urlDuplicateQueue){
bloomFilter.put(url);
}
}
@Override
public boolean isDuplicate(Request request, Task task) {
String url = request.getUrl();
boolean isDuplicate = bloomFilter.mightContain(url);
if (!isDuplicate) {
bloomFilter.put(url);
urlDuplicateQueue.add(url);
this.db.commit();
counter.incrementAndGet();
}
return isDuplicate;
}
@Override
public void resetDuplicateCheck(Task task) {
this.bloomFilter = BloomFilter.create(Funnels.stringFunnel(Charsets.UTF_8), 200000, 1E-7);
this.urlDuplicateQueue.clear();
}
@Override
public int getTotalRequestsCount(Task task) {
return counter.get();
}
}

89
webmagic-samples/src/main/java/us/codecraft/webmagic/recover/MmapQueueScheduler.java 100644
View File

@ -0,0 +1,89 @@
package us.codecraft.webmagic.recover;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.lang3.StringUtils;
import org.mapdb.DB;
import org.mapdb.DBMaker;
import org.mapdb.IndexTreeList;
import org.mapdb.Serializer;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.scheduler.DuplicateRemovedScheduler;
import us.codecraft.webmagic.scheduler.component.DuplicateRemover;
import java.io.IOException;
/**
* @author ：linweisen
* @date ：Created in 2021/4/9 14:38
* @description：${description}
* @modified By：
* @version: 1.0
*/
public class MmapQueueScheduler extends DuplicateRemovedScheduler {
private DB db;
private static String DATABASE_NAME = "queue";
private IndexTreeList<String> queue;
private static ObjectMapper mapper;
public MmapQueueScheduler(DuplicateRemover duplicateRemover, String path) {
super.setDuplicateRemover(duplicateRemover);
String queuePath = path;
DB db = DBMaker.fileDB(queuePath)
.fileMmapEnableIfSupported()
.fileMmapPreclearDisable()
.cleanerHackEnable()
.closeOnJvmShutdown()
.transactionEnable()
.concurrencyScale(128)
.make();
this.db = db;
this.mapper = new ObjectMapper();
this.queue = db.indexTreeList(MmapQueueScheduler.DATABASE_NAME, Serializer.STRING).createOrOpen();
}
@Override
public Request poll(Task task) {
if (this.queue.size() > 0){
String s = queue.remove(0);
return fromJson(s, Request.class);
}else{
return null;
}
}
@Override
public void pushWhenNoDuplicate(Request request, Task task) {
queue.add(toJson(request));
this.db.commit();
}
public String toJson(Object object) {
try {
return mapper.writeValueAsString(object);
} catch (IOException e) {
logger.warn("write to json string error:" + object, e);
return null;
}
}
public <T> T fromJson(String jsonString, Class<T> clazz) {
if (StringUtils.isEmpty(jsonString)) {
return null;
}
try {
return mapper.readValue(jsonString, clazz);
} catch (IOException e) {
logger.warn("parse json string error:" + jsonString, e);
return null;
}
}
}

22
webmagic-samples/src/main/java/us/codecraft/webmagic/recover/RecoverSample.java 100644
View File

@ -0,0 +1,22 @@
package us.codecraft.webmagic.recover;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.samples.SinaBlogProcessor;
import us.codecraft.webmagic.scheduler.component.DuplicateRemover;
/**
* @author code4crafter@gmail.com <br>
*/
public class RecoverSample {
public static void main(String[] args) {
String storage = "queue";
String duplicate = "duplicate";
Spider spider = new Spider(new SinaBlogProcessor());
DuplicateRemover remover = new DuplicateStorageRemover(duplicate);
spider.setScheduler(new MmapQueueScheduler(remover, storage));
spider.addUrl("http://blog.sina.com.cn/s/articlelist_1487828712_0_1.html")
.run();
}
}