Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

#34 Close reader in FileCacheQueueScheduler

master
yihua.huang 2013-11-08 14:59:09 +08:00
parent d48ccc6740
commit b838c4e433
3 changed files with 39 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
release-note.md
View File

@ -1,6 +1,6 @@
Release Notes Release Notes
---- ----
See old versions in [https://github.com/code4craft/webmagic/releases](https://github.com/code4craft/webmagic/releases) See latest versions in [https://github.com/code4craft/webmagic/releases](https://github.com/code4craft/webmagic/releases)
*2012-9-4* `version：0.3.0` *2012-9-4* `version：0.3.0`

8
webmagic-extension/src/main/java/us/codecraft/webmagic/example/BaiduBaike.java
View File

@ -46,4 +46,12 @@ public class BaiduBaike{
} }
ooSpider.close(); ooSpider.close();
} }
public String getName() {
return name;
}
public String getDescription() {
return description;
}
} }

22
webmagic-extension/src/main/java/us/codecraft/webmagic/scheduler/FileCacheQueueScheduler.java
View File

@ -1,5 +1,6 @@
package us.codecraft.webmagic.scheduler; package us.codecraft.webmagic.scheduler;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.math.NumberUtils; import org.apache.commons.lang3.math.NumberUtils;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import us.codecraft.webmagic.Request; import us.codecraft.webmagic.Request;
@ -94,6 +95,9 @@ public class FileCacheQueueScheduler implements Scheduler {
urls = new LinkedHashSet<String>(); urls = new LinkedHashSet<String>();
readCursorFile(); readCursorFile();
readUrlFile(); readUrlFile();
} catch (FileNotFoundException e) {
//init
logger.info("init cache file " + getFileName(fileUrlAllName));
} catch (IOException e) { } catch (IOException e) {
logger.error("init file error", e); logger.error("init file error", e);
} }
@ -101,7 +105,9 @@ public class FileCacheQueueScheduler implements Scheduler {
private void readUrlFile() throws IOException { private void readUrlFile() throws IOException {
String line; String line;
BufferedReader fileUrlReader = new BufferedReader(new FileReader(getFileName(fileUrlAllName))); BufferedReader fileUrlReader = null;
try {
fileUrlReader = new BufferedReader(new FileReader(getFileName(fileUrlAllName)));
int lineReaded = 0; int lineReaded = 0;
while ((line = fileUrlReader.readLine()) != null) { while ((line = fileUrlReader.readLine()) != null) {
urls.add(line.trim()); urls.add(line.trim());
@ -110,15 +116,27 @@ public class FileCacheQueueScheduler implements Scheduler {
queue.add(new Request(line)); queue.add(new Request(line));
} }
} }
} finally {
if (fileUrlReader != null) {
IOUtils.closeQuietly(fileUrlReader);
}
}
} }
private void readCursorFile() throws IOException { private void readCursorFile() throws IOException {
BufferedReader fileCursorReader = new BufferedReader(new FileReader(getFileName(fileCursor))); BufferedReader fileCursorReader = null;
try {
new BufferedReader(new FileReader(getFileName(fileCursor)));
String line; String line;
//read the last number //read the last number
while ((line = fileCursorReader.readLine()) != null) { while ((line = fileCursorReader.readLine()) != null) {
cursor = new AtomicInteger(NumberUtils.toInt(line)); cursor = new AtomicInteger(NumberUtils.toInt(line));
} }
} finally {
if (fileCursorReader != null) {
IOUtils.closeQuietly(fileCursorReader);
}
}
} }
private String getFileName(String filename) { private String getFileName(String filename) {