webMagic-Mobai
/
magic-Dependency
mirror of https://gitea.qinmian.online/Mobai/magic-Dependency.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

remove duplicate code #421

master
yihua.huang 2017-02-25 12:01:12 +08:00
parent 42f1018010
commit 3a796b9413
1 changed files with 5 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
View File

@ -305,7 +305,7 @@ public class Spider implements Runnable, Task {
initComponent();
logger.info("Spider " + getUUID() + " started!");
while (!Thread.currentThread().isInterrupted() && stat.get() == STAT_RUNNING) {
Request request = scheduler.poll(this);
final Request request = scheduler.poll(this);
if (request == null) {
if (threadPool.getThreadAlive() == 0 && exitWhenComplete) {
break;
@ -313,16 +313,15 @@ public class Spider implements Runnable, Task {
// wait until new url added
waitNewUrl();
} else {
final Request requestFinal = request;
threadPool.execute(new Runnable() {
@Override
public void run() {
try {
processRequest(requestFinal);
onSuccess(requestFinal);
processRequest(request);
onSuccess(request);
} catch (Exception e) {
onError(requestFinal);
logger.error("process request " + requestFinal + " error", e);
onError(request);
logger.error("process request " + request + " error", e);
} finally {
pageCount.incrementAndGet();
signalNewUrl();