Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

remove duplicate code #421

master
yihua.huang 2017-02-25 12:01:12 +08:00
parent 42f1018010
commit 3a796b9413
1 changed files with 5 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
View File

@ -305,7 +305,7 @@ public class Spider implements Runnable, Task {
initComponent(); initComponent();
logger.info("Spider " + getUUID() + " started!"); logger.info("Spider " + getUUID() + " started!");
while (!Thread.currentThread().isInterrupted() && stat.get() == STAT_RUNNING) { while (!Thread.currentThread().isInterrupted() && stat.get() == STAT_RUNNING) {
Request request = scheduler.poll(this); final Request request = scheduler.poll(this);
if (request == null) { if (request == null) {
if (threadPool.getThreadAlive() == 0 && exitWhenComplete) { if (threadPool.getThreadAlive() == 0 && exitWhenComplete) {
break; break;
@ -313,16 +313,15 @@ public class Spider implements Runnable, Task {
// wait until new url added // wait until new url added
waitNewUrl(); waitNewUrl();
} else { } else {
final Request requestFinal = request;
threadPool.execute(new Runnable() { threadPool.execute(new Runnable() {
@Override @Override
public void run() { public void run() {
try { try {
processRequest(requestFinal); processRequest(request);
onSuccess(requestFinal); onSuccess(request);
} catch (Exception e) { } catch (Exception e) {
onError(requestFinal); onError(request);
logger.error("process request " + requestFinal + " error", e); logger.error("process request " + request + " error", e);
} finally { } finally {
pageCount.incrementAndGet(); pageCount.incrementAndGet();
signalNewUrl(); signalNewUrl();