Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

when download error, throw an exception instead of calling onError and returning peacefully. #105

master
Bo LIANG 2014-04-17 17:53:12 +08:00
parent 023c2ac84e
commit 08fa3b01c1
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
View File

@ -1,9 +1,11 @@
package us.codecraft.webmagic; package us.codecraft.webmagic;
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.downloader.Downloader; import us.codecraft.webmagic.downloader.Downloader;
import us.codecraft.webmagic.downloader.HttpClientDownloader; import us.codecraft.webmagic.downloader.HttpClientDownloader;
import us.codecraft.webmagic.monitor.SpiderListener; import us.codecraft.webmagic.monitor.SpiderListener;
@ -318,7 +320,7 @@ public class Spider implements Runnable, Task {
onSuccess(requestFinal); onSuccess(requestFinal);
} catch (Exception e) { } catch (Exception e) {
onError(requestFinal); onError(requestFinal);
logger.error("download " + requestFinal + " error", e); logger.error("process request " + requestFinal + " error", e);
} finally { } finally {
threadAlive.decrementAndGet(); threadAlive.decrementAndGet();
pageCount.incrementAndGet(); pageCount.incrementAndGet();
@ -399,9 +401,8 @@ public class Spider implements Runnable, Task {
protected void processRequest(Request request) { protected void processRequest(Request request) {
Page page = downloader.download(request, this); Page page = downloader.download(request, this);
if (page == null) { if (page == null) {
onError(request);
sleep(site.getSleepTime()); sleep(site.getSleepTime());
return; throw new IllegalStateException("download error");
} }
// for cycle retry // for cycle retry
if (page.isNeedCycleRetry()) { if (page.isNeedCycleRetry()) {