当page为null时,意味着非正常的响应状态,应该抛出异常,否则SpiderListener的onSuccess方法和onError方法都会执行
parent
c0b8e8f8ae
commit
5f106c9c69
|
@ -408,9 +408,7 @@ public class Spider implements Runnable, Task {
|
|||
protected void processRequest(Request request) {
|
||||
Page page = downloader.download(request, this);
|
||||
if (page == null) {
|
||||
sleep(site.getRetrySleepTime());
|
||||
onError(request);
|
||||
return;
|
||||
throw new RuntimeException("unaccpetable response status");
|
||||
}
|
||||
// for cycle retry
|
||||
if (page.isNeedCycleRetry()) {
|
||||
|
|
Loading…
Reference in New Issue