当page为null时,意味着非正常的响应状态,应该抛出异常,否则SpiderListener的onSuccess方法和onError方法都会执行
parent
c0b8e8f8ae
commit
5f106c9c69
|
@ -408,9 +408,7 @@ public class Spider implements Runnable, Task {
|
||||||
protected void processRequest(Request request) {
|
protected void processRequest(Request request) {
|
||||||
Page page = downloader.download(request, this);
|
Page page = downloader.download(request, this);
|
||||||
if (page == null) {
|
if (page == null) {
|
||||||
sleep(site.getRetrySleepTime());
|
throw new RuntimeException("unaccpetable response status");
|
||||||
onError(request);
|
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
// for cycle retry
|
// for cycle retry
|
||||||
if (page.isNeedCycleRetry()) {
|
if (page.isNeedCycleRetry()) {
|
||||||
|
|
Loading…
Reference in New Issue