将Spider.processRequest()抛出异常改回原来的逻辑
parent
97592d6720
commit
eeb607fd0e
|
@ -403,7 +403,9 @@ public class Spider implements Runnable, Task {
|
||||||
protected void processRequest(Request request) {
|
protected void processRequest(Request request) {
|
||||||
Page page = downloader.download(request, this);
|
Page page = downloader.download(request, this);
|
||||||
if (page == null) {
|
if (page == null) {
|
||||||
throw new RuntimeException("unaccpetable response status");
|
sleep(site.getSleepTime());
|
||||||
|
onError(request);
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
// for cycle retry
|
// for cycle retry
|
||||||
if (page.isNeedCycleRetry()) {
|
if (page.isNeedCycleRetry()) {
|
||||||
|
|
Loading…
Reference in New Issue