Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

more error log on page code error #601

master
yihua.huang 2017-06-17 10:36:41 +08:00
parent 6bd1eed25e
commit 4111b07263
2 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
webmagic-core/src/main/java/us/codecraft/webmagic/Spider.java
View File

@ -418,6 +418,8 @@ public class Spider implements Runnable, Task {
pipeline.process(page.getResultItems(), this);
}
}
} else {
logger.info("page status code error, page {} , code: {}", request.getUrl(), page.getStatusCode());
}
sleep(site.getSleepTime());
return;

3
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpClientDownloader.java
View File

@ -76,7 +76,6 @@ public class HttpClientDownloader extends AbstractDownloader {
if (task == null || task.getSite() == null) {
throw new NullPointerException("task or site can not be null");
}
logger.debug("downloading page {}", request.getUrl());
CloseableHttpResponse httpResponse = null;
CloseableHttpClient httpClient = getHttpClient(task.getSite());
Proxy proxy = proxyProvider != null ? proxyProvider.getProxy(task) : null;
@ -86,7 +85,7 @@ public class HttpClientDownloader extends AbstractDownloader {
httpResponse = httpClient.execute(requestContext.getHttpUriRequest(), requestContext.getHttpClientContext());
page = handleResponse(request, task.getSite().getCharset(), httpResponse, task);
onSuccess(request);
logger.debug("downloading page success {}", page);
logger.info("downloading page success {}", request.getUrl());
return page;
} catch (IOException e) {
logger.warn("download page {} error", request.getUrl(), e);