Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

remove synchronize in Page #411

master
yihua.huang 2017-02-25 19:42:12 +08:00
parent 3a796b9413
commit ad69963005
1 changed files with 13 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
webmagic-core/src/main/java/us/codecraft/webmagic/Page.java
View File

@ -107,14 +107,12 @@ public class Page {
* @param requests requests * @param requests requests
*/ */
public void addTargetRequests(List<String> requests) { public void addTargetRequests(List<String> requests) {
synchronized (targetRequests) { for (String s : requests) {
for (String s : requests) { if (StringUtils.isBlank(s) || s.equals("#") || s.startsWith("javascript:")) {
if (StringUtils.isBlank(s) || s.equals("#") || s.startsWith("javascript:")) { continue;
continue;
}
s = UrlUtils.canonicalizeUrl(s, url.toString());
targetRequests.add(new Request(s));
} }
s = UrlUtils.canonicalizeUrl(s, url.toString());
targetRequests.add(new Request(s));
} }
} }
@ -125,14 +123,12 @@ public class Page {
* @param priority priority * @param priority priority
*/ */
public void addTargetRequests(List<String> requests, long priority) { public void addTargetRequests(List<String> requests, long priority) {
synchronized (targetRequests) { for (String s : requests) {
for (String s : requests) { if (StringUtils.isBlank(s) || s.equals("#") || s.startsWith("javascript:")) {
if (StringUtils.isBlank(s) || s.equals("#") || s.startsWith("javascript:")) { continue;
continue;
}
s = UrlUtils.canonicalizeUrl(s, url.toString());
targetRequests.add(new Request(s).setPriority(priority));
} }
s = UrlUtils.canonicalizeUrl(s, url.toString());
targetRequests.add(new Request(s).setPriority(priority));
} }
} }
@ -145,10 +141,8 @@ public class Page {
if (StringUtils.isBlank(requestString) || requestString.equals("#")) { if (StringUtils.isBlank(requestString) || requestString.equals("#")) {
return; return;
} }
synchronized (targetRequests) { requestString = UrlUtils.canonicalizeUrl(requestString, url.toString());
requestString = UrlUtils.canonicalizeUrl(requestString, url.toString()); targetRequests.add(new Request(requestString));
targetRequests.add(new Request(requestString));
}
} }
/** /**
@ -157,9 +151,7 @@ public class Page {
* @param request request * @param request request
*/ */
public void addTargetRequest(Request request) { public void addTargetRequest(Request request) {
synchronized (targetRequests) { targetRequests.add(request);
targetRequests.add(request);
}
} }
/** /**