Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor addTargetRequests method to eliminate redundant code. (#1138)

master
Harikrishna 2023-11-26 08:26:06 +05:30 committed by GitHub
parent a911104076
commit 7c20290ce4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 27 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
webmagic-core/src/main/java/us/codecraft/webmagic/Page.java
View File

@ -49,7 +49,7 @@ public class Page {
private byte[] bytes; private byte[] bytes;
private List<Request> targetRequests = new ArrayList<Request>(); private List<Request> targetRequests = new ArrayList<>();
private String charset; private String charset;
@ -142,13 +142,7 @@ public class Page {
* @param requests requests * @param requests requests
*/ */
public void addTargetRequests(Iterable<String> requests) { public void addTargetRequests(Iterable<String> requests) {
for (String s : requests) { addTargetRequests(requests, 0); // Default priority is 0
if (StringUtils.isBlank(s) || s.equals("#") || s.startsWith("javascript:")) {
continue;
}
s = UrlUtils.canonicalizeUrl(s, url.toString());
targetRequests.add(new Request(s));
}
} }
/** /**
@ -158,14 +152,33 @@ public class Page {
* @param priority priority * @param priority priority
*/ */
public void addTargetRequests(Iterable<String> requests, long priority) { public void addTargetRequests(Iterable<String> requests, long priority) {
for (String s : requests) { if(requests == null) {
if (StringUtils.isBlank(s) || s.equals("#") || s.startsWith("javascript:")) { return;
continue; }
}
s = UrlUtils.canonicalizeUrl(s, url.toString()); for (String req : requests) {
targetRequests.add(new Request(s).setPriority(priority)); addRequestIfValid(req, priority);
} }
} }
/**
* Helper method to add a request if it's valid.
*
* @param url URL to add
* @param priority Priority for the URL
*/
private void addRequestIfValid(String url, long priority) {
if (StringUtils.isBlank(url) || url.equals("#") || url.startsWith("javascript:")) {
return;
}
String canonicalizedUrl = UrlUtils.canonicalizeUrl(url, this.url.toString());
Request req = new Request(canonicalizedUrl);
if(priority > 0) {
req.setPriority(priority);
}
targetRequests.add(req);
}
/** /**
* add url to fetch * add url to fetch