webMagic-Mobai
/
magic-Dependency
mirror of https://gitea.qinmian.online/Mobai/magic-Dependency.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactored Code to Resolve Implementation Code Smells (#1151) 

* Initial Commit

* Assignment 1 Submission

* Resolving Implementation Smells
master
ayushi250317 2024-03-28 13:45:12 -03:00 committed by GitHub
parent 80842d72db
commit 28ac8bf9c4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 60 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
webmagic-core/src/main/java/us/codecraft/webmagic/downloader/HttpUriRequestConverter.java
View File

@ -42,7 +42,9 @@ public class HttpUriRequestConverter {
HttpClientContext httpContext = new HttpClientContext(); HttpClientContext httpContext = new HttpClientContext();
if (proxy != null && proxy.getUsername() != null) { if (proxy != null && proxy.getUsername() != null) {
AuthState authState = new AuthState(); AuthState authState = new AuthState();
authState.update(new BasicScheme(ChallengeState.PROXY), new UsernamePasswordCredentials(proxy.getUsername(), proxy.getPassword())); BasicScheme proxyAuthScheme = new BasicScheme(ChallengeState.PROXY);
UsernamePasswordCredentials proxyCredentials = new UsernamePasswordCredentials(proxy.getUsername(), proxy.getPassword());
authState.update(proxyAuthScheme, proxyCredentials);
httpContext.setAttribute(HttpClientContext.PROXY_AUTH_STATE, authState); httpContext.setAttribute(HttpClientContext.PROXY_AUTH_STATE, authState);
} }
if (request.getCookies() != null && !request.getCookies().isEmpty()) { if (request.getCookies() != null && !request.getCookies().isEmpty()) {

104
webmagic-extension/src/main/java/us/codecraft/webmagic/model/PageModelExtractor.java
View File

@ -234,63 +234,23 @@ class PageModelExtractor {
o = clazz.newInstance(); o = clazz.newInstance();
for (FieldExtractor fieldExtractor : fieldExtractors) { for (FieldExtractor fieldExtractor : fieldExtractors) {
if (fieldExtractor.isMulti()) { if (fieldExtractor.isMulti()) {
List<String> value; List<String> value=getMultiValueFromSource(page, fieldExtractor, html, isRaw);
switch (fieldExtractor.getSource()) {
case RawHtml:
value = page.getHtml().selectDocumentForList(fieldExtractor.getSelector());
break;
case Html:
if (isRaw) {
value = page.getHtml().selectDocumentForList(fieldExtractor.getSelector());
} else {
value = fieldExtractor.getSelector().selectList(html);
}
break;
case Url:
value = fieldExtractor.getSelector().selectList(page.getUrl().toString());
break;
case RawText:
value = fieldExtractor.getSelector().selectList(page.getRawText());
break;
default:
value = fieldExtractor.getSelector().selectList(html);
}
if ((value == null || value.size() == 0) && fieldExtractor.isNotNull()) { if ((value == null || value.size() == 0) && fieldExtractor.isNotNull()) {
return null; return null;
} }
if (fieldExtractor.getObjectFormatter() != null) { if (fieldExtractor.getObjectFormatter() != null) {
List<Object> converted = convert(value, fieldExtractor.getObjectFormatter()); List<Object> converted = convertMultiValue(value, fieldExtractor.getObjectFormatter());
setField(o, fieldExtractor, converted); setField(o, fieldExtractor, converted);
} else { } else {
setField(o, fieldExtractor, value); setField(o, fieldExtractor, value);
} }
} else { } else {
String value; String value=getSingleValueFromSource(page, fieldExtractor, html, isRaw);
switch (fieldExtractor.getSource()) {
case RawHtml:
value = page.getHtml().selectDocument(fieldExtractor.getSelector());
break;
case Html:
if (isRaw) {
value = page.getHtml().selectDocument(fieldExtractor.getSelector());
} else {
value = fieldExtractor.getSelector().select(html);
}
break;
case Url:
value = fieldExtractor.getSelector().select(page.getUrl().toString());
break;
case RawText:
value = fieldExtractor.getSelector().select(page.getRawText());
break;
default:
value = fieldExtractor.getSelector().select(html);
}
if (value == null && fieldExtractor.isNotNull()) { if (value == null && fieldExtractor.isNotNull()) {
return null; return null;
} }
if (fieldExtractor.getObjectFormatter() != null) { if (fieldExtractor.getObjectFormatter() != null) {
Object converted = convert(value, fieldExtractor.getObjectFormatter()); Object converted = convertSingleValue(value, fieldExtractor.getObjectFormatter());
if (converted == null && fieldExtractor.isNotNull()) { if (converted == null && fieldExtractor.isNotNull()) {
return null; return null;
} }
@ -313,7 +273,57 @@ class PageModelExtractor {
return o; return o;
} }
private Object convert(String value, ObjectFormatter objectFormatter) { private List<String> getMultiValueFromSource(Page page, FieldExtractor fieldExtractor, String html, boolean isRaw) {
List<String> value;
switch (fieldExtractor.getSource()) {
case RawHtml:
value = page.getHtml().selectDocumentForList(fieldExtractor.getSelector());
break;
case Html:
if (isRaw) {
value = page.getHtml().selectDocumentForList(fieldExtractor.getSelector());
} else {
value = fieldExtractor.getSelector().selectList(html);
}
break;
case Url:
value = fieldExtractor.getSelector().selectList(page.getUrl().toString());
break;
case RawText:
value = fieldExtractor.getSelector().selectList(page.getRawText());
break;
default:
value = fieldExtractor.getSelector().selectList(html);
}
return value;
}
private String getSingleValueFromSource(Page page, FieldExtractor fieldExtractor, String html, boolean isRaw) {
String value;
switch (fieldExtractor.getSource()) {
case RawHtml:
value = page.getHtml().selectDocument(fieldExtractor.getSelector());
break;
case Html:
if (isRaw) {
value = page.getHtml().selectDocument(fieldExtractor.getSelector());
} else {
value = fieldExtractor.getSelector().select(html);
}
break;
case Url:
value = fieldExtractor.getSelector().select(page.getUrl().toString());
break;
case RawText:
value = fieldExtractor.getSelector().select(page.getRawText());
break;
default:
value = fieldExtractor.getSelector().select(html);
}
return value;
}
private Object convertSingleValue(String value, ObjectFormatter objectFormatter) {
try { try {
Object format = objectFormatter.format(value); Object format = objectFormatter.format(value);
logger.debug("String {} is converted to {}", value, format); logger.debug("String {} is converted to {}", value, format);
@ -324,10 +334,10 @@ class PageModelExtractor {
return null; return null;
} }
private List<Object> convert(List<String> values, ObjectFormatter objectFormatter) { private List<Object> convertMultiValue(List<String> values, ObjectFormatter objectFormatter) {
List<Object> objects = new ArrayList<Object>(); List<Object> objects = new ArrayList<Object>();
for (String value : values) { for (String value : values) {
Object converted = convert(value, objectFormatter); Object converted = convertSingleValue(value, objectFormatter);
if (converted != null) { if (converted != null) {
objects.add(converted); objects.add(converted);
} }