Mobai
/
magic-Dependency
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

#38 Change algorithm of SmartContentSelector

master
yihua.huang 2013-11-23 13:56:55 +08:00
parent 296a68920e
commit 04fcf3193f
2 changed files with 61 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/OschinaBlogPageProcesser.java
View File

@ -23,7 +23,7 @@ public class OschinaBlogPageProcesser implements PageProcessor {
//skip this page //skip this page
page.setSkip(true); page.setSkip(true);
} }
page.putField("content", page.getHtml().xpath("//div[@class='BlogContent']/tidyText()").toString()); page.putField("content", page.getHtml().smartContent().toString());
page.putField("tags", page.getHtml().xpath("//div[@class='BlogTags']/a/text()").all()); page.putField("tags", page.getHtml().xpath("//div[@class='BlogTags']/a/text()").all());
} }

131
webmagic-core/src/main/java/us/codecraft/webmagic/selector/SmartContentSelector.java
View File

@ -1,100 +1,87 @@
package us.codecraft.webmagic.selector; package us.codecraft.webmagic.selector;
import org.apache.log4j.Logger;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import us.codecraft.webmagic.utils.Experimental; import us.codecraft.webmagic.utils.Experimental;
import java.util.*; import java.util.ArrayList;
import java.util.concurrent.atomic.AtomicInteger; import java.util.Arrays;
import java.util.List;
/** /**
* Extract the text content of html.<br> * Borrowed from https://code.google.com/p/cx-extractor/
* Using Readability algorithm: find parents of all p tags.
* *
* @author code4crafter@gmail.com <br> * @author code4crafter@gmail.com <br>
* @since 0.1.0 * @since 0.4.1
*
*/ */
@Experimental @Experimental
public class SmartContentSelector implements Selector { public class SmartContentSelector implements Selector {
private Logger logger = Logger.getLogger(getClass());
public SmartContentSelector() { public SmartContentSelector() {
} }
@Override @Override
public String select(String text) { public String select(String html) {
HtmlCleaner htmlCleaner = new HtmlCleaner(); html = html.replaceAll("(?is)<!DOCTYPE.*?>", "");
TagNode tagNode = htmlCleaner.clean(text); html = html.replaceAll("(?is)<!--.*?-->", ""); // remove html comment
if (tagNode == null) { html = html.replaceAll("(?is)<script.*?>.*?</script>", ""); // remove javascript
return null; html = html.replaceAll("(?is)<style.*?>.*?</style>", ""); // remove css
html = html.replaceAll("&.{2,5};|&#.{2,5};", " "); // remove special char
html = html.replaceAll("(?is)<.*?>", "");
List<String> lines;
int blocksWidth =3;
int threshold =86;
int start;
int end;
StringBuilder text = new StringBuilder();
ArrayList<Integer> indexDistribution = new ArrayList<Integer>();
lines = Arrays.asList(html.split("\n"));
for (int i = 0; i < lines.size() - blocksWidth; i++) {
int wordsNum = 0;
for (int j = i; j < i + blocksWidth; j++) {
lines.set(j, lines.get(j).replaceAll("\\s+", ""));
wordsNum += lines.get(j).length();
} }
TagNode[] nodes = tagNode.getElementsByName("p", true); indexDistribution.add(wordsNum);
TagNode[] pres = tagNode.getElementsByName("pre", true);
Map<TagNode, Double> pDensityCountMap = new HashMap<TagNode, Double>();
countPdensity(nodes, pDensityCountMap);
countPdensity(pres, pDensityCountMap);
for (TagNode pre : pres) {
addCounter(pre, pDensityCountMap, 2);
}
List<Map.Entry<TagNode, Double>> sortList = new ArrayList<Map.Entry<TagNode, Double>>();
if (pDensityCountMap.size() == 0) {
return null;
}
for (Map.Entry<TagNode, Double> entry : pDensityCountMap.entrySet()) {
// if (logger.isDebugEnabled()) {
// logger.debug("p\t" + entry.getKey().getName() + "#" + entry.getKey().getAttributeByName("id") +
// "@" + entry.getKey().getAttributeByName("class") + ":" + entry.getValue());
// }
sortList.add(entry);
} }
Collections.sort(sortList, new Comparator<Map.Entry<TagNode, Double>>() { start = -1; end = -1;
@Override boolean boolstart = false, boolend = false;
public int compare(Map.Entry<TagNode, Double> o1, Map.Entry<TagNode, Double> o2) { text.setLength(0);
Double d1 = o1.getValue();
Double d2 = o2.getValue();
return -d1.compareTo(d2);
}
});
TagNode contentNode = sortList.get(0).getKey();
if (logger.isDebugEnabled()) {
logger.debug("p\t" + contentNode.getName() + "#" + contentNode.getAttributeByName("id") +
"@" + contentNode.getAttributeByName("class"));
}
return htmlCleaner.getInnerHtml(contentNode);
}
private void addCounter(TagNode node, Map<TagNode, Double> countMap, double delta) { for (int i = 0; i < indexDistribution.size() - 1; i++) {
Double counter = countMap.get(node); if (indexDistribution.get(i) > threshold && ! boolstart) {
if (counter == null) { if (indexDistribution.get(i+1).intValue() != 0
countMap.put(node, delta); || indexDistribution.get(i+2).intValue() != 0
} else { || indexDistribution.get(i+3).intValue() != 0) {
countMap.put(node, counter + delta); boolstart = true;
} start = i;
}
private static final double parentWeight = 0.7;
private void countPdensity(TagNode[] nodes, Map<TagNode, Double> pDensityCountMap) {
for (TagNode node : nodes) {
if (node == null) {
continue; continue;
} }
TagNode parent = node.getParent(); }
double pDensity = 1; if (boolstart) {
while (parent != null) { if (indexDistribution.get(i).intValue() == 0
addCounter(parent, pDensityCountMap, pDensity); || indexDistribution.get(i+1).intValue() == 0) {
parent = parent.getParent(); end = i;
pDensity = pDensity * parentWeight; boolend = true;
} }
} }
StringBuilder tmp = new StringBuilder();
if (boolend) {
//System.out.println(start+1 + "\t\t" + end+1);
for (int ii = start; ii <= end; ii++) {
if (lines.get(ii).length() < 5) continue;
tmp.append(lines.get(ii) + "\n");
} }
String str = tmp.toString();
private TagNode findLowestCommonParent(List<TagNode> tagNodes, int maxMargin, Map<TagNode, AtomicInteger> countMap) { //System.out.println(str);
TagNode contentNode = tagNodes.get(0); if (str.contains("Copyright") ) continue;
return contentNode; text.append(str);
boolstart = boolend = false;
}
}
return text.toString();
} }
@Override @Override