Merge pull request #1107 from hooyantsing/develop

修复 HtmlCleaner 无法正常解析 tr 和 td 标签的问题

webmagic-core/src/main/java/us/codecraft/webmagic/selector/BaseElementSelector.java

@@ -3,6 +3,7 @@ package us.codecraft.webmagic.selector;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
+import us.codecraft.webmagic.utils.BaseSelectorUtils;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -13,16 +14,9 @@ import java.util.List;
  */
 public abstract class BaseElementSelector implements Selector, ElementSelector {
     private Document parse(String text) {
-        if (text == null) {
-            return null;
-        }
-
         // Jsoup could not parse <tr></tr> or <td></td> tag directly
         // https://stackoverflow.com/questions/63607740/jsoup-couldnt-parse-tr-tag
-        if ((text.startsWith("<tr>") && text.endsWith("</tr>"))
+        text = BaseSelectorUtils.preParse(text);
-                || (text.startsWith("<td>") && text.endsWith("</td>"))) {
-            text = "<table>" + text + "</table>";
-        }
         return Jsoup.parse(text);
     }
 

webmagic-core/src/main/java/us/codecraft/webmagic/utils/BaseSelectorUtils.java

@@ -0,0 +1,23 @@
+package us.codecraft.webmagic.utils;
+
+/**
+ * @author hooy
+ */
+public class BaseSelectorUtils {
+
+    /**
+     * Jsoup/HtmlCleaner could not parse "tr" or "td" tag directly
+     * https://stackoverflow.com/questions/63607740/jsoup-couldnt-parse-tr-tag
+     *
+     * @param text - the html string
+     * @return text
+     */
+    public static String preParse(String text) {
+        if (((text.startsWith("<tr>") || text.startsWith("<tr ")) && text.endsWith("</tr>"))
+                || ((text.startsWith("<td>") || text.startsWith("<td ")) && text.endsWith("</td>"))) {
+            text = "<table>" + text + "</table>";
+        }
+        return text;
+    }
+
+}

webmagic-saxon/src/main/java/us/codecraft/webmagic/selector/Xpath2Selector.java

@@ -8,6 +8,7 @@ import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
 import javax.xml.namespace.NamespaceContext;
+import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.transform.OutputKeys;
 import javax.xml.transform.Transformer;
 import javax.xml.transform.TransformerFactory;
@@ -29,13 +30,14 @@ import org.w3c.dom.NodeList;
 
 import net.sf.saxon.lib.NamespaceConstant;
 import net.sf.saxon.xpath.XPathEvaluator;
+import us.codecraft.webmagic.utils.BaseSelectorUtils;
 
 /**
  * 支持xpath2.0的选择器。包装了HtmlCleaner和Saxon HE。<br>
  *
  * @author code4crafter@gmail.com <br>
- *         Date: 13-4-21
+ * Date: 13-4-21
- *         Time: 上午9:39
+ * Time: 上午9:39
  */
 public class Xpath2Selector implements Selector {
 
@@ -111,14 +113,11 @@ public class Xpath2Selector implements Selector {
     @Override
     public String select(String text) {
         try {
-            HtmlCleaner htmlCleaner = new HtmlCleaner();
-            TagNode tagNode = htmlCleaner.clean(text);
-            Document document = new DomSerializer(new CleanerProperties()).createDOM(tagNode);
             Object result;
             try {
-                result = xPathExpression.evaluate(document, XPathConstants.NODESET);
+                result = xPathExpression.evaluate(parse(text), XPathConstants.NODESET);
             } catch (XPathExpressionException e) {
-                result = xPathExpression.evaluate(document, XPathConstants.STRING);
+                result = xPathExpression.evaluate(parse(text), XPathConstants.STRING);
             }
             if (result instanceof NodeList) {
                 NodeList nodeList = (NodeList) result;
@@ -147,14 +146,11 @@ public class Xpath2Selector implements Selector {
     public List<String> selectList(String text) {
         List<String> results = new ArrayList<String>();
         try {
-            HtmlCleaner htmlCleaner = new HtmlCleaner();
-            TagNode tagNode = htmlCleaner.clean(text);
-            Document document = new DomSerializer(new CleanerProperties()).createDOM(tagNode);
             Object result;
             try {
-                result = xPathExpression.evaluate(document, XPathConstants.NODESET);
+                result = xPathExpression.evaluate(parse(text), XPathConstants.NODESET);
             } catch (XPathExpressionException e) {
-                result = xPathExpression.evaluate(document, XPathConstants.STRING);
+                result = xPathExpression.evaluate(parse(text), XPathConstants.STRING);
             }
             if (result instanceof NodeList) {
                 NodeList nodeList = (NodeList) result;
@@ -179,4 +175,12 @@ public class Xpath2Selector implements Selector {
         }
         return results;
     }
+
+    private Document parse(String text) throws ParserConfigurationException {
+        // HtmlCleaner could not parse <tr></tr> or <td></td> tag directly
+        text = BaseSelectorUtils.preParse(text);
+        HtmlCleaner htmlCleaner = new HtmlCleaner();
+        TagNode tagNode = htmlCleaner.clean(text);
+        return new DomSerializer(new CleanerProperties()).createDOM(tagNode);
+    }
 }

webmagic-saxon/src/test/java/us/codecraft/webmagic/selector/XpathSelectorTest.java

@@ -11,6 +11,9 @@ import org.junit.Assert;
 import org.junit.Ignore;
 import org.junit.Test;
 
+import us.codecraft.webmagic.Page;
+import us.codecraft.webmagic.Spider;
+import us.codecraft.webmagic.processor.PageProcessor;
 import us.codecraft.xsoup.XPathEvaluator;
 import us.codecraft.xsoup.Xsoup;
 
@@ -1385,35 +1388,52 @@ public class XpathSelectorTest {
         Assert.assertEquals("http://www.oschina.net/", selectList.get(0));
     }
 
+    @Ignore("test parse <table> <tr> <td> tag")
+    @Test
+    public void htmlCleanerParseTest() {
+        Spider.create(new RuoxiaPageProcessor()).addUrl("http://www.ruoxia.com/top/dianji/month").thread(1).run();
+    }
+
+    class RuoxiaPageProcessor implements PageProcessor {
+        @Override
+        public void process(Page page) {
+            List<String> items = new Xpath2Selector("//div[@class=\"bd\"]//tbody/tr").selectList(page.getRawText());
+            for (String item : items) {
+                String name = new Xpath2Selector("//td[3]/div/a[1]/text()").select(item);
+                System.out.println(name);
+            }
+        }
+    }
+
     @Ignore("take long time")
     @Test
     public void performanceTest() {
         Xpath2Selector xpath2Selector = new Xpath2Selector("//a");
-        long time =System.currentTimeMillis();
+        long time = System.currentTimeMillis();
         for (int i = 0; i < 1000; i++) {
             xpath2Selector.selectList(html);
         }
-        System.out.println(System.currentTimeMillis()-time);
+        System.out.println(System.currentTimeMillis() - time);
 
         XpathSelector xpathSelector = new XpathSelector("//a");
-        time =System.currentTimeMillis();
+        time = System.currentTimeMillis();
         for (int i = 0; i < 1000; i++) {
             xpathSelector.selectList(html);
         }
-        System.out.println(System.currentTimeMillis()-time);
+        System.out.println(System.currentTimeMillis() - time);
 
-        time =System.currentTimeMillis();
+        time = System.currentTimeMillis();
         for (int i = 0; i < 1000; i++) {
             xpath2Selector.selectList(html);
         }
         System.out.println(System.currentTimeMillis() - time);
 
         CssSelector cssSelector = new CssSelector("a");
-        time =System.currentTimeMillis();
+        time = System.currentTimeMillis();
         for (int i = 0; i < 1000; i++) {
             cssSelector.selectList(html);
         }
-        System.out.println("css "+(System.currentTimeMillis()-time));
+        System.out.println("css " + (System.currentTimeMillis() - time));
     }
 
     @Ignore("take long time")
@@ -1425,54 +1445,54 @@ public class XpathSelectorTest {
         TagNode tagNode = htmlCleaner.clean(html);
         Document document = Jsoup.parse(html);
 
-        long time =System.currentTimeMillis();
+        long time = System.currentTimeMillis();
         for (int i = 0; i < 2000; i++) {
             htmlCleaner.clean(html);
         }
-        System.out.println(System.currentTimeMillis()-time);
+        System.out.println(System.currentTimeMillis() - time);
 
-        time =System.currentTimeMillis();
+        time = System.currentTimeMillis();
         for (int i = 0; i < 2000; i++) {
             tagNode.evaluateXPath("//a");
         }
-        System.out.println(System.currentTimeMillis()-time);
+        System.out.println(System.currentTimeMillis() - time);
 
         System.out.println("=============");
 
-        time =System.currentTimeMillis();
+        time = System.currentTimeMillis();
         for (int i = 0; i < 2000; i++) {
             Jsoup.parse(html);
         }
-        System.out.println(System.currentTimeMillis()-time);
+        System.out.println(System.currentTimeMillis() - time);
 
-        time =System.currentTimeMillis();
+        time = System.currentTimeMillis();
         for (int i = 0; i < 2000; i++) {
             document.select("a");
         }
-        System.out.println(System.currentTimeMillis()-time);
+        System.out.println(System.currentTimeMillis() - time);
 
         System.out.println("=============");
 
-        time =System.currentTimeMillis();
+        time = System.currentTimeMillis();
         for (int i = 0; i < 2000; i++) {
             htmlCleaner.clean(html);
         }
-        System.out.println(System.currentTimeMillis()-time);
+        System.out.println(System.currentTimeMillis() - time);
 
-        time =System.currentTimeMillis();
+        time = System.currentTimeMillis();
         for (int i = 0; i < 2000; i++) {
             tagNode.evaluateXPath("//a");
         }
-        System.out.println(System.currentTimeMillis()-time);
+        System.out.println(System.currentTimeMillis() - time);
 
         System.out.println("=============");
 
         XPathEvaluator compile = Xsoup.compile("//a");
-        time =System.currentTimeMillis();
+        time = System.currentTimeMillis();
         for (int i = 0; i < 2000; i++) {
             compile.evaluate(document);
         }
-        System.out.println(System.currentTimeMillis()-time);
+        System.out.println(System.currentTimeMillis() - time);
 
     }