From eb20589e29747949c2cfdb47a79a7c7d26f887ac Mon Sep 17 00:00:00 2001 From: luccioman Date: Sat, 10 Feb 2018 11:56:28 +0100 Subject: Fixed issue #158 : completed div CSS class ignore in crawl --- .../net/yacy/document/parser/htmlParserTest.java | 102 +++++++++++++++++++++ 1 file changed, 102 insertions(+) (limited to 'test/java') diff --git a/test/java/net/yacy/document/parser/htmlParserTest.java b/test/java/net/yacy/document/parser/htmlParserTest.java index 4366d8c4b..5c4b62c28 100644 --- a/test/java/net/yacy/document/parser/htmlParserTest.java +++ b/test/java/net/yacy/document/parser/htmlParserTest.java @@ -13,6 +13,7 @@ import java.nio.charset.StandardCharsets; import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Set; import org.junit.Test; @@ -138,6 +139,107 @@ public class htmlParserTest extends TestCase { } } } + + /** + * Test the htmlParser.parse() method, when filtering out div elements on their CSS class. + * + * @throws Exception + * when an unexpected error occurred + */ + @Test + public void testParseHtmlDivClassFilter() throws Exception { + final AnchorURL url = new AnchorURL("http://localhost/test.html"); + final String mimetype = "text/html"; + final StringBuilder testHtml = new StringBuilder("Test document"); + + testHtml.append("
Top text"); + testHtml.append("Top link"); + testHtml.append("
"); + + testHtml.append("
Some optional content"); + testHtml.append("Link from optional block"); + testHtml.append("
"); + + testHtml.append("

A paragraph

"); + + testHtml.append("
Text-only optional block
"); + + testHtml.append("
"); + testHtml.append("
"); + testHtml.append("
"); + testHtml.append("

Child text at depth 3

"); + testHtml.append("
"); + + testHtml.append("
\"Our
"); + + final htmlParser parser = new htmlParser(); + + /* No CSS class filter */ + try (InputStream sourceStream = new ByteArrayInputStream( + testHtml.toString().getBytes(StandardCharsets.UTF_8));) { + final Document[] docs = parser.parse(url, mimetype, null, new VocabularyScraper(), 0, sourceStream); + final Document doc = docs[0]; + final String parsedDext = doc.getTextString(); + + /* Check everything has been parsed */ + assertEquals(2, doc.getAnchors().size()); + assertEquals(1, doc.getImages().size()); + assertEquals(1, doc.getLinkedDataTypes().size()); + assertTrue(parsedDext.contains("Top")); + assertTrue(parsedDext.contains("Some")); + assertTrue(parsedDext.contains("from")); + assertTrue(parsedDext.contains("paragraph")); + assertTrue(parsedDext.contains("Text-only")); + assertTrue(parsedDext.contains("depth")); + } + + /* Filter on CSS classes with no matching elements */ + try (InputStream sourceStream = new ByteArrayInputStream( + testHtml.toString().getBytes(StandardCharsets.UTF_8));) { + final Set ignore = new HashSet<>(); + ignore.add("opt"); + ignore.add("head"); + ignore.add("container"); + final Document[] docs = parser.parse(url, mimetype, null, new VocabularyScraper(), 0, sourceStream); + final Document doc = docs[0]; + final String parsedDext = doc.getTextString(); + + /* Check everything has been parsed */ + assertEquals(2, doc.getAnchors().size()); + assertEquals(1, doc.getImages().size()); + assertEquals(1, doc.getLinkedDataTypes().size()); + assertTrue(parsedDext.contains("Top")); + assertTrue(parsedDext.contains("Some")); + assertTrue(parsedDext.contains("from")); + assertTrue(parsedDext.contains("paragraph")); + assertTrue(parsedDext.contains("Text-only")); + assertTrue(parsedDext.contains("depth")); + } + + /* Filter on CSS class with matching elements */ + try (InputStream sourceStream = new ByteArrayInputStream( + testHtml.toString().getBytes(StandardCharsets.UTF_8));) { + final Set ignore = new HashSet<>(); + ignore.add("optional"); + final Document[] docs = parser.parse(url, mimetype, null, ignore, new VocabularyScraper(), 0, sourceStream); + final Document doc = docs[0]; + final String parsedDext = doc.getTextString(); + + /* Check matching blocks have been ignored */ + assertEquals(1, doc.getAnchors().size()); + assertEquals("http://localhost/top.html", doc.getAnchors().iterator().next().toString()); + assertEquals(0, doc.getLinkedDataTypes().size()); + assertEquals(0, doc.getImages().size()); + assertFalse(parsedDext.contains("Some")); + assertFalse(parsedDext.contains("from")); + assertFalse(parsedDext.contains("depth")); + + /* Check non-matching blocks have been normally parsed */ + assertTrue(parsedDext.contains("Top")); + assertTrue(parsedDext.contains("Text-only")); + assertTrue(parsedDext.contains("paragraph")); + } + } /** * Test the htmlParser.parseWithLimits() method with test content within bounds. -- cgit v1.2.3