diff options
| author | Michael Peter Christen <mc@yacy.net> | 2017-12-09 22:29:35 +0100 |
|---|---|---|
| committer | Michael Peter Christen <mc@yacy.net> | 2017-12-09 22:29:35 +0100 |
| commit | 25573bd5abf52c3bd4f483e83f47739f1ddb39ae (patch) | |
| tree | 94629e4419d3a641975e492a8f473704efeb23b8 /test | |
| parent | 607b39b427f76ec139df5d9d5479cf09a0d6fe4a (diff) | |
added a crawl filter based on <div> tag class names
When a crawl is started, a new field to exclude content from scraping is
available. The field can be identified with the class name of div tags.
All text contained in such a div tag where the configured class name(s)
match are not indexed, while the remaining page is indexed.
Diffstat (limited to 'test')
| -rw-r--r-- | test/java/net/yacy/document/parser/html/ContentScraperTest.java | 3 | ||||
| -rw-r--r-- | test/java/net/yacy/document/parser/htmlParserTest.java | 9 |
2 files changed, 7 insertions, 5 deletions
diff --git a/test/java/net/yacy/document/parser/html/ContentScraperTest.java b/test/java/net/yacy/document/parser/html/ContentScraperTest.java index 021141fe3..d5ab17b1a 100644 --- a/test/java/net/yacy/document/parser/html/ContentScraperTest.java +++ b/test/java/net/yacy/document/parser/html/ContentScraperTest.java @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; import java.util.Date; +import java.util.HashSet; import java.util.List; import java.util.Set; @@ -145,7 +146,7 @@ public class ContentScraperTest { + "<time datetime='2016-12-23'>23. Dezember 2016</time>" // html5 time tag + "</body></html>"; - ContentScraper scraper = new ContentScraper(root, 10, new VocabularyScraper(), 0); + ContentScraper scraper = new ContentScraper(root, 10, new HashSet<String>(), new VocabularyScraper(), 0); final Writer writer = new TransformerWriter(null, null, scraper, null, false); FileUtils.copy(new StringReader(page), writer); diff --git a/test/java/net/yacy/document/parser/htmlParserTest.java b/test/java/net/yacy/document/parser/htmlParserTest.java index e6b67bd35..4366d8c4b 100644 --- a/test/java/net/yacy/document/parser/htmlParserTest.java +++ b/test/java/net/yacy/document/parser/htmlParserTest.java @@ -10,6 +10,7 @@ import java.io.InputStream; import java.net.MalformedURLException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; +import java.util.HashSet; import java.util.List; import java.util.Locale; @@ -265,7 +266,7 @@ public class htmlParserTest extends TestCase { + "<figure><img width=\"550px\" title=\"image as exemple\" alt=\"image as exemple\" src=\"./img/my_image.png\"></figrue>" // + img width 550 (+html5 figure) + "</body></html>"; - ContentScraper scraper = parseToScraper(url, charset, new VocabularyScraper(), 0, testhtml, 10, 10); + ContentScraper scraper = parseToScraper(url, charset, new HashSet<String>(), new VocabularyScraper(), 0, testhtml, 10, 10); List<AnchorURL> anchorlist = scraper.getAnchors(); String linktxt = anchorlist.get(0).getTextProperty(); @@ -307,7 +308,7 @@ public class htmlParserTest extends TestCase { } testHtml.append("</p></body></html>"); - ContentScraper scraper = parseToScraper(url, charset, new VocabularyScraper(), 0, testHtml.toString(), Integer.MAX_VALUE, Integer.MAX_VALUE); + ContentScraper scraper = parseToScraper(url, charset, new HashSet<String>(), new VocabularyScraper(), 0, testHtml.toString(), Integer.MAX_VALUE, Integer.MAX_VALUE); assertEquals(nestingDepth, scraper.getAnchors().size()); assertEquals(1, scraper.getImages().size()); @@ -328,7 +329,7 @@ public class htmlParserTest extends TestCase { + "<p>" + textSource + "</p>" + "</body></html>"; - ContentScraper scraper = parseToScraper(url, charset, new VocabularyScraper(), 0, testhtml, 10, 10); + ContentScraper scraper = parseToScraper(url, charset, new HashSet<String>(), new VocabularyScraper(), 0, testhtml, 10, 10); String txt = scraper.getText(); System.out.println("ScraperTagTest: [" + textSource + "] = [" + txt + "]"); @@ -357,7 +358,7 @@ public class htmlParserTest extends TestCase { + "</head>\n" + "<body>" + textSource + "</body>\n" + "</html>"; - ContentScraper scraper = parseToScraper(url, charset, new VocabularyScraper(), 0, testhtml, 10, 10); + ContentScraper scraper = parseToScraper(url, charset, new HashSet<String>(), new VocabularyScraper(), 0, testhtml, 10, 10); String txt = scraper.getText(); System.out.println("ScraperScriptTagTest: [" + textSource + "] = [" + txt + "]"); |