From 58b98347296d6417d59c09efdc74b0695cb35ec8 Mon Sep 17 00:00:00 2001 From: luccioman Date: Fri, 2 Feb 2018 09:31:40 +0100 Subject: Added HTML microdata typed items parsing capability. This adds the possibility for the HTML parser to gather typed items URLs annotated in HTML tags with itemscope and itemtype attributes (see microdata specification https://www.w3.org/TR/microdata/ ), notably Types from the schema.org vocabulary, but also Types/Classes from any other vocabulary, such as the common ones listed in the RDFa core context ( https://www.w3.org/2011/rdfa-context/rdfa-1.1.html ). --- .../document/parser/html/ContentScraperTest.java | 78 ++++++++++++++++++++++ 1 file changed, 78 insertions(+) (limited to 'test') diff --git a/test/java/net/yacy/document/parser/html/ContentScraperTest.java b/test/java/net/yacy/document/parser/html/ContentScraperTest.java index d5ab17b1a..b430f58f3 100644 --- a/test/java/net/yacy/document/parser/html/ContentScraperTest.java +++ b/test/java/net/yacy/document/parser/html/ContentScraperTest.java @@ -29,8 +29,11 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.Collection; import java.util.Date; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import org.junit.Assert; @@ -338,5 +341,80 @@ public class ContentScraperTest { Assert.assertEquals("{abc}{def}", ContentScraper.removeUnpairedBrackets("{abc}{def}", '{', '}')); Assert.assertEquals("{{abc}{def}}", ContentScraper.removeUnpairedBrackets("{{abc}{def}}", '{', '}')); } + + /** + * Test microdata itemtype attribute parsing + * @throws IOException + */ + @Test + public void testParseMicroDataItemType() throws IOException { + final String htmlHeader = "Test document"; + final DigestURL docUrl = new DigestURL("http://example.org/microdata.html"); + + + final Map html2Results = new HashMap<>(); + /* Basic microdata syntax example with no item type */ + String html = htmlHeader + "

My name is Elizabeth.

"; + String[] expectedUrls = {}; + html2Results.put(html, expectedUrls); + + /* Nested items with no item type */ + html = "
\n" + + "

Name: Amanda

\n" + + "

Band: Jazz Band (12 players)

\n" + + "
"; + expectedUrls = new String[0]; + html2Results.put(html, expectedUrls); + + /* One typed item */ + html = htmlHeader + "
\"Our
"; + expectedUrls = new String[]{"https://schema.org/LocalBusiness"}; + html2Results.put(html, expectedUrls); + + /* more than one type per item */ + html = htmlHeader + "
" + + "
Name:\n" + + "
Tank Locomotive (DB 80)\n" + + "
Product code:\n" + + "
33041\n" + + "
Scale:\n" + + "
HO\n" + + "
Digital:\n" + + "
Delta\n" + + "
"; + expectedUrls = new String[]{"https://md.example.com/loco", "https://md.example.com/lighting"}; + html2Results.put(html, expectedUrls); + + /* Nested typed items */ + html = htmlHeader + "
\n" + + " Panasonic White 60L Refrigerator\n" + + " \"\"\n" + + "
\n" + + " Rated 3.5/5\n" + + " (based on 11 customer reviews)\n" + + "
\n" + + "
"; + expectedUrls = new String[]{"http://schema.org/Product", "http://schema.org/AggregateRating"}; + html2Results.put(html, expectedUrls); + + + for (final Entry html2Result : html2Results.entrySet()) { + ContentScraper scraper = new ContentScraper(docUrl, 10, new HashSet(), new VocabularyScraper(), 0); + try (final Writer writer = new TransformerWriter(null, null, scraper, null, false)) { + FileUtils.copy(new StringReader(html2Result.getKey()), writer); + + final Set expected = new HashSet<>(); + for (final String url : html2Result.getValue()) { + expected.add(new DigestURL(url)); + } + + Assert.assertEquals(expected.size(), scraper.getLinkedDataTypes().size()); + Assert.assertTrue(expected.containsAll(scraper.getLinkedDataTypes())); + } finally { + scraper.close(); + } + } + } } -- cgit v1.2.3