From 3fb449b3b6d977953d0f0757ad42da9006cebd51 Mon Sep 17 00:00:00 2001 From: luccioman Date: Thu, 6 Dec 2018 20:18:00 +0100 Subject: Properly resolve relative URLs against document URL in html base tags Fixes issue #256 --- .../document/parser/html/ContentScraperTest.java | 100 +++++++++++++++++++++ 1 file changed, 100 insertions(+) (limited to 'test/java') diff --git a/test/java/net/yacy/document/parser/html/ContentScraperTest.java b/test/java/net/yacy/document/parser/html/ContentScraperTest.java index 236824fa1..1990a2fcd 100644 --- a/test/java/net/yacy/document/parser/html/ContentScraperTest.java +++ b/test/java/net/yacy/document/parser/html/ContentScraperTest.java @@ -342,6 +342,106 @@ public class ContentScraperTest { Assert.assertEquals("{{abc}{def}}", ContentScraper.removeUnpairedBrackets("{{abc}{def}}", '{', '}')); } + /** + * Test base tag URL resolution + * @throws IOException when an unexpected error occurred + */ + @Test + public void testBaseTagUrlResolution() throws IOException { + final String htmlHeaderBeginning = "Test document"; + final DigestURL docUrl = new DigestURL("http://example.org/parent/base.html"); + + final String htmlLinksList = ""; + + + final Map html2Results = new HashMap<>(); + /* No base tag */ + String html = htmlHeaderBeginning + "" + htmlLinksList; + String[] expectedUrls = { "http://example.org/sameDomain/absolute.html", + "http://localhost/otherDomain/absolute.html", "http://example.org/sameDomain/scheme-relative.html", + "http://example.net/otherDomain/scheme-relative.html", "http://example.org/path/absolute.html", + "http://example.org/parent/path/relative/schemeless.html" }; + html2Results.put(html, expectedUrls); + + /* Base with absolute href on same domain */ + html = htmlHeaderBeginning + "" + + "" + htmlLinksList; + expectedUrls = new String[]{ "http://example.org/sameDomain/absolute.html", + "http://localhost/otherDomain/absolute.html", "http://example.org/sameDomain/scheme-relative.html", + "http://example.net/otherDomain/scheme-relative.html", "http://example.org/path/absolute.html", + "http://example.org/base/path/relative/schemeless.html" }; + html2Results.put(html, expectedUrls); + + /* Base with absolute href on another domain */ + html = htmlHeaderBeginning + "" + + "" + htmlLinksList; + expectedUrls = new String[]{ "http://example.org/sameDomain/absolute.html", + "http://localhost/otherDomain/absolute.html", "http://example.org/sameDomain/scheme-relative.html", + "http://example.net/otherDomain/scheme-relative.html", "http://example.net/path/absolute.html", + "http://example.net/base/path/relative/schemeless.html" }; + html2Results.put(html, expectedUrls); + + /* Base with scheme-relative href on same domain */ + html = htmlHeaderBeginning + "" + + "" + htmlLinksList; + expectedUrls = new String[]{ "http://example.org/sameDomain/absolute.html", + "http://localhost/otherDomain/absolute.html", "http://example.org/sameDomain/scheme-relative.html", + "http://example.net/otherDomain/scheme-relative.html", "http://example.org/path/absolute.html", + "http://example.org/base/path/relative/schemeless.html" }; + html2Results.put(html, expectedUrls); + + /* Base with scheme-relative href on another domain */ + html = htmlHeaderBeginning + "" + + "" + htmlLinksList; + expectedUrls = new String[]{ "http://example.org/sameDomain/absolute.html", + "http://localhost/otherDomain/absolute.html", "http://example.org/sameDomain/scheme-relative.html", + "http://example.net/otherDomain/scheme-relative.html", "http://example.net/path/absolute.html", + "http://example.net/base/path/relative/schemeless.html" }; + html2Results.put(html, expectedUrls); + + /* Base with path-absolute relative href */ + html = htmlHeaderBeginning + "" + + "" + htmlLinksList; + expectedUrls = new String[]{ "http://example.org/sameDomain/absolute.html", + "http://localhost/otherDomain/absolute.html", "http://example.org/sameDomain/scheme-relative.html", + "http://example.net/otherDomain/scheme-relative.html", "http://example.org/path/absolute.html", + "http://example.org/base/path/relative/schemeless.html" }; + html2Results.put(html, expectedUrls); + + /* Base with path-relative-scheme-less relative href */ + html = htmlHeaderBeginning + "" + + "" + htmlLinksList; + expectedUrls = new String[]{ "http://example.org/sameDomain/absolute.html", + "http://localhost/otherDomain/absolute.html", "http://example.org/sameDomain/scheme-relative.html", + "http://example.net/otherDomain/scheme-relative.html", "http://example.org/path/absolute.html", + "http://example.org/parent/base/path/relative/schemeless.html" }; + html2Results.put(html, expectedUrls); + + for (final Entry html2Result : html2Results.entrySet()) { + ContentScraper scraper = new ContentScraper(docUrl, 10, new HashSet(), new VocabularyScraper(), 0); + try (final Writer writer = new TransformerWriter(null, null, scraper, false)) { + FileUtils.copy(new StringReader(html2Result.getKey()), writer); + + final Set expected = new HashSet<>(); + for (final String url : html2Result.getValue()) { + expected.add(new DigestURL(url)); + } + + Assert.assertEquals(expected.size(), scraper.getAnchors().size()); + Assert.assertTrue(expected.containsAll(scraper.getAnchors())); + } finally { + scraper.close(); + } + } + } + /** * Test microdata itemtype attribute parsing * @throws IOException -- cgit v1.2.3