summaryrefslogtreecommitdiff
path: root/test/java
diff options
context:
space:
mode:
authorluccioman <luccioman@users.noreply.github.com>2017-07-08 09:04:03 +0200
committerluccioman <luccioman@users.noreply.github.com>2017-07-08 09:04:03 +0200
commitbf55f1d6e582eb126e74d18e0bea2be542efda68 (patch)
tree890bafc9b681e0e3736f8e33bb7eb35842aaa117 /test/java
parent2a87b08cea67f8f2ae46e318c1c3945e8520ec53 (diff)
Started support of partial parsing on large streamed resources.
Thus enable getpageinfo_p API to return something in a reasonable amount of time on resources over MegaBytes size range. Support added first with the generic XML parser, for other formats regular crawler limits apply as usual.
Diffstat (limited to 'test/java')
-rw-r--r--test/java/net/yacy/document/parser/GenericXMLParserTest.java99
-rw-r--r--test/java/net/yacy/document/parser/html/ContentScraperTest.java26
2 files changed, 125 insertions, 0 deletions
diff --git a/test/java/net/yacy/document/parser/GenericXMLParserTest.java b/test/java/net/yacy/document/parser/GenericXMLParserTest.java
index d475b037b..b76e4c0fd 100644
--- a/test/java/net/yacy/document/parser/GenericXMLParserTest.java
+++ b/test/java/net/yacy/document/parser/GenericXMLParserTest.java
@@ -25,6 +25,7 @@ package net.yacy.document.parser;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
import java.io.ByteArrayInputStream;
import java.io.File;
@@ -358,5 +359,103 @@ public class GenericXMLParserTest {
inStream.close();
}
}
+
+ /**
+ * Test URLs detection when applying limits.
+ *
+ * @throws Exception
+ * when an unexpected error occurred
+ */
+ @Test
+ public void testParseWithLimits() throws Exception {
+ String xhtml = "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>"
+ + "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">"
+ + "<html xmlns=\"http://www.w3.org/1999/xhtml\">" + "<head>"
+ + "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />"
+ + "<title>XHTML content URLs test</title>" + "</head>" + "<body>" + "<p>Here are some YaCy URLs: "
+ + "Home page : http://yacy.net - International Forum : "
+ + "http://forum.yacy.de "
+ + "and this is a mention to a relative URL : /document.html</p>"
+ + "<p>Here are YaCy<a href=\"http://mantis.tokeek.de\">bug tracker</a> and <a href=\"http://www.yacy-websearch.net/wiki/\">Wiki</a>."
+ + "And this is a relative link to another <a href=\"/document2.html\">sub document</a></p>"
+ + "</body>" + "</html>";
+
+ /* Content within limits */
+ InputStream inStream = new ByteArrayInputStream(xhtml.getBytes(StandardCharsets.UTF_8.name()));
+ final String contentTypeHeader = "text/xhtml";
+ String charsetFromHttpHeader = HeaderFramework.getCharacterEncoding(contentTypeHeader);
+ DigestURL location = new DigestURL("http://localhost/testfile.xml");
+ try {
+ Document[] documents = this.parser.parseWithLimits(location, contentTypeHeader, charsetFromHttpHeader, new VocabularyScraper(), 0, inStream, Integer.MAX_VALUE, Long.MAX_VALUE);
+ assertEquals(1, documents.length);
+ assertFalse(documents[0].isPartiallyParsed());
+
+ Collection<AnchorURL> detectedAnchors = documents[0].getAnchors();
+ assertNotNull(detectedAnchors);
+ assertEquals(5, detectedAnchors.size());
+ assertTrue(detectedAnchors.contains(new AnchorURL("http://www.w3.org/1999/xhtml")));
+ assertTrue(detectedAnchors.contains(new AnchorURL("http://yacy.net")));
+ assertTrue(detectedAnchors.contains(new AnchorURL("http://forum.yacy.de")));
+ assertTrue(detectedAnchors.contains(new AnchorURL("http://mantis.tokeek.de")));
+ assertTrue(detectedAnchors.contains(new AnchorURL("http://www.yacy-websearch.net/wiki/")));
+ } finally {
+ inStream.close();
+ }
+
+ /* Links limit exceeded */
+ inStream = new ByteArrayInputStream(xhtml.getBytes(StandardCharsets.UTF_8.name()));
+ try {
+ Document[] documents = this.parser.parseWithLimits(location, contentTypeHeader, charsetFromHttpHeader,
+ new VocabularyScraper(), 0, inStream, 2, Long.MAX_VALUE);
+ assertEquals(1, documents.length);
+ assertTrue(documents[0].isPartiallyParsed());
+
+ Collection<AnchorURL> detectedAnchors = documents[0].getAnchors();
+ assertNotNull(detectedAnchors);
+ assertEquals(2, detectedAnchors.size());
+ assertTrue(detectedAnchors.contains(new AnchorURL("http://www.w3.org/1999/xhtml")));
+ assertTrue(detectedAnchors.contains(new AnchorURL("http://yacy.net")));
+ } finally {
+ inStream.close();
+ }
+
+ /* Bytes limit exceeded */
+ StringBuilder xhtmlBuilder = new StringBuilder("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>")
+ .append("<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">")
+ .append("<html xmlns=\"http://www.w3.org/1999/xhtml\">")
+ .append("<head>")
+ .append("<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />")
+ .append("<title>XHTML content URLs test</title>")
+ .append("</head>")
+ .append("<body><p>Here are some YaCy URLs: ")
+ .append("Home page : http://yacy.net - International Forum : ")
+ .append("http://forum.yacy.de ")
+ .append("and this is a mention to a relative URL : /document.html</p>");
+
+ /* Add some filler text to reach a total size beyond SAX parser internal input stream buffers */
+ while(xhtmlBuilder.length() < 1024 * 10) {
+ xhtmlBuilder.append("<p>Some text to parse</p>");
+ }
+
+ int firstBytes = xhtmlBuilder.toString().getBytes(StandardCharsets.UTF_8.name()).length;
+ xhtmlBuilder.append("<p>Here are YaCy<a href=\"http://mantis.tokeek.de\">bug tracker</a> and <a href=\"http://www.yacy-websearch.net/wiki/\">Wiki</a>.")
+ .append("And this is a relative link to another <a href=\"/document2.html\">sub document</a></p>")
+ .append("</body></html>");
+ inStream = new ByteArrayInputStream(xhtmlBuilder.toString().getBytes(StandardCharsets.UTF_8.name()));
+ try {
+ Document[] documents = this.parser.parseWithLimits(location, contentTypeHeader, charsetFromHttpHeader, new VocabularyScraper(), 0, inStream, Integer.MAX_VALUE, firstBytes);
+ assertEquals(1, documents.length);
+ assertTrue(documents[0].isPartiallyParsed());
+
+ Collection<AnchorURL> detectedAnchors = documents[0].getAnchors();
+ assertNotNull(detectedAnchors);
+ assertEquals(3, detectedAnchors.size());
+ assertTrue(detectedAnchors.contains(new AnchorURL("http://www.w3.org/1999/xhtml")));
+ assertTrue(detectedAnchors.contains(new AnchorURL("http://yacy.net")));
+ assertTrue(detectedAnchors.contains(new AnchorURL("http://forum.yacy.de")));
+ } finally {
+ inStream.close();
+ }
+ }
}
diff --git a/test/java/net/yacy/document/parser/html/ContentScraperTest.java b/test/java/net/yacy/document/parser/html/ContentScraperTest.java
index a37510816..021141fe3 100644
--- a/test/java/net/yacy/document/parser/html/ContentScraperTest.java
+++ b/test/java/net/yacy/document/parser/html/ContentScraperTest.java
@@ -268,6 +268,32 @@ public class ContentScraperTest {
}
/**
+ * Test absolute URLs detection in plain text with maxURLs parameter
+ * @throws MalformedURLException should not happen
+ */
+ @Test
+ public void testFindAbsoluteURLsMaxURLs() throws MalformedURLException {
+ final String text = "Some test URLS : http://yacy.net - http://forum.yacy.de - https://en.wikipedia.org";
+
+ /* No limit */
+ ArrayList<AnchorURL> detectedURLs = new ArrayList<>();
+ ContentScraper.findAbsoluteURLs(text, detectedURLs, null, Long.MAX_VALUE);
+ Assert.assertEquals(3, detectedURLs.size());
+
+ /* Test from zero limit, to limit value equals to the total number of URLs in text */
+ for(int limit = 0; limit <=3; limit++) {
+ detectedURLs = new ArrayList<>();
+ ContentScraper.findAbsoluteURLs(text, detectedURLs, null, limit);
+ Assert.assertEquals(limit, detectedURLs.size());
+ }
+
+ /* Limit greater than total number of URLs in text */
+ detectedURLs = new ArrayList<>();
+ ContentScraper.findAbsoluteURLs(text, detectedURLs, null, 4);
+ Assert.assertEquals(3, detectedURLs.size());
+ }
+
+ /**
* Test unpaired brackets cleaning
*/
@Test
generated by cgit v1.2.3 (git 2.46.0) at 2026-08-08 11:20:54 +0000