diff options
| -rw-r--r-- | source/net/yacy/search/snippet/TextSnippet.java | 116 | ||||
| -rw-r--r-- | test/java/net/yacy/search/snippet/TextSnippetTest.java | 28 |
2 files changed, 117 insertions, 27 deletions
diff --git a/source/net/yacy/search/snippet/TextSnippet.java b/source/net/yacy/search/snippet/TextSnippet.java index 328988cf0..7b0271281 100644 --- a/source/net/yacy/search/snippet/TextSnippet.java +++ b/source/net/yacy/search/snippet/TextSnippet.java @@ -24,14 +24,15 @@ package net.yacy.search.snippet;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.regex.Pattern;
+import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Locale; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.id.DigestURL;
@@ -397,20 +398,91 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip * @param errortext
* @param beginTime the time in milliseconds when TextSnippet creation started
*/
- private void init(
- final DigestURL url,
- final String line,
- final boolean isMarked,
- final ResultClass errorCode,
- final String errortext,
- final long beginTime) {
- this.urlhash = url.hash();
- this.line = line;
- this.isMarked = isMarked;
- this.resultStatus = errorCode;
- this.error = errortext;
- TextSnippet.statistics.addTextSnippetStatistics(url, System.currentTimeMillis() - beginTime, this.resultStatus);
- }
+ private void init( + final DigestURL url, + final String line, + final boolean isMarked, + final ResultClass errorCode, + final String errortext, + final long beginTime) { + this.urlhash = url.hash(); + this.line = sanitizeSnippetLine(line); + this.isMarked = isMarked; + this.resultStatus = errorCode; + this.error = errortext; + TextSnippet.statistics.addTextSnippetStatistics(url, System.currentTimeMillis() - beginTime, this.resultStatus); + } + + /** + * Remove suspicious JavaScript code fragments from snippet lines. + * This keeps snippets readable even when parser output contains leaked inline script chunks. + * @param line raw snippet line + * @return sanitized snippet line + */ + static String sanitizeSnippetLine(final String line) { + if (line == null || line.indexOf('{') < 0) { + return line; + } + final StringBuilder sanitized = new StringBuilder(line.length()); + int i = 0; + while (i < line.length()) { + final char c = line.charAt(i); + if (c != '{') { + sanitized.append(c); + i++; + continue; + } + final int closePos = line.indexOf('}', i + 1); + if (closePos < 0) { + sanitized.append(c); + i++; + continue; + } + final String blockContent = line.substring(i + 1, closePos); + if (isLikelyJavaScriptBlock(blockContent)) { + if (sanitized.length() > 0 && sanitized.charAt(sanitized.length() - 1) != ' ') { + sanitized.append(' '); + } + i = closePos + 1; + // Also drop immediate trailing JS closure residues, e.g. "); } }" + while (i < line.length()) { + final char tail = line.charAt(i); + if (Character.isWhitespace(tail) || tail == ')' || tail == '}' || tail == ';' || tail == '"' || tail == '\'') { + i++; + } else { + break; + } + } + } else { + sanitized.append(line, i, closePos + 1); + i = closePos + 1; + } + } + return sanitized.toString().replaceAll("\\s{2,}", " ").trim(); + } + + private static boolean isLikelyJavaScriptBlock(final String blockContent) { + if (blockContent == null) { + return false; + } + final String candidate = blockContent.trim(); + if (candidate.isEmpty()) { + return false; + } + final String lower = candidate.toLowerCase(Locale.ROOT); + return lower.contains(";") + || lower.contains("=>") + || lower.contains("function") + || lower.contains("this.") + || lower.contains("window.") + || lower.contains("document.") + || lower.contains("return ") + || lower.contains("var ") + || lower.contains("let ") + || lower.contains("const ") + || lower.contains("$nexttick") + || lower.contains("getboundingclientrect"); + } /**
* @return true when a snippet text is available for the document corresponding to this.urlhash
diff --git a/test/java/net/yacy/search/snippet/TextSnippetTest.java b/test/java/net/yacy/search/snippet/TextSnippetTest.java index 2d12c5d24..c89cee47e 100644 --- a/test/java/net/yacy/search/snippet/TextSnippetTest.java +++ b/test/java/net/yacy/search/snippet/TextSnippetTest.java @@ -158,8 +158,8 @@ public class TextSnippetTest { * checking poper encoding of remaining html in raw snippet line.
*/
@Test
- public void testDescriptionline() throws MalformedURLException {
- String rawtestline = "Über großer test case </span> <pre> <hr><hr /></pre>"; // test line with html, risk of snippet format issue
+ public void testDescriptionline() throws MalformedURLException { + String rawtestline = "Über großer test case </span> <pre> <hr><hr /></pre>"; // test line with html, risk of snippet format issue DigestURL url = new DigestURL("http://localhost/page.html");
QueryGoal qg = new QueryGoal("test");
@@ -196,9 +196,27 @@ public class TextSnippetTest { TextSnippet.ResultClass.SOURCE_METADATA, "");
sniptxt = ts.descriptionline(qg);
System.out.println("testDescriptionline: (with numbers) snippet="+sniptxt);
- assertTrue ("number (.) broken up",sniptxt.contains("1.83"));
- assertTrue ("number (,) broken up",sniptxt.contains("3,14"));
- }
+ assertTrue ("number (.) broken up",sniptxt.contains("1.83")); + assertTrue ("number (,) broken up",sniptxt.contains("3,14")); + } + + @Test + public void testDescriptionlineRemovesSuspiciousJavaScriptBraceBlock() throws MalformedURLException { + final String rawtestline = "Springe zur Suche Springe zum Fußbereich { this.mobileMainNavHeight = el.getBoundingClientRect().height; }); } }"; + final DigestURL url = new DigestURL("http://localhost/page.html"); + final QueryGoal qg = new QueryGoal("Suche"); + + final TextSnippet ts = new TextSnippet( + url, + rawtestline, + true, + TextSnippet.ResultClass.SOURCE_METADATA, ""); + + final String sniptxt = ts.descriptionline(qg); + assertTrue(sniptxt.contains("Springe zur Suche")); + assertFalse(sniptxt.contains("getBoundingClientRect")); + assertFalse(sniptxt.contains("mobileMainNavHeight")); + } /**
* Run text snippet extraction from a given plain text file.
|