Remove suspicious JavaScript code fragments from snippet lines.

2 files changed, 117 insertions, 27 deletions

diff --git a/source/net/yacy/search/snippet/TextSnippet.java b/source/net/yacy/search/snippet/TextSnippet.java
index 328988cf0..7b0271281 100644
--- a/source/net/yacy/search/snippet/TextSnippet.java
+++ b/source/net/yacy/search/snippet/TextSnippet.java
@@ -24,14 +24,15 @@
 

 package net.yacy.search.snippet;

 

-import java.io.IOException;

-import java.util.ArrayList;

-import java.util.Comparator;

-import java.util.HashSet;

-import java.util.Iterator;

-import java.util.List;

-import java.util.Set;

-import java.util.regex.Pattern;

+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Locale;
+import java.util.List;
+import java.util.Set;
+import java.util.regex.Pattern;
 

 import net.yacy.cora.document.encoding.ASCII;

 import net.yacy.cora.document.id.DigestURL;

@@ -397,20 +398,91 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
      * @param errortext

      * @param beginTime the time in milliseconds when TextSnippet creation started

      */

-    private void init(

-            final DigestURL url,

-            final String line,

-            final boolean isMarked,

-            final ResultClass errorCode,

-            final String errortext,

-            final long beginTime) {

-        this.urlhash = url.hash();

-        this.line = line;

-        this.isMarked = isMarked;

-        this.resultStatus = errorCode;

-        this.error = errortext;

-		TextSnippet.statistics.addTextSnippetStatistics(url, System.currentTimeMillis() - beginTime, this.resultStatus);

-    }

+    private void init(
+            final DigestURL url,
+            final String line,
+            final boolean isMarked,
+            final ResultClass errorCode,
+            final String errortext,
+            final long beginTime) {
+        this.urlhash = url.hash();
+        this.line = sanitizeSnippetLine(line);
+        this.isMarked = isMarked;
+        this.resultStatus = errorCode;
+        this.error = errortext;
+		TextSnippet.statistics.addTextSnippetStatistics(url, System.currentTimeMillis() - beginTime, this.resultStatus);
+    }
+
+    /**
+     * Remove suspicious JavaScript code fragments from snippet lines.
+     * This keeps snippets readable even when parser output contains leaked inline script chunks.
+     * @param line raw snippet line
+     * @return sanitized snippet line
+     */
+    static String sanitizeSnippetLine(final String line) {
+        if (line == null || line.indexOf('{') < 0) {
+            return line;
+        }
+        final StringBuilder sanitized = new StringBuilder(line.length());
+        int i = 0;
+        while (i < line.length()) {
+            final char c = line.charAt(i);
+            if (c != '{') {
+                sanitized.append(c);
+                i++;
+                continue;
+            }
+            final int closePos = line.indexOf('}', i + 1);
+            if (closePos < 0) {
+                sanitized.append(c);
+                i++;
+                continue;
+            }
+            final String blockContent = line.substring(i + 1, closePos);
+            if (isLikelyJavaScriptBlock(blockContent)) {
+                if (sanitized.length() > 0 && sanitized.charAt(sanitized.length() - 1) != ' ') {
+                    sanitized.append(' ');
+                }
+                i = closePos + 1;
+                // Also drop immediate trailing JS closure residues, e.g. "); } }"
+                while (i < line.length()) {
+                    final char tail = line.charAt(i);
+                    if (Character.isWhitespace(tail) || tail == ')' || tail == '}' || tail == ';' || tail == '"' || tail == '\'') {
+                        i++;
+                    } else {
+                        break;
+                    }
+                }
+            } else {
+                sanitized.append(line, i, closePos + 1);
+                i = closePos + 1;
+            }
+        }
+        return sanitized.toString().replaceAll("\\s{2,}", " ").trim();
+    }
+
+    private static boolean isLikelyJavaScriptBlock(final String blockContent) {
+        if (blockContent == null) {
+            return false;
+        }
+        final String candidate = blockContent.trim();
+        if (candidate.isEmpty()) {
+            return false;
+        }
+        final String lower = candidate.toLowerCase(Locale.ROOT);
+        return lower.contains(";")
+                || lower.contains("=>")
+                || lower.contains("function")
+                || lower.contains("this.")
+                || lower.contains("window.")
+                || lower.contains("document.")
+                || lower.contains("return ")
+                || lower.contains("var ")
+                || lower.contains("let ")
+                || lower.contains("const ")
+                || lower.contains("$nexttick")
+                || lower.contains("getboundingclientrect");
+    }
 

     /**

      * @return true when a snippet text is available for the document corresponding to this.urlhash

diff --git a/test/java/net/yacy/search/snippet/TextSnippetTest.java b/test/java/net/yacy/search/snippet/TextSnippetTest.java
index 2d12c5d24..c89cee47e 100644
--- a/test/java/net/yacy/search/snippet/TextSnippetTest.java
+++ b/test/java/net/yacy/search/snippet/TextSnippetTest.java
@@ -158,8 +158,8 @@ public class TextSnippetTest {
      * checking poper encoding of remaining html in raw snippet line.

      */

     @Test

-    public void testDescriptionline() throws MalformedURLException {

-        String rawtestline = "Über großer test case </span> <pre> <hr><hr /></pre>"; // test line with html, risk of snippet format issue

+    public void testDescriptionline() throws MalformedURLException {
+        String rawtestline = "Über großer test case </span> <pre> <hr><hr /></pre>"; // test line with html, risk of snippet format issue
 

         DigestURL url = new DigestURL("http://localhost/page.html");

         QueryGoal qg = new QueryGoal("test");

@@ -196,9 +196,27 @@ public class TextSnippetTest {
             TextSnippet.ResultClass.SOURCE_METADATA, "");

         sniptxt = ts.descriptionline(qg);

         System.out.println("testDescriptionline: (with numbers) snippet="+sniptxt);

-        assertTrue ("number (.) broken up",sniptxt.contains("1.83"));

-        assertTrue ("number (,) broken up",sniptxt.contains("3,14"));

-    }

+        assertTrue ("number (.) broken up",sniptxt.contains("1.83"));
+        assertTrue ("number (,) broken up",sniptxt.contains("3,14"));
+    }
+
+    @Test
+    public void testDescriptionlineRemovesSuspiciousJavaScriptBraceBlock() throws MalformedURLException {
+        final String rawtestline = "Springe zur Suche Springe zum Fußbereich { this.mobileMainNavHeight = el.getBoundingClientRect().height; }); } }";
+        final DigestURL url = new DigestURL("http://localhost/page.html");
+        final QueryGoal qg = new QueryGoal("Suche");
+
+        final TextSnippet ts = new TextSnippet(
+                url,
+                rawtestline,
+                true,
+                TextSnippet.ResultClass.SOURCE_METADATA, "");
+
+        final String sniptxt = ts.descriptionline(qg);
+        assertTrue(sniptxt.contains("Springe zur Suche"));
+        assertFalse(sniptxt.contains("getBoundingClientRect"));
+        assertFalse(sniptxt.contains("mobileMainNavHeight"));
+    }
     

 	/**

 	 * Run text snippet extraction from a given plain text file.