Fix for 'n-' fragments that appeared after wrong parsing markdown

2 files changed, 11 insertions, 1 deletions

diff --git a/source/net/yacy/document/parser/html/ContentScraper.java b/source/net/yacy/document/parser/html/ContentScraper.java
index 289a34765..3825ad997 100644
--- a/source/net/yacy/document/parser/html/ContentScraper.java
+++ b/source/net/yacy/document/parser/html/ContentScraper.java
@@ -1208,7 +1208,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {
             String text = this.content.trim().toString();

             Matcher m;

             while ((m = this.tripleN.matcher(text)).find()) text = m.replaceAll("\n\n");

-            while ((m = this.doubleNd.matcher(text)).find()) text = m.replaceAll("\\n- ");

+            while ((m = this.doubleNd.matcher(text)).find()) text = m.replaceAll("\n- ");
             return text;

         } catch (final OutOfMemoryError e) {

             ConcurrentLog.logException(e);

diff --git a/source/net/yacy/search/snippet/TextSnippet.java b/source/net/yacy/search/snippet/TextSnippet.java
index a9f7ed411..f6c79bec9 100644
--- a/source/net/yacy/search/snippet/TextSnippet.java
+++ b/source/net/yacy/search/snippet/TextSnippet.java
@@ -94,6 +94,15 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
             Pattern.compile("!\\[(.*?)\\]\\((.+?)\\)");
     private static final Pattern MARKDOWN_EMPHASIS_PATTERN =
             Pattern.compile("(\\*\\*|__|~~|`|\\*|_)");
+    /**
+     * Temporary compatibility patch for legacy snippet entries in existing indexes :
+     * older snippet generation could persist a literal "\n" before markdown prefixes
+     * (notably list items), which later rendered as visible "n-" fragments.
+     * Convert only these escaped markdown line starts back to real newlines so the
+     * normal markdown stripping below can remove them.
+     */
+    private static final Pattern LEGACY_ESCAPED_NEWLINE_MARKDOWN_PREFIX_PATTERN =
+            Pattern.compile("\\\\n(?=\\s{0,3}(#{1,6}\\s+|>\\s+|[-*+]\\s+|\\d+\\.\\s+))");
     private static final Pattern MARKDOWN_PREFIX_PATTERN =
             Pattern.compile("(?m)^\\s{0,3}(#{1,6}\\s+|>\\s+|[-*+]\\s+|\\d+\\.\\s+)");
 

@@ -500,6 +509,7 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
         }
         String sanitized = line;
         sanitized = sanitized.replace("```", " ");
+        sanitized = LEGACY_ESCAPED_NEWLINE_MARKDOWN_PREFIX_PATTERN.matcher(sanitized).replaceAll("\n");
         sanitized = MARKDOWN_IMAGE_PATTERN.matcher(sanitized).replaceAll("$1");
         sanitized = MARKDOWN_LINK_PATTERN.matcher(sanitized).replaceAll("$1");
         sanitized = MARKDOWN_PREFIX_PATTERN.matcher(sanitized).replaceAll("");