Strip markdown syntax from snippet text intended for browser display

2 files changed, 109 insertions, 119 deletions

diff --git a/source/net/yacy/ai/RAGAugmentor.java b/source/net/yacy/ai/RAGAugmentor.java
index d1dbb18db..d28e33c2a 100644
--- a/source/net/yacy/ai/RAGAugmentor.java
+++ b/source/net/yacy/ai/RAGAugmentor.java
@@ -45,7 +45,6 @@ import org.json.JSONObject;
 import net.yacy.cora.document.analysis.Classification;
 import net.yacy.cora.document.id.DigestURL;
 import net.yacy.cora.document.id.MultiProtocolURL;
-import net.yacy.cora.federate.solr.SolrType;
 import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector;
 import net.yacy.cora.federate.yacy.CacheStrategy;
 import net.yacy.cora.lod.vocabulary.Tagging;
@@ -107,47 +106,10 @@ public final class RAGAugmentor {
     public static JSONArray searchResults(String query, int count, final boolean includeSnippet, final Set<String> boostTerms) {
         final JSONArray results = new JSONArray();
         if (query == null || query.length() == 0 || count == 0) return results;
-        Switchboard sb = Switchboard.getSwitchboard();
-        EmbeddedSolrConnector connector = sb.index.fulltext().getDefaultEmbeddedConnector();
-        final SolrQuery params = new SolrQuery();
-        // Base query and parser setup.
-        params.setQuery(query);
-        params.set("defType", "edismax");
-        // Static field boosts favor title/headings over body text.
-        params.set("qf",
-            CollectionSchema.title.getSolrFieldName() + "^3 " +
-            CollectionSchema.text_t.getSolrFieldName() + "^1 " +
-            CollectionSchema.sku.getSolrFieldName() + "^0.5 " +
-            CollectionSchema.h1_txt.getSolrFieldName() + "^2");
-        params.set("pf",
-            CollectionSchema.title.getSolrFieldName() + "^5 " +
-            CollectionSchema.text_t.getSolrFieldName() + "^2");
-        final List<String> bqParts = new ArrayList<>();
-        if (boostTerms != null && !boostTerms.isEmpty()) {
-            // Apply weak boosts to overlap terms, keeping lexical query dominant.
-            for (String term : boostTerms) {
-                if (term == null || term.isEmpty()) continue;
-                bqParts.add(CollectionSchema.title.getSolrFieldName() + ":" + term + "^0.5");
-                bqParts.add(CollectionSchema.h1_txt.getSolrFieldName() + ":" + term + "^0.4");
-                bqParts.add(CollectionSchema.text_t.getSolrFieldName() + ":" + term + "^0.2");
-            }
-        }
-        // Slightly prefer archive/container file extensions in this use case.
-        bqParts.add("(" + CollectionSchema.url_file_ext_s.getSolrFieldName() + ":(zip rar 7z tar gz bz2 xz tgz))^0.1");
-        params.set("bq", String.join(" ", bqParts));
-        params.setRows(count);
-        params.setStart(0);
-        params.setFacet(false);
-        params.clearSorts();
-        // Fetch only fields needed for tool / markdown rendering.
-        params.setFields(
-            CollectionSchema.sku.getSolrFieldName(), CollectionSchema.title.getSolrFieldName(), CollectionSchema.text_t.getSolrFieldName(),
-            CollectionSchema.description_txt.getSolrFieldName(), CollectionSchema.keywords.getSolrFieldName(), CollectionSchema.synonyms_sxt.getSolrFieldName(),
-            CollectionSchema.h1_txt.getSolrFieldName(), CollectionSchema.h2_txt.getSolrFieldName(), CollectionSchema.h3_txt.getSolrFieldName(),
-            CollectionSchema.h4_txt.getSolrFieldName(), CollectionSchema.h5_txt.getSolrFieldName(), CollectionSchema.h6_txt.getSolrFieldName()
-        );
-        params.setIncludeScore(true);
-        params.set("df", CollectionSchema.text_t.getSolrFieldName());
+        final Switchboard sb = Switchboard.getSwitchboard();
+        final EmbeddedSolrConnector connector = sb.index.fulltext().getDefaultEmbeddedConnector();
+        final QueryParams theQuery = buildTextQueryParams(query, count, QueryParams.Searchdom.LOCAL);
+        final SolrQuery params = theQuery.solrQuery(Classification.ContentDomain.TEXT, false, false, false);
 
         try {
             final SolrDocumentList sdl = connector.getDocumentListByParams(params);
@@ -158,11 +120,11 @@ public final class RAGAugmentor {
                     final JSONObject result = new JSONObject(true);
                     String url = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName());
                     result.put("url", url == null ? "" : url.trim());
-                    String title = getOneString(doc, CollectionSchema.title);
+                    String title = firstFieldString(doc.getFieldValue(CollectionSchema.title.getSolrFieldName()));
                     result.put("title", title == null ? "" : title.trim());
                     if (includeSnippet) {
                         // Use indexed text body as quick snippet source.
-                        String text = (String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName());
+                        String text = firstFieldString(doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName()));
                         result.put("text", limitSnippet(text == null ? "" : text.trim(), 2000));
                     }
                     results.put(result);
@@ -250,44 +212,7 @@ public final class RAGAugmentor {
         final JSONArray results = new JSONArray();
         if (query == null || query.length() == 0 || count == 0) return results;
         final Switchboard sb = Switchboard.getSwitchboard();
-        final RankingProfile ranking = sb.getRanking();
-        final int timezoneOffset = 0;
-        final QueryModifier modifier = new QueryModifier(timezoneOffset);
-        // Parse modifiers and normalize effective query string.
-        String querystring = modifier.parse(query);
-        if (querystring.length() == 0) querystring = query == null ? "" : query.trim();
-        if (querystring.length() == 0) return results;
-        final QueryGoal qg = new QueryGoal(querystring);
-        // Construct a standard text-domain global query.
-        final QueryParams theQuery = new QueryParams(
-                qg,
-                modifier,
-                0,
-                "",
-                Classification.ContentDomain.TEXT,
-                "",
-                timezoneOffset,
-                new HashSet<Tagging.Metatag>(),
-                CacheStrategy.IFFRESH,
-                count,
-                0,
-                ".*",
-                null,
-                null,
-                QueryParams.Searchdom.GLOBAL,
-                null,
-                true,
-                DigestURL.hosthashess(sb.getConfig("search.excludehosth", "")),
-                MultiProtocolURL.TLD_any_zone_filter,
-                null,
-                false,
-                sb.index,
-                ranking,
-                ClientIdentification.yacyIntranetCrawlerAgent.userAgent(),
-                0.0d,
-                0.0d,
-                0.0d,
-                sb.getConfigSet("search.navigation"));
+        final QueryParams theQuery = buildTextQueryParams(query, count, QueryParams.Searchdom.GLOBAL);
         final SearchEvent theSearch = SearchEventCache.getEvent(
                 theQuery,
                 sb.peers,
@@ -332,6 +257,55 @@ public final class RAGAugmentor {
     }
 
     /**
+     * Build a standard YaCy text query using the shared query parser and ranking
+     * configuration. This gives local and global RAG retrieval the same Solr query
+     * semantics as the normal search stack.
+     *
+     * @param query raw query string
+     * @param count maximum number of results to retrieve
+     * @param searchdom local or global search scope
+     * @return shared query parameters ready for Solr/event execution
+     */
+    private static QueryParams buildTextQueryParams(final String query, final int count, final QueryParams.Searchdom searchdom) {
+        final Switchboard sb = Switchboard.getSwitchboard();
+        final RankingProfile ranking = sb.getRanking();
+        final int timezoneOffset = 0;
+        final QueryModifier modifier = new QueryModifier(timezoneOffset);
+        String querystring = modifier.parse(query);
+        if (querystring.length() == 0) querystring = query == null ? "" : query.trim();
+        final QueryGoal qg = new QueryGoal(querystring);
+        return new QueryParams(
+                qg,
+                modifier,
+                0,
+                "",
+                Classification.ContentDomain.TEXT,
+                "",
+                timezoneOffset,
+                new HashSet<Tagging.Metatag>(),
+                CacheStrategy.IFFRESH,
+                count,
+                0,
+                ".*",
+                null,
+                null,
+                searchdom,
+                null,
+                true,
+                DigestURL.hosthashess(sb.getConfig("search.excludehosth", "")),
+                MultiProtocolURL.TLD_any_zone_filter,
+                null,
+                false,
+                sb.index,
+                ranking,
+                ClientIdentification.yacyIntranetCrawlerAgent.userAgent(),
+                0.0d,
+                0.0d,
+                0.0d,
+                sb.getConfigSet("search.navigation"));
+    }
+
+    /**
      * Uses an LLM list schema prompt to compute likely discriminative search
      * terms for a user prompt.
      *
@@ -490,24 +464,6 @@ public final class RAGAugmentor {
     }
 
     /**
-     * Reads a single string value from a (possibly multivalued) Solr field.
-     *
-     * @param doc source Solr document
-     * @param field schema field descriptor
-     * @return first string value or empty string
-     */
-    private static String getOneString(SolrDocument doc, CollectionSchema field) {
-        assert field.isMultiValued();
-        assert field.getType() == SolrType.string || field.getType() == SolrType.text_general;
-        Object r = doc.getFieldValue(field.getSolrFieldName());
-        if (r == null) return "";
-        if (r instanceof ArrayList) {
-            return (String) ((ArrayList<?>) r).get(0);
-        }
-        return r.toString();
-    }
-
-    /**
      * Represents one candidate snippet around a search result and stores its
      * relevance score relative to the query.
      */
diff --git a/source/net/yacy/search/snippet/TextSnippet.java b/source/net/yacy/search/snippet/TextSnippet.java
index 7b0271281..948e41c69 100644
--- a/source/net/yacy/search/snippet/TextSnippet.java
+++ b/source/net/yacy/search/snippet/TextSnippet.java
@@ -86,8 +86,16 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
     /**

      * <code>[^\\p{L}\\p{N}]</code>

      */

-    private static final Pattern p4 =

-            Pattern.compile("[^\\p{L}\\p{N}]");

+    private static final Pattern p4 =
+            Pattern.compile("[^\\p{L}\\p{N}]");
+    private static final Pattern MARKDOWN_LINK_PATTERN =
+            Pattern.compile("\\[(.+?)\\]\\((.+?)\\)");
+    private static final Pattern MARKDOWN_IMAGE_PATTERN =
+            Pattern.compile("!\\[(.*?)\\]\\((.+?)\\)");
+    private static final Pattern MARKDOWN_EMPHASIS_PATTERN =
+            Pattern.compile("(\\*\\*|__|~~|`|\\*|_)");
+    private static final Pattern MARKDOWN_PREFIX_PATTERN =
+            Pattern.compile("(?m)^\\s{0,3}(#{1,6}\\s+|>\\s+|[-*+]\\s+|\\d+\\.\\s+)");
 

     public static class Cache {

         private final ARC<String, String> cache;

@@ -147,8 +155,11 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
     /** URL hash of this snippet */

     private byte[] urlhash;

     

-    /** The raw (unmodified) line from source ( use getDescriptionLine() to get the html encoded version for display) */

-    private String line;

+    /** The raw line from source, kept for RAG and non-display consumers */
+    private String rawline;
+
+    /** Display-oriented snippet line with markdown formatting stripped */
+    private String line;
     

     /** Set to true when query words are already marked in the input text */

     private boolean isMarked;

@@ -406,7 +417,8 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
             final String errortext,
             final long beginTime) {
         this.urlhash = url.hash();
-        this.line = sanitizeSnippetLine(line);
+        this.rawline = sanitizeSnippetLine(line);
+        this.line = stripMarkdownForDisplay(this.rawline);
         this.isMarked = isMarked;
         this.resultStatus = errorCode;
         this.error = errortext;
@@ -461,6 +473,28 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
         return sanitized.toString().replaceAll("\\s{2,}", " ").trim();
     }
 
+    /**
+     * Strip markdown syntax from snippet text intended for browser display while
+     * preserving readable content. The raw snippet remains available through
+     * getLineRaw() for RAG consumers.
+     * @param line raw snippet text
+     * @return display-friendly plain text
+     */
+    static String stripMarkdownForDisplay(final String line) {
+        if (line == null || line.isEmpty()) {
+            return line;
+        }
+        String sanitized = line;
+        sanitized = sanitized.replace("```", " ");
+        sanitized = MARKDOWN_IMAGE_PATTERN.matcher(sanitized).replaceAll("$1");
+        sanitized = MARKDOWN_LINK_PATTERN.matcher(sanitized).replaceAll("$1");
+        sanitized = MARKDOWN_PREFIX_PATTERN.matcher(sanitized).replaceAll("");
+        sanitized = MARKDOWN_EMPHASIS_PATTERN.matcher(sanitized).replaceAll("");
+        sanitized = sanitized.replaceAll("\\\\([\\\\`*_{}\\[\\]()#+\\-.!>~|])", "$1");
+        sanitized = sanitized.replaceAll("\\s{2,}", " ").trim();
+        return sanitized;
+    }
+
     private static boolean isLikelyJavaScriptBlock(final String blockContent) {
         if (blockContent == null) {
             return false;
@@ -487,8 +521,8 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
     /**

      * @return true when a snippet text is available for the document corresponding to this.urlhash

      */

-    public boolean exists() {

-        return this.line != null;

+    public boolean exists() {
+        return this.rawline != null;
     }

 

     /**

@@ -501,8 +535,8 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
     /**

      * @return the raw snippet text line, with query words eventually already marked

      */

-    public String getLineRaw() {

-        return (this.line == null) ? "" : this.line;

+    public String getLineRaw() {
+        return (this.rawline == null) ? "" : this.rawline;
     }

 

     public String getError() {

@@ -563,16 +597,16 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
      * @param queryGoal

      * @return html encoded snippet line

      */

-    public String descriptionline(QueryGoal queryGoal) {

-        if (descriptionline != null) return descriptionline;

-        if (this.isMarked) {

-            // html encode source, keep <b>..</b>

-            descriptionline = CharacterCoding.unicode2html(this.getLineRaw(), false).replaceAll("&lt;b&gt;(.+?)&lt;/b&gt;", "<b>$1</b>");

-        } else {

-            descriptionline = this.getLineMarked(queryGoal);

-        }

-        return descriptionline;

-    }

+    public String descriptionline(QueryGoal queryGoal) {
+        if (descriptionline != null) return descriptionline;
+        if (this.isMarked) {
+            // html encode source, keep <b>..</b>
+            descriptionline = CharacterCoding.unicode2html(this.line == null ? "" : this.line, false).replaceAll("&lt;b&gt;(.+?)&lt;/b&gt;", "<b>$1</b>");
+        } else {
+            descriptionline = this.getLineMarked(queryGoal);
+        }
+        return descriptionline;
+    }
     

     @Override

     public int compareTo(final TextSnippet o) {