added global search to chat

2 files changed, 304 insertions, 44 deletions

diff --git a/htroot/yacychat.html b/htroot/yacychat.html
index cd56fa9a8..50d77f6e3 100644
--- a/htroot/yacychat.html
+++ b/htroot/yacychat.html
@@ -214,7 +214,7 @@
         border-radius: 999px;
         background: #eef2f7;
         overflow: hidden;
-        width: 380px;
+        width: 500px;
         max-width: 100%;
         height: 22px;
       }
@@ -657,12 +657,12 @@
           <div class="search-default-control">
             <span>Default Dialog Augmentation:</span>
             <div class="search-default-options" id="searchDefaultOptions" role="radiogroup" aria-label="Attach search results by default">
-              <input type="radio" id="searchDefaultNone" name="searchDefault" value="none"/>
-              <label for="searchDefaultNone">no search results, allow attachments</label>
+              <input type="radio" id="searchDefaultNone" name="searchDefault" value="no"/>
+              <label for="searchDefaultNone">no search, allow attachments</label>
               <input type="radio" id="searchDefaultLocal" name="searchDefault" value="local" checked="checked"/>
               <label for="searchDefaultLocal">use local search</label>
-              <input type="radio" id="searchDefaultGlobal" name="searchDefault" value="global" class="is-future" disabled="disabled"/>
-              <label for="searchDefaultGlobal" class="is-future">use global search</label>
+              <input type="radio" id="searchDefaultGlobal" name="searchDefault" value="global"/>
+              <label for="searchDefaultGlobal">use global search</label>
               <span class="search-default-slider" aria-hidden="true"></span>
             </div>
           </div>
@@ -723,7 +723,7 @@
     const STORAGE_KEY = 'yacychat_recent_pairs';
     const SEARCH_DEFAULT_KEY = 'yacychat_search_default';
     const SEARCH_DEFAULTS = {
-        none: 'none',
+        none: 'no',
         local: 'local',
         global: 'global'
     };
@@ -737,7 +737,7 @@
         },
         busy: false,
         attachment: null,
-        searchMode: false,
+        searchMode: SEARCH_DEFAULTS.none,
         searchDefault: SEARCH_DEFAULTS.local,
         assistantSeen: false,
         showSystem: false
@@ -1215,7 +1215,7 @@
     function clearAttachment(options = {}) {
         const { applyDefault = true } = options;
         state.attachment = null;
-        state.searchMode = false;
+        state.searchMode = SEARCH_DEFAULTS.none;
         dom.fileInput.value = '';
         dom.attachmentFilename.textContent = 'Attach PNG/JPG or text (.txt/.md/.tex)';
         dom.attachmentRow.classList.remove('has-attachment');
@@ -1227,7 +1227,7 @@
     }
 
     function setComposerAttachment(attachment) {
-        state.searchMode = false;
+        state.searchMode = SEARCH_DEFAULTS.none;
         dom.attachmentRow.classList.remove('search-mode');
         state.attachment = attachment ? cloneAttachment(attachment) : null;
         if (state.attachment) {
@@ -1244,16 +1244,26 @@
         if (isLoading) {
             dom.attachmentFilename.textContent = 'Loading attachment...';
         } else if (!state.attachment) {
-            dom.attachmentFilename.textContent = state.searchMode ? 'Attach Search Results' : 'Attach PNG/JPG or text (.txt/.md/.tex)';
+            dom.attachmentFilename.textContent = state.searchMode !== SEARCH_DEFAULTS.none ? 'Attach Search Results' : 'Attach PNG/JPG or text (.txt/.md/.tex)';
         }
     }
 
-    function setSearchModeActive(enabled) {
-        state.searchMode = !!enabled;
+    function setSearchModeActive(mode) {
+        let nextMode = mode;
+        if (mode === true) {
+            nextMode = state.searchDefault === SEARCH_DEFAULTS.none ? SEARCH_DEFAULTS.local : state.searchDefault;
+        }
+        if (!nextMode || nextMode === false) {
+            nextMode = SEARCH_DEFAULTS.none;
+        }
+        if (nextMode !== SEARCH_DEFAULTS.local && nextMode !== SEARCH_DEFAULTS.global) {
+            nextMode = SEARCH_DEFAULTS.none;
+        }
+        state.searchMode = nextMode;
         state.attachment = null;
         dom.fileInput.value = '';
         dom.attachmentRow.classList.remove('has-attachment');
-        if (state.searchMode) {
+        if (state.searchMode !== SEARCH_DEFAULTS.none) {
             dom.attachmentRow.classList.add('search-mode');
             dom.attachmentFilename.textContent = 'Attach Search Results';
         } else {
@@ -1264,16 +1274,16 @@
 
     function applySearchDefaultToComposer() {
         if (!dom.attachmentRow || state.attachment) return;
-        if (state.searchDefault === SEARCH_DEFAULTS.local) {
-            setSearchModeActive(true);
+        if (state.searchDefault !== SEARCH_DEFAULTS.none) {
+            setSearchModeActive(state.searchDefault);
         } else {
-            setSearchModeActive(false);
+            setSearchModeActive(SEARCH_DEFAULTS.none);
         }
     }
 
     function updateSearchDefaultSlider() {
         if (!dom.searchDefaultOptions) return;
-        const inputs = Array.from(dom.searchDefaultOptions.querySelectorAll('input[name="searchDefault"]:not(.is-future)'));
+        const inputs = Array.from(dom.searchDefaultOptions.querySelectorAll('input[name="searchDefault"]'));
         const activeIndex = inputs.findIndex(input => input.checked);
         const segments = inputs.length || 2;
         dom.searchDefaultOptions.style.setProperty('--segments', segments);
@@ -1295,6 +1305,9 @@
         } catch (err) {
             console.warn('Failed to load search default', err);
         }
+        if (stored === 'none') {
+            stored = SEARCH_DEFAULTS.none;
+        }
         if (stored && Object.values(SEARCH_DEFAULTS).includes(stored)) {
             state.searchDefault = stored;
         }
@@ -1448,6 +1461,7 @@
 
     async function streamChat(userMessage, assistantNode, options = {}) {
         const { onFirstToken, includeUserInPayload = true, pushUserToState = true, currentUserIndex = -1 } = options;
+        const requestStart = performance.now();
         ensureSystemMessage();
         const sanitizedMessages = state.messages
             .map((msg, index) => {
@@ -1462,12 +1476,14 @@
             messages: sanitizedUserMessage ? [...sanitizedMessages, sanitizedUserMessage] : sanitizedMessages,
             stream: true
         };
+        console.debug('[yacychat] payload search modes', payload.messages.map(m => ({ role: m.role, search: m.search })));
         const headers = { 'Content-Type': 'application/json' };
         const response = await fetch(state.config.apiHost.replace(/\/$/, '') + '/v1/chat/completions', {
             method: 'POST',
             headers,
             body: JSON.stringify(payload)
         });
+        console.debug('[yacychat] request ms', Math.round(performance.now() - requestStart));
         if (!response.ok) {
             if (response.status === 429) {
                 throw new Error('Rate limit reached: please wait and try again. The server is protecting itself from overload.');
@@ -1502,11 +1518,12 @@
             const searchName = parsed['search-filename'];
             const searchBase64 = parsed['search-text-base64'];
             if (!searchName || !searchBase64) return;
+            console.debug('[yacychat] received search attachment', { searchName, size: searchBase64.length, ms: Math.round(performance.now() - requestStart) });
             const dataUrl = `data:text/markdown;base64,${searchBase64}`;
             const textContent = base64ToUtf8(searchBase64);
             for (let i = state.messages.length - 1; i >= 0; i--) {
                 const msg = state.messages[i];
-                if (msg && msg.role === 'user' && msg.search) {
+                if (msg && msg.role === 'user' && msg.search && msg.search !== SEARCH_DEFAULTS.none) {
                     msg.attachment = {
                         kind: 'text',
                         name: searchName,
@@ -1621,6 +1638,7 @@
                 ? false
                 : (thinkAutoClosed && !thinkAutoCloseFired);
         state.messages.push({ role: 'assistant', content: assistantText, thinkOpen: storedThinkOpen });
+        console.debug('[yacychat] response chars', assistantText.length, 'total ms', Math.round(performance.now() - requestStart));
         state.assistantSeen = true;
         persistConversation();
         updateClearChatVisibility();
@@ -1651,13 +1669,17 @@
                     filename: state.attachment.name
                 });
             }
-            return {
-                role: 'user',
-                content: parts,
-                search: !!state.searchMode
-            };
+            const message = { role: 'user', content: parts };
+            if (state.searchMode !== SEARCH_DEFAULTS.none) {
+                message.search = state.searchMode;
+            }
+            return message;
         }
-        return { role: 'user', content: promptText, search: !!state.searchMode };
+        const message = { role: 'user', content: promptText };
+        if (state.searchMode !== SEARCH_DEFAULTS.none) {
+            message.search = state.searchMode;
+        }
+        return message;
     }
 
     function stripMessageForApi(message, options = {}) {
@@ -1675,7 +1697,11 @@
             }
         }
         const sanitized = { role: message.role, content };
-        if (message.search) sanitized.search = true;
+        if (message.search === true) {
+            sanitized.search = SEARCH_DEFAULTS.local;
+        } else if (message.search === SEARCH_DEFAULTS.local || message.search === SEARCH_DEFAULTS.global) {
+            sanitized.search = message.search;
+        }
         return sanitized;
     }
 
@@ -1719,9 +1745,18 @@
 
     function persistConversation() {
         try {
+            const messages = state.messages.map(msg => {
+                if (!msg) return msg;
+                const safe = { ...msg };
+                if (safe.attachment) {
+                    const { kind, name, mime, size } = safe.attachment;
+                    safe.attachment = { kind, name, mime, size };
+                }
+                return safe;
+            });
             const payload = {
                 model: state.config.model,
-                messages: state.messages
+                messages
             };
             localStorage.setItem(STORAGE_KEY, JSON.stringify(payload));
         } catch (err) {
@@ -2222,12 +2257,15 @@
             userMessage.attachment = userAttachment;
         }
         // add user message to state immediately so edit/trim is available right away
-        state.messages.push({
+        const userEntry = {
             role: 'user',
             content: userMessage.content,
-            attachment: userAttachment,
-            search: !!state.searchMode
-        });
+            attachment: userAttachment
+        };
+        if (state.searchMode !== SEARCH_DEFAULTS.none) {
+            userEntry.search = state.searchMode;
+        }
+        state.messages.push(userEntry);
         const userIndex = state.messages.length - 1;
         const preview = formatUserPreview(prompt);
         state.busy = true;
@@ -2253,11 +2291,12 @@
         dom.fileInput.click();
     });
     dom.searchButton?.addEventListener('click', () => {
-        setSearchModeActive(true);
+        const mode = state.searchDefault === SEARCH_DEFAULTS.none ? SEARCH_DEFAULTS.local : state.searchDefault;
+        setSearchModeActive(mode);
     });
     dom.fileInput.addEventListener('change', handleFileChange);
     dom.clearFileButton.addEventListener('click', () => {
-        const allowDefault = !(state.searchMode && !state.attachment);
+        const allowDefault = !(state.searchMode !== SEARCH_DEFAULTS.none && !state.attachment);
         clearAttachment({ applyDefault: allowDefault });
     });
     dom.clearChatButton?.addEventListener('click', clearChatHistory);
diff --git a/source/net/yacy/http/servlets/RAGProxyServlet.java b/source/net/yacy/http/servlets/RAGProxyServlet.java
index 499cb1a07..c5e552c47 100644
--- a/source/net/yacy/http/servlets/RAGProxyServlet.java
+++ b/source/net/yacy/http/servlets/RAGProxyServlet.java
@@ -33,9 +33,11 @@ import java.util.AbstractMap;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Base64;
+import java.util.Collection;
 import java.util.Comparator;
 import java.util.Deque;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -65,11 +67,27 @@ import org.json.JSONObject;
 import org.json.JSONTokener;
 
 import net.yacy.ai.LLM;
+import net.yacy.cora.document.analysis.Classification;
+import net.yacy.cora.document.id.DigestURL;
+import net.yacy.cora.document.id.MultiProtocolURL;
 import net.yacy.cora.federate.solr.SolrType;
 import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector;
+import net.yacy.cora.federate.yacy.CacheStrategy;
+import net.yacy.cora.lod.vocabulary.Tagging;
+import net.yacy.cora.protocol.ClientIdentification;
 import net.yacy.cora.protocol.Domains;
+import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.kelondro.data.meta.URIMetadataNode;
 import net.yacy.search.Switchboard;
+import net.yacy.search.SwitchboardConstants;
+import net.yacy.search.query.QueryGoal;
+import net.yacy.search.query.QueryModifier;
+import net.yacy.search.query.QueryParams;
+import net.yacy.search.query.SearchEvent;
+import net.yacy.search.query.SearchEventCache;
+import net.yacy.search.ranking.RankingProfile;
 import net.yacy.search.schema.CollectionSchema;
+import net.yacy.search.snippet.TextSnippet;
 
 /**
  * This class implements a Retrieval Augmented Generation ("RAG") proxy which
@@ -177,6 +195,10 @@ public class RAGProxyServlet extends HttpServlet {
             JSONArray messages = bodyObject.optJSONArray("messages");
             final String systemPrefix = sb.getConfig("ai.llm-system-prefix", LLM_SYSTEM_PREFIX_DEFAULT);
             final String userPrefix = sb.getConfig("ai.llm-user-prefix", LLM_USER_PREFIX_DEFAULT);
+            
+            // debug
+            //System.out.println(messages.toString());
+            
             for (int i = 0; i < messages.length(); i++) {
                 JSONObject message = messages.getJSONObject(i);
                 if (message.optString("role", "").equals("user")) {
@@ -186,17 +208,28 @@ public class RAGProxyServlet extends HttpServlet {
             }
             UserObject userObject = new UserObject(messages.getJSONObject(messages.length() - 1));
             String user = userObject.getContentText(); // this is the latest prompt
-            boolean rag = userObject.getSearch();
+            final String userPrompt = user;
+            String ragMode = userObject.getSearchMode();
+            ConcurrentLog.info("RAGProxy", "ragMode=" + ragMode + " userChars=" + (user == null ? 0 : user.length()));
             //List<DataURL> data_urls = userObject.getContentAttachments(); // this list is a copy of the content data_urls
             
             // RAG
             String searchResultQuery = "";
             String searchResultMarkdown = "";
-            if (rag) {
+            if (!"no".equals(ragMode)) {
                 // modify system and user prompt here in bodyObject to enable RAG
                 final String queryPrefix = sb.getConfig("ai.llm-query-generator-prefix", LLM_QUERY_GENERATOR_PREFIX_DEFAULT);
+                final long queryStart = System.currentTimeMillis();
                 searchResultQuery = this.searchWordsForPrompt(llm4tldr.llm, llm4tldr.model, queryPrefix + user);
-                searchResultMarkdown = searchResultsAsMarkdown(searchResultQuery, 10);
+                final long queryElapsed = System.currentTimeMillis() - queryStart;
+                final Set<String> boostTerms = intersectTokens(userPrompt, searchResultQuery, 8);
+                final long searchStart = System.currentTimeMillis();
+                searchResultMarkdown = searchResultsAsMarkdown(searchResultQuery, 10, "global".equals(ragMode), boostTerms);
+                final long searchElapsed = System.currentTimeMillis() - searchStart;
+                ConcurrentLog.info(
+                    "RAGProxy",
+                    "searchQuery=\"" + searchResultQuery + "\" queryMs=" + queryElapsed + " searchMs=" + searchElapsed +
+                    " markdownChars=" + searchResultMarkdown.length() + " boostTerms=" + boostTerms.size());
                 user += userPrefix;
                 user += searchResultMarkdown;
                 userObject.setContentText(user);
@@ -304,7 +337,7 @@ public class RAGProxyServlet extends HttpServlet {
 
     public final static class UserObject {
         private JSONObject userObject;
-        
+
         public UserObject(JSONObject userObject) {
             this.userObject = userObject;
         }
@@ -325,9 +358,19 @@ public class RAGProxyServlet extends HttpServlet {
             this.normalize();
         }
         
-        public boolean getSearch() {
-            boolean search = this.userObject.optBoolean("search", false);
-            return search;
+        public String getSearchMode() {
+            Object raw = this.userObject.opt("search");
+            if (raw instanceof Boolean) {
+                return ((Boolean) raw) ? "local" : "no";
+            }
+            final String search = this.userObject.optString("search", "").trim().toLowerCase();
+            if (search.isEmpty() || "no".equals(search) || "false".equals(search)) {
+                return "no";
+            }
+            if ("local".equals(search) || "global".equals(search)) {
+                return search;
+            }
+            return "no";
         }
         
         public String getContentText() {
@@ -440,13 +483,38 @@ public class RAGProxyServlet extends HttpServlet {
     }
     
     public static JSONArray searchResults(String query, int count, final boolean includeSnippet) {
+        return searchResults(query, count, includeSnippet, new LinkedHashSet<>());
+    }
+
+    public static JSONArray searchResults(String query, int count, final boolean includeSnippet, final Set<String> boostTerms) {
         final JSONArray results = new JSONArray();
         if (query == null || query.length() == 0 || count == 0) return results;
         Switchboard sb = Switchboard.getSwitchboard();
         EmbeddedSolrConnector connector = sb.index.fulltext().getDefaultEmbeddedConnector();
         // construct query
         final SolrQuery params = new SolrQuery();
-        params.setQuery(CollectionSchema.text_t.getSolrFieldName() + ":" + query);
+        params.setQuery(query);
+        params.set("defType", "edismax");
+        params.set("qf",
+            CollectionSchema.title.getSolrFieldName() + "^3 " +
+            CollectionSchema.text_t.getSolrFieldName() + "^1 " +
+            CollectionSchema.sku.getSolrFieldName() + "^0.5 " +
+            CollectionSchema.h1_txt.getSolrFieldName() + "^2");
+        params.set("pf",
+            CollectionSchema.title.getSolrFieldName() + "^5 " +
+            CollectionSchema.text_t.getSolrFieldName() + "^2");
+        params.set("mm", "2<75%");
+        final List<String> bqParts = new ArrayList<>();
+        if (boostTerms != null && !boostTerms.isEmpty()) {
+            for (String term : boostTerms) {
+                if (term == null || term.isEmpty()) continue;
+                bqParts.add(CollectionSchema.title.getSolrFieldName() + ":" + term + "^0.5");
+                bqParts.add(CollectionSchema.h1_txt.getSolrFieldName() + ":" + term + "^0.4");
+                bqParts.add(CollectionSchema.text_t.getSolrFieldName() + ":" + term + "^0.2");
+            }
+        }
+        bqParts.add("(" + CollectionSchema.url_file_ext_s.getSolrFieldName() + ":(zip rar 7z tar gz bz2 xz tgz))^0.1");
+        params.set("bq", String.join(" ", bqParts));
         params.setRows(count);
         params.setStart(0);
         params.setFacet(false);
@@ -469,7 +537,7 @@ public class RAGProxyServlet extends HttpServlet {
                     result.put("title", title == null ? "" : title.trim());
                     if (includeSnippet) {
                         String text = (String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName());
-                        result.put("text", text == null ? "" : text.trim());
+                        result.put("text", limitSnippet(text == null ? "" : text.trim(), 2000));
                     }
                 results.put(result);
                 } catch (JSONException e) {
@@ -482,8 +550,14 @@ public class RAGProxyServlet extends HttpServlet {
         }
     }
     
-    public static String searchResultsAsMarkdown(String query, int count) {
-        JSONArray searchResults = searchResults(query, count, true);
+    public static String searchResultsAsMarkdown(String query, int count, boolean global) {
+        return searchResultsAsMarkdown(query, count, global, new LinkedHashSet<>());
+    }
+
+    public static String searchResultsAsMarkdown(String query, int count, boolean global, final Set<String> boostTerms) {
+        final long searchStart = System.currentTimeMillis();
+        JSONArray searchResults = global ? searchResultsGlobal(query, count, true) : searchResults(query, count, true, boostTerms);
+        ConcurrentLog.info("RAGProxy", "searchResults=" + searchResults.length() + " global=" + global + " searchMs=" + (System.currentTimeMillis() - searchStart));
         StringBuilder sb = new StringBuilder();
         
         // collect snippets
@@ -494,6 +568,8 @@ public class RAGProxyServlet extends HttpServlet {
                 String title = r.optString("title", "");
                 String url = r.optString("url", "");
                 String text = r.optString("text", "");
+                if (title.isEmpty()) title = url;
+                if (text.isEmpty()) text = title;
                 if (title.length() > 0 && text.length() > 0) {
                     Snippet snippet = new Snippet(query, text, url, title, 256); // we always compute a snippet because that gives us a hint if the query appears at all
                     if (snippet.getText().length() > 0) results.add(snippet);
@@ -504,7 +580,9 @@ public class RAGProxyServlet extends HttpServlet {
         // sort snippets again by score
         results.sort(Comparator.comparingDouble(Snippet::getScore));
         
-        for (int i  = 0; i < results.size() / 2; i++) {
+        int limit = results.size() / 2;
+        if (results.size() > 0 && limit == 0) limit = 1;
+        for (int i  = 0; i < limit; i++) {
             Snippet snippet = results.get(i);
             sb.append("## ").append(snippet.getTitle()).append("\n");
             sb.append(snippet.text).append("\n");
@@ -512,8 +590,130 @@ public class RAGProxyServlet extends HttpServlet {
             sb.append("\n\n");
         }
         
+        ConcurrentLog.info("RAGProxy", "markdownChars=" + sb.length() + " snippetCount=" + results.size());
         return sb.toString();
     }
+
+    public static JSONArray searchResultsGlobal(String query, int count, final boolean includeSnippet) {
+        final JSONArray results = new JSONArray();
+        if (query == null || query.length() == 0 || count == 0) return results;
+        final Switchboard sb = Switchboard.getSwitchboard();
+        final RankingProfile ranking = sb.getRanking();
+        final int timezoneOffset = 0;
+        final QueryModifier modifier = new QueryModifier(timezoneOffset);
+        String querystring = modifier.parse(query);
+        if (querystring.length() == 0) {
+            querystring = query == null ? "" : query.trim();
+        }
+        if (querystring.length() == 0) return results;
+        final QueryGoal qg = new QueryGoal(querystring);
+        final QueryParams theQuery = new QueryParams(
+                qg,
+                modifier,
+                0,
+                "",
+                Classification.ContentDomain.TEXT,
+                "",
+                timezoneOffset,
+                new HashSet<Tagging.Metatag>(),
+                CacheStrategy.IFFRESH,
+                count,
+                0,
+                ".*",
+                null,
+                null,
+                QueryParams.Searchdom.GLOBAL,
+                null,
+                true,
+                DigestURL.hosthashess(sb.getConfig("search.excludehosth", "")),
+                MultiProtocolURL.TLD_any_zone_filter,
+                null,
+                false,
+                sb.index,
+                ranking,
+                ClientIdentification.yacyIntranetCrawlerAgent.userAgent(),
+                0.0d,
+                0.0d,
+                0.0d,
+                sb.getConfigSet("search.navigation"));
+        final SearchEvent theSearch = SearchEventCache.getEvent(
+                theQuery,
+                sb.peers,
+                sb.tables,
+                (sb.isRobinsonMode()) ? sb.clusterhashes : null,
+                false,
+                sb.loader,
+                (int) sb.getConfigLong(
+                        SwitchboardConstants.REMOTESEARCH_MAXCOUNT_USER,
+                        sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXCOUNT_DEFAULT, 10)),
+                sb.getConfigLong(
+                        SwitchboardConstants.REMOTESEARCH_MAXTIME_USER,
+                        sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000)));
+        final long timeout = sb.getConfigLong(
+                SwitchboardConstants.REMOTESEARCH_MAXTIME_USER,
+                sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000));
+        waitForFeedingAndResort(theSearch, timeout);
+        for (int i = 0; i < count; i++) {
+            URIMetadataNode node = theSearch.oneResult(i, timeout);
+            if (node == null) break;
+            try {
+                final JSONObject result = new JSONObject(true);
+                result.put("url", node.urlstring());
+                result.put("title", node.title());
+                if (includeSnippet) {
+                    String text = node.snippet();
+                    if (text == null || text.isEmpty()) {
+                        TextSnippet snippet = node.textSnippet();
+                        if (snippet != null && snippet.exists() && !snippet.getErrorCode().fail()) {
+                            text = snippet.getLineRaw();
+                        }
+                    }
+                    if (text == null || text.isEmpty()) {
+                        text = firstFieldString(node.getFieldValue(CollectionSchema.description_txt.getSolrFieldName()));
+                    }
+                    if (text == null || text.isEmpty()) {
+                        text = firstFieldString(node.getFieldValue(CollectionSchema.text_t.getSolrFieldName()));
+                    }
+                    result.put("text", limitSnippet(text == null ? "" : text.trim(), 2000));
+                }
+                results.put(result);
+            } catch (JSONException e) {
+                // skip this result
+            }
+        }
+        return results;
+    }
+
+    private static String limitSnippet(String text, int maxChars) {
+        if (text == null) return "";
+        if (maxChars <= 0 || text.length() <= maxChars) return text;
+        return text.substring(0, maxChars);
+    }
+
+    private static String firstFieldString(Object value) {
+        if (value == null) return "";
+        if (value instanceof Collection) {
+            for (Object item : (Collection<?>) value) {
+                if (item != null) return item.toString();
+            }
+            return "";
+        }
+        return value.toString();
+    }
+
+    private static void waitForFeedingAndResort(SearchEvent search, long timeoutMs) {
+        if (search == null || timeoutMs <= 0) return;
+        final long end = System.currentTimeMillis() + timeoutMs;
+        while (!search.isFeedingFinished() && System.currentTimeMillis() < end) {
+            try {
+                Thread.sleep(100);
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                break;
+            }
+        }
+        search.resortCachedResults();
+    }
     
     
     public static class Snippet {
@@ -710,6 +910,27 @@ public class RAGProxyServlet extends HttpServlet {
         return queryWordSet;
     }
 
+    private static Set<String> intersectTokens(String originalPrompt, String computedQuery, int maxTerms) {
+        Set<String> promptTerms = querySet(originalPrompt == null ? "" : originalPrompt);
+        Set<String> queryTerms = querySet(computedQuery == null ? "" : computedQuery);
+        Set<String> intersection = new LinkedHashSet<>();
+        for (String term : promptTerms) {
+            if (!queryTerms.contains(term)) continue;
+            final String cleaned = cleanToken(term);
+            if (cleaned.isEmpty()) continue;
+            intersection.add(cleaned);
+            if (maxTerms > 0 && intersection.size() >= maxTerms) break;
+        }
+        return intersection;
+    }
+
+    private static String cleanToken(String term) {
+        if (term == null) return "";
+        String cleaned = term.replaceAll("[^A-Za-z0-9]", "");
+        if (cleaned.length() < 2) return "";
+        return cleaned.toLowerCase();
+    }
+
     private static JSONObject responseLine(String payload) {
         JSONObject j = new JSONObject(true);
         try {