Merge branch 'master' of https://github.com/yacy/yacy_search_server

7 files changed, 309 insertions, 61 deletions

diff --git a/htroot/yacychat.html b/htroot/yacychat.html
index d94fc2a8b..072e25317 100644
--- a/htroot/yacychat.html
+++ b/htroot/yacychat.html
@@ -410,6 +410,18 @@
       .toolcalls-button:hover {
         background: #c3cbd9;
       }
+      .toolcalls-button.pending {
+        cursor: default;
+        opacity: 0.75;
+        animation: toolcall-pulse 1s ease-in-out infinite;
+      }
+      .toolcalls-button.pending:hover {
+        background: #d1d9e6;
+      }
+      @keyframes toolcall-pulse {
+        0%, 100% { opacity: 0.45; }
+        50% { opacity: 1; }
+      }
       .button-active {
         background: #337ab7;
         color: #fff;
@@ -1300,56 +1312,7 @@
             copyBtn.innerHTML = '<span class="glyphicon glyphicon-copy" aria-hidden="true"></span>';
             copyBtn.addEventListener('click', () => copyAssistant(body));
             entry.appendChild(copyBtn);
-            if (Array.isArray(toolCalls) && toolCalls.length > 0) {
-                const toolbar = document.createElement('div');
-                toolbar.className = 'toolcall-toolbar';
-                const panels = [];
-                const resultsById = new Map();
-                const resultsByName = new Map();
-                if (Array.isArray(toolResults)) {
-                    for (const result of toolResults) {
-                        if (!result) continue;
-                        if (result.id) resultsById.set(result.id, result);
-                        if (result.tool_call_id) resultsById.set(result.tool_call_id, result);
-                        const rname = result?.name || result?.function?.name;
-                        if (rname && !resultsByName.has(rname)) resultsByName.set(rname, result);
-                    }
-                }
-
-                for (let i = 0; i < toolCalls.length; i++) {
-                    const call = toolCalls[i];
-                    if (!call) continue;
-                    const fnName = call?.function?.name || call?.name || 'tool';
-                    const matchedResult = (call.id && resultsById.get(call.id)) || resultsByName.get(fnName) || null;
-
-                    const toolBtn = document.createElement('button');
-                    toolBtn.type = 'button';
-                    toolBtn.className = 'toolcalls-button';
-                    toolBtn.title = `Tool: ${fnName}`;
-                    toolBtn.innerHTML = '<span class="glyphicon glyphicon-wrench" aria-hidden="true"></span>';
-
-                    const toolPanel = createToolCallPanel(call, matchedResult, toolCalls.length, i);
-                    toolPanel.style.display = 'none';
-                    toolPanel.addEventListener('click', evt => evt.stopPropagation());
-                    entry.appendChild(toolPanel);
-                    panels.push(toolPanel);
-
-                    toolBtn.addEventListener('click', event => {
-                        event.stopPropagation();
-                        const open = toolPanel.style.display !== 'none';
-                        for (const panel of panels) panel.style.display = 'none';
-                        toolPanel.style.display = open ? 'none' : 'block';
-                    });
-                    toolbar.appendChild(toolBtn);
-                }
-                entry.appendChild(toolbar);
-
-                document.addEventListener('click', evt => {
-                    if (!entry.contains(evt.target)) {
-                        for (const panel of panels) panel.style.display = 'none';
-                    }
-                });
-            }
+            renderToolCallControls(entry, toolCalls, toolResults);
         }
         if (role === 'user' && typeof messageIndex === 'number') {
             const trimBtn = document.createElement('button');
@@ -1377,6 +1340,89 @@
         return body;
     }
 
+    function renderToolCallControls(entry, toolCalls, toolResults, pendingToolCalls = []) {
+        if (!entry) return;
+        if (entry._toolcallOutsideClickHandler) {
+            document.removeEventListener('click', entry._toolcallOutsideClickHandler);
+            entry._toolcallOutsideClickHandler = null;
+        }
+        Array.prototype.slice.call(entry.children).forEach(node => {
+            if (node.classList && (node.classList.contains('toolcall-toolbar') || node.classList.contains('toolcalls-popover'))) {
+                node.remove();
+            }
+        });
+        if (!Array.isArray(toolCalls) || toolCalls.length === 0) return;
+
+        const toolbar = document.createElement('div');
+        toolbar.className = 'toolcall-toolbar';
+        const panels = [];
+        const resultsById = new Map();
+        const resultsByName = new Map();
+        if (Array.isArray(toolResults)) {
+            for (const result of toolResults) {
+                if (!result) continue;
+                if (result.id) resultsById.set(result.id, result);
+                if (result.tool_call_id) resultsById.set(result.tool_call_id, result);
+                const rname = result?.name || result?.function?.name;
+                if (rname && !resultsByName.has(rname)) resultsByName.set(rname, result);
+            }
+        }
+
+        for (let i = 0; i < toolCalls.length; i++) {
+            const call = toolCalls[i];
+            if (!call) continue;
+            const fnName = call?.function?.name || call?.name || 'tool';
+            const matchedResult = (call.id && resultsById.get(call.id)) || resultsByName.get(fnName) || null;
+            const pending = isPendingToolCall(call, pendingToolCalls);
+
+            const toolBtn = document.createElement('button');
+            toolBtn.type = 'button';
+            toolBtn.className = pending ? 'toolcalls-button pending' : 'toolcalls-button';
+            toolBtn.title = pending ? `Preparing tool: ${fnName}` : `Tool: ${fnName}`;
+            toolBtn.disabled = pending;
+            toolBtn.innerHTML = '<span class="glyphicon glyphicon-wrench" aria-hidden="true"></span>';
+
+            if (!pending) {
+                const toolPanel = createToolCallPanel(call, matchedResult, toolCalls.length, i);
+                toolPanel.style.display = 'none';
+                toolPanel.addEventListener('click', evt => evt.stopPropagation());
+                entry.appendChild(toolPanel);
+                panels.push(toolPanel);
+
+                toolBtn.addEventListener('click', event => {
+                    event.stopPropagation();
+                    const open = toolPanel.style.display !== 'none';
+                    for (const panel of panels) panel.style.display = 'none';
+                    toolPanel.style.display = open ? 'none' : 'block';
+                });
+            }
+            toolbar.appendChild(toolBtn);
+        }
+        entry.appendChild(toolbar);
+
+        entry._toolcallOutsideClickHandler = evt => {
+            if (!entry.contains(evt.target)) {
+                for (const panel of panels) panel.style.display = 'none';
+            }
+        };
+        document.addEventListener('click', entry._toolcallOutsideClickHandler);
+    }
+
+    function isPendingToolCall(call, pendingToolCalls) {
+        if (!call || !Array.isArray(pendingToolCalls)) return false;
+        return pendingToolCalls.some(pending => sameToolCall(call, pending));
+    }
+
+    function sameToolCall(a, b) {
+        if (!a || !b) return false;
+        if (a.id && b.id) return a.id === b.id;
+        const an = a?.function?.name || a?.name || '';
+        const bn = b?.function?.name || b?.name || '';
+        const aa = a?.function?.arguments || a?.arguments || '';
+        const ba = b?.function?.arguments || b?.arguments || '';
+        return an === bn && aa === ba;
+    }
+
     function clearAttachment(options = {}) {
         const { applyDefault = true } = options;
         state.attachment = null;
@@ -1788,6 +1834,22 @@
             }
         };
 
+        const renderLiveToolControls = () => {
+            if (!assistantNode || collectedToolResults.length === 0 || collectedToolCalls.length === 0) return;
+            const entry = assistantNode.closest('.chat-turn.assistant');
+            if (!entry) return;
+            renderToolCallControls(entry, collectedToolCalls, collectedToolResults);
+        };
+
+        const renderPendingToolControls = () => {
+            if (!assistantNode || roundToolCalls.length === 0) return;
+            const entry = assistantNode.closest('.chat-turn.assistant');
+            if (!entry) return;
+            const pendingCalls = normalizeToolCalls(roundToolCalls);
+            const visibleCalls = appendToolCalls(collectedToolCalls, pendingCalls);
+            renderToolCallControls(entry, visibleCalls, collectedToolResults, pendingCalls);
+        };
+
         const processLine = line => {
             const trimmed = (line || '').trim();
             if (!trimmed) return false;
@@ -1797,17 +1859,23 @@
             if (!clean) return false;
             try {
                 const parsed = JSON.parse(clean);
+                let toolMetadataChanged = false;
                 applySearchAttachment(parsed);
                 if (Array.isArray(parsed['tool-calls']) && parsed['tool-calls'].length > 0) {
                     const injectedToolCalls = normalizeToolCalls(parsed['tool-calls']);
                     if (injectedToolCalls.length > 0) {
-                        collectedToolCalls = injectedToolCalls;
+                        collectedToolCalls = appendToolCalls(collectedToolCalls, injectedToolCalls);
                         roundToolCalls = [];
+                        toolMetadataChanged = true;
                     }
                 }
                 if (Array.isArray(parsed['tool-results']) && parsed['tool-results'].length > 0) {
                     const normalizedResults = normalizeToolResults(parsed['tool-results']);
                     collectedToolResults = collectedToolResults.concat(normalizedResults);
+                    toolMetadataChanged = true;
+                }
+                if (toolMetadataChanged) {
+                    renderLiveToolControls();
                 }
                 const choice = parsed?.choices?.[0];
                 const delta = choice?.delta;
@@ -1815,6 +1883,7 @@
                 const toolDelta = delta?.tool_calls;
                 if (toolDelta) {
                     roundToolCalls = mergeToolCalls(roundToolCalls, toolDelta);
+                    renderPendingToolControls();
                 }
                 if (delta?.function_call) {
                     roundToolCalls = mergeToolCalls(roundToolCalls, [{
@@ -1822,6 +1891,7 @@
                         type: 'function',
                         function: delta.function_call
                     }]);
+                    renderPendingToolControls();
                 }
                 if (finishReason === 'tool_calls' || finishReason === 'function_call') {
                     const finalized = normalizeToolCalls(roundToolCalls);
@@ -2134,6 +2204,19 @@
             }));
     }
 
+    function appendToolCalls(existing, incoming) {
+        const result = Array.isArray(existing) ? [...existing] : [];
+        if (!Array.isArray(incoming)) return result;
+        for (const call of incoming) {
+            if (!call) continue;
+            const id = call.id || '';
+            if (id && result.some(existingCall => existingCall && existingCall.id === id)) continue;
+            if (!id && result.some(existingCall => sameToolCall(existingCall, call))) continue;
+            result.push(call);
+        }
+        return result;
+    }
+
     function mergeToolCalls(existing, deltaCalls) {
         const result = Array.isArray(existing) ? [...existing] : [];
         if (!Array.isArray(deltaCalls)) return result;
diff --git a/source/net/yacy/ai/ToolProvider.java b/source/net/yacy/ai/ToolProvider.java
index f03d1f327..83058f315 100644
--- a/source/net/yacy/ai/ToolProvider.java
+++ b/source/net/yacy/ai/ToolProvider.java
@@ -43,6 +43,7 @@ import net.yacy.ai.tools.SearchTool;
 import net.yacy.ai.tools.SelfReflectTool;
 import net.yacy.ai.tools.TableOpsTool;
 import net.yacy.ai.tools.UnitConverterTool;
+import net.yacy.ai.tools.UpdatePlanTool;
 import net.yacy.ai.tools.WebFetchTool;
 import net.yacy.ai.tools.WikipediaLinkCreatorTool;
 import net.yacy.search.Switchboard;
@@ -74,6 +75,7 @@ public final class ToolProvider {
             new UnitConverterTool(),
             new HttpJsonTool(),
             new TableOpsTool(),
+            new UpdatePlanTool(),
             new SelfReflectTool(),
             new ChitChatTool()
     );
diff --git a/source/net/yacy/ai/tools/UpdatePlanTool.java b/source/net/yacy/ai/tools/UpdatePlanTool.java
new file mode 100644
index 000000000..86df52c64
--- /dev/null
+++ b/source/net/yacy/ai/tools/UpdatePlanTool.java
@@ -0,0 +1,155 @@
+/**
+ *  UpdatePlanTool
+ *  Copyright 2026 by Michael Peter Christen
+ *  First released 06.02.2026 at https://yacy.net
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public License
+ *  along with this program in the file lgpl21.txt
+ *  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.ai.tools;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import net.yacy.ai.ToolHandler;
+
+/**
+ * A conversation-planning tool for LLM agents.
+ * <p>
+ * This tool intentionally does not persist plan state itself. In the simplest
+ * integration, the chat transcript is the source of truth: the client stores
+ * tool calls and tool results as part of the conversation, finds the latest
+ * successful {@code update_plan} result, and renders that normalized plan as
+ * the current state. No separate plan API is required for that model.
+ * <p>
+ * Recommended usage: each call should contain the complete current plan
+ * snapshot, not a partial patch. This keeps transcript replay, UI rendering,
+ * and history compaction straightforward because the latest accepted tool
+ * result is authoritative.
+ */
+public class UpdatePlanTool implements ToolHandler {
+
+    private static final String NAME = "update_plan";
+    private static final String STATUS_PENDING = "pending";
+    private static final String STATUS_IN_PROGRESS = "in_progress";
+    private static final String STATUS_COMPLETED = "completed";
+
+    @Override
+    public JSONObject definition() throws JSONException {
+        JSONObject tool = new JSONObject(true);
+        tool.put("type", "function");
+        JSONObject fn = new JSONObject(true);
+        fn.put("name", NAME);
+        fn.put("description", "Update the visible task plan. Use this to show progress on multi-step work. At most one plan item should be in_progress at a time.");
+
+        JSONObject params = new JSONObject(true);
+        params.put("type", "object");
+        JSONObject props = new JSONObject(true);
+
+        JSONObject explanation = new JSONObject(true);
+        explanation.put("type", "string");
+        explanation.put("description", "Optional short explanation of the current plan or why it changed.");
+        props.put("explanation", explanation);
+
+        JSONObject step = new JSONObject(true);
+        step.put("type", "string");
+        step.put("description", "Short description of the task step.");
+
+        JSONObject status = new JSONObject(true);
+        status.put("type", "string");
+        status.put("enum", new JSONArray().put(STATUS_PENDING).put(STATUS_IN_PROGRESS).put(STATUS_COMPLETED));
+        status.put("description", "Current status of this step.");
+
+        JSONObject itemProps = new JSONObject(true);
+        itemProps.put("step", step);
+        itemProps.put("status", status);
+
+        JSONObject item = new JSONObject(true);
+        item.put("type", "object");
+        item.put("properties", itemProps);
+        item.put("required", new JSONArray().put("step").put("status"));
+        item.put("additionalProperties", false);
+
+        JSONObject plan = new JSONObject(true);
+        plan.put("type", "array");
+        plan.put("description", "Ordered list of plan items.");
+        plan.put("items", item);
+        props.put("plan", plan);
+
+        params.put("properties", props);
+        params.put("required", new JSONArray().put("plan"));
+        params.put("additionalProperties", false);
+        fn.put("parameters", params);
+        tool.put("function", fn);
+        return tool;
+    }
+
+    @Override
+    public int maxCallsPerTurn() {
+        return 10;
+    }
+
+    @Override
+    public String execute(String arguments) {
+        final JSONObject args;
+        try {
+            args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments);
+        } catch (JSONException e) {
+            return ToolHandler.errorJson("Invalid arguments JSON");
+        }
+
+        final JSONArray plan = args.optJSONArray("plan");
+        if (plan == null) return ToolHandler.errorJson("Missing plan");
+
+        final JSONArray normalizedPlan = new JSONArray();
+        int inProgressCount = 0;
+        try {
+            for (int i = 0; i < plan.length(); i++) {
+                final JSONObject item = plan.optJSONObject(i);
+                if (item == null) return ToolHandler.errorJson("Plan item at index " + i + " must be an object");
+
+                final String step = item.optString("step", "").trim();
+                if (step.isEmpty()) return ToolHandler.errorJson("Plan item at index " + i + " is missing step");
+
+                final String status = item.optString("status", "").trim();
+                if (!isValidStatus(status)) return ToolHandler.errorJson("Invalid status at index " + i + ": " + status);
+                if (STATUS_IN_PROGRESS.equals(status)) inProgressCount++;
+
+                final JSONObject normalizedItem = new JSONObject(true);
+                normalizedItem.put("step", step);
+                normalizedItem.put("status", status);
+                normalizedPlan.put(normalizedItem);
+            }
+
+            if (inProgressCount > 1) {
+                return ToolHandler.errorJson("Only one plan item can be in_progress");
+            }
+
+            final JSONObject result = new JSONObject(true);
+            result.put("tool", NAME);
+            result.put("accepted", true);
+            result.put("step_count", normalizedPlan.length());
+            result.put("in_progress_count", inProgressCount);
+            return result.toString();
+        } catch (JSONException e) {
+            return ToolHandler.errorJson("Failed to build update_plan response");
+        }
+    }
+
+    private static boolean isValidStatus(final String status) {
+        return STATUS_PENDING.equals(status) || STATUS_IN_PROGRESS.equals(status) || STATUS_COMPLETED.equals(status);
+    }
+}
diff --git a/source/net/yacy/htroot/ViewFile.java b/source/net/yacy/htroot/ViewFile.java
index 25d9d0072..0ec25cf4d 100644
--- a/source/net/yacy/htroot/ViewFile.java
+++ b/source/net/yacy/htroot/ViewFile.java
@@ -279,13 +279,14 @@ public class ViewFile {
                 final String content = document.getTextString();

                 // content = wikiCode.replaceHTML(content); // added by Marc Nause

                 prop.put("viewMode", VIEW_MODE_AS_PARSED_TEXT);

-                prop.put("viewMode_title", document.dc_title());

-                prop.put("viewMode_creator", document.dc_creator());

-                prop.put("viewMode_subject", document.dc_subject(','));

-                prop.put("viewMode_description", document.dc_description().length == 0 ? new String[]{""} : document.dc_description());

-                prop.put("viewMode_publisher", document.dc_publisher());

-                prop.put("viewMode_format", document.dc_format());

-                prop.put("viewMode_identifier", document.dc_identifier());

+                prop.putHTML("viewMode_title", document.dc_title());

+                prop.putHTML("viewMode_creator", document.dc_creator());

+                prop.putHTML("viewMode_subject", document.dc_subject(','));

+                final String[] descs = document.dc_description();

+                prop.putHTML("viewMode_description", descs.length == 0 ? "" : String.join("; ", descs));

+                prop.putHTML("viewMode_publisher", document.dc_publisher());

+                prop.putHTML("viewMode_format", document.dc_format());

+                prop.putHTML("viewMode_identifier", document.dc_identifier());

                 prop.put("viewMode_source", url.toNormalform(false));

                 prop.put("viewMode_lat", document.lat());

                 prop.put("viewMode_lon", document.lon());

diff --git a/source/net/yacy/htroot/yacychat.java b/source/net/yacy/htroot/yacychat.java
index 0037d4f04..74c081dac 100644
--- a/source/net/yacy/htroot/yacychat.java
+++ b/source/net/yacy/htroot/yacychat.java
@@ -29,7 +29,14 @@ public class yacychat {
 
     // system prompt comes from configuration; default is empty
     final String systemPrompt = sb.getConfig("ai.system-prompt", net.yacy.http.servlets.RAGProxyServlet.LLM_SYSTEM_PROMPT_DEFAULT);
-    prop.put("system_prompt", systemPrompt);
+    // escape for safe embedding in a JS single-quoted string literal
+    final String systemPromptJs = systemPrompt
+        .replace("\\", "\\\\")
+        .replace("'", "\\'")
+        .replace("\r\n", "\\n")
+        .replace("\n", "\\n")
+        .replace("\r", "\\n");
+    prop.put("system_prompt", systemPromptJs);
     prop.put("topmenu", sb.getConfigBool("ai.shield.show-chat-link", false) ? (sb.getConfigBool("publicTopmenu", true) ? 1 : 0) : 2);
 
     String promoteChatPageGreeting = env.getConfig("promoteChatPageGreeting", "");
diff --git a/source/net/yacy/htroot/yacysearch.java b/source/net/yacy/htroot/yacysearch.java
index dbe77d1ec..05647672d 100644
--- a/source/net/yacy/htroot/yacysearch.java
+++ b/source/net/yacy/htroot/yacysearch.java
@@ -188,7 +188,7 @@ public class yacysearch {
             prop.put("offset", "0");
             prop.put("resource", "global");
             prop.put("urlmaskfilter", (post == null) ? ".*" : post.get("urlmaskfilter", ".*"));
-            prop.put("prefermaskfilter", (post == null) ? "" : post.get("prefermaskfilter", ""));
+            prop.putHTML("prefermaskfilter", (post == null) ? "" : post.get("prefermaskfilter", ""));
             prop.put("indexof", "off");
             prop.put("constraint", "");
             prop.put("depth", "0");
diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java
index 582f0e02f..7bc87f39d 100644
--- a/source/net/yacy/search/query/SearchEvent.java
+++ b/source/net/yacy/search/query/SearchEvent.java
@@ -354,7 +354,7 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
             this.imagePageCounter = query.offset;
         }
         this.loader = loader;
-        this.nodeStack = new WeakPriorityBlockingQueue<>(max_results_node, false);
+        this.nodeStack = new WeakPriorityBlockingQueue<>(max_results_node + (query != null ? query.offset + query.itemsPerPage() : 0), false);
         this.maxExpectedRemoteReferences = new AtomicInteger(0);
         this.expectedRemoteReferences = new AtomicInteger(0);
         this.excludeintext_image = Switchboard.getSwitchboard().getConfigBool("search.excludeintext.image", true);