diff options
21 files changed, 4687 insertions, 617 deletions
diff --git a/htroot/yacychat.html b/htroot/yacychat.html index 50d77f6e3..76ae0ebf5 100644 --- a/htroot/yacychat.html +++ b/htroot/yacychat.html @@ -83,6 +83,21 @@ color: #ffffff; border: 1px solid #2c3e50; } + + /* --- TOOL: Tool Call Output --- */ + .chat-turn.tool { + background-color: #f0f7e8; + border-left-width: 5px; + border-left-style: solid; + border-color: #b7d7a8; + color: #24421a; + } + + .chat-turn.tool legend { + background: #3d6b2f; + color: #ffffff; + border: 1px solid #3d6b2f; + } /* --- SYSTEM: Log Entry Style --- */ .chat-turn.system { @@ -147,6 +162,68 @@ font-size: 0.85rem; } + .toolcalls-popover { + position: absolute; + z-index: 20; + top: 24px; + right: 8px; + min-width: 280px; + max-width: min(560px, calc(100vw - 40px)); + max-height: 380px; + overflow: auto; + border: 1px solid #c3cbd9; + border-radius: 6px; + background: #eef2f7; + box-shadow: 3px 3px 0px rgba(0,0,0,0.15); + padding: 8px 10px; + } + + .toolcalls-popover-header { + font-weight: 700; + margin-bottom: 6px; + } + + .toolcalls-list { + margin-top: 6px; + display: flex; + flex-direction: column; + gap: 6px; + } + + .toolcall-item { + border: 1px solid #d6dbe5; + border-radius: 4px; + background: #f5f7fb; + padding: 6px 8px; + } + + .toolcall-name { + font-weight: 700; + margin-bottom: 4px; + } + + .toolcall-args { + margin: 0; + white-space: pre-wrap; + word-break: break-word; + font-size: 0.95rem; + line-height: 1.4; + } + + .toolcall-result-title { + margin-top: 6px; + margin-bottom: 4px; + font-weight: 700; + } + + .toolcall-result { + margin: 0; + white-space: pre-wrap; + word-break: break-word; + font-size: 0.95rem; + line-height: 1.4; + } + .chat-body.markdown h1, .chat-body.markdown h2, .chat-body.markdown h3, @@ -316,7 +393,8 @@ .attachment-button, .search-button, .copy-button, - .user-trim-button { + .user-trim-button, + .toolcalls-button { width: 24px; height: 24px; border: none; @@ -335,7 +413,8 @@ .attachment-button:hover, .search-button:hover, .copy-button:hover, - .user-trim-button:hover { + .user-trim-button:hover, + .toolcalls-button:hover { background: #c3cbd9; } @@ -374,6 +453,20 @@ right: 8px; } + .toolcalls-button { + position: static; + } + + .toolcall-toolbar { + position: absolute; + top: -6px; + right: 36px; + display: inline-flex; + flex-direction: row; + gap: 4px; + align-items: center; + } + .user-delete-button { position: absolute; top: -6px; @@ -775,6 +868,7 @@ const ALLOWED_EXTENSIONS = new Set([...IMAGE_EXTENSIONS, ...TEXT_EXTENSIONS]); const FILE_ACCEPT_TYPES = [...ALLOWED_MIME_TYPES, ...ALLOWED_EXTENSIONS].join(','); + if (dom.fileInput) { dom.fileInput.setAttribute('accept', FILE_ACCEPT_TYPES); } @@ -966,18 +1060,31 @@ return result; } + function stripToolProtocolMarkup(text) { + if (!text || typeof text !== 'string') return text || ''; + // Remove legacy/model-emitted tool protocol snippets like: + // <function=name> + // <parameter=key> + // value + let cleaned = text.replace(/^\s*<function=[^>\n]+>\s*$[\s\S]*?(?=^\s*$|$)/gim, ''); + cleaned = cleaned.replace(/^\s*<parameter=[^>\n]+>\s*$[\s\S]*?(?=^\s*$|$)/gim, ''); + // Collapse excessive blank lines introduced by stripping. + cleaned = cleaned.replace(/\n{3,}/g, '\n\n'); + return cleaned; + } + function stripThinkFromContent(content) { - if (typeof content === 'string') return stripThinkBlocks(content); + if (typeof content === 'string') return stripToolProtocolMarkup(stripThinkBlocks(content)); if (Array.isArray(content)) { return content.map(part => { if (part?.type === 'text' && typeof part.text === 'string') { - return { ...part, text: stripThinkBlocks(part.text) }; + return { ...part, text: stripToolProtocolMarkup(stripThinkBlocks(part.text)) }; } return part; }); } if (content && typeof content === 'object' && typeof content.text === 'string') { - return { ...content, text: stripThinkBlocks(content.text) }; + return { ...content, text: stripToolProtocolMarkup(stripThinkBlocks(content.text)) }; } return content; } @@ -1143,7 +1250,7 @@ } function appendMessage(role, text, opts = {}) { - const { skipScroll = false, skipVisibilityUpdate = false, attachment = null, messageIndex = undefined } = opts; + const { skipScroll = false, skipVisibilityUpdate = false, attachment = null, messageIndex = undefined, toolCalls = null, toolResults = null } = opts; const entry = document.createElement('fieldset'); entry.className = `chat-turn ${role}`; if (role === 'assistant' || role === 'user') { @@ -1154,6 +1261,8 @@ legend.textContent = 'Assistant'; } else if (role === 'user') { legend.textContent = 'User'; + } else if (role === 'tool') { + legend.textContent = 'Tool'; } else { legend.textContent = 'System'; } @@ -1172,7 +1281,6 @@ attachmentContainer.appendChild(createAttachmentChip(attachment)); entry.appendChild(attachmentContainer); } - dom.messages.appendChild(entry); if (typeof opts.messageIndex === 'number') { entry.dataset.messageIndex = String(opts.messageIndex); @@ -1185,6 +1293,56 @@ copyBtn.innerHTML = '<span class="glyphicon glyphicon-copy" aria-hidden="true"></span>'; copyBtn.addEventListener('click', () => copyAssistant(body)); entry.appendChild(copyBtn); + if (Array.isArray(toolCalls) && toolCalls.length > 0) { + const toolbar = document.createElement('div'); + toolbar.className = 'toolcall-toolbar'; + const panels = []; + const resultsById = new Map(); + const resultsByName = new Map(); + if (Array.isArray(toolResults)) { + for (const result of toolResults) { + if (!result) continue; + if (result.id) resultsById.set(result.id, result); + if (result.tool_call_id) resultsById.set(result.tool_call_id, result); + const rname = result?.name || result?.function?.name; + if (rname && !resultsByName.has(rname)) resultsByName.set(rname, result); + } + } + + for (let i = 0; i < toolCalls.length; i++) { + const call = toolCalls[i]; + if (!call) continue; + const fnName = call?.function?.name || call?.name || 'tool'; + const matchedResult = (call.id && resultsById.get(call.id)) || resultsByName.get(fnName) || null; + + const toolBtn = document.createElement('button'); + toolBtn.type = 'button'; + toolBtn.className = 'toolcalls-button'; + toolBtn.title = `Tool: ${fnName}`; + toolBtn.innerHTML = '<span class="glyphicon glyphicon-wrench" aria-hidden="true"></span>'; + + const toolPanel = createToolCallPanel(call, matchedResult, toolCalls.length, i); + toolPanel.style.display = 'none'; + toolPanel.addEventListener('click', evt => evt.stopPropagation()); + entry.appendChild(toolPanel); + panels.push(toolPanel); + + toolBtn.addEventListener('click', event => { + event.stopPropagation(); + const open = toolPanel.style.display !== 'none'; + for (const panel of panels) panel.style.display = 'none'; + toolPanel.style.display = open ? 'none' : 'block'; + }); + toolbar.appendChild(toolBtn); + } + entry.appendChild(toolbar); + + document.addEventListener('click', evt => { + if (!entry.contains(evt.target)) { + for (const panel of panels) panel.style.display = 'none'; + } + }); + } } if (role === 'user' && typeof messageIndex === 'number') { const trimBtn = document.createElement('button'); @@ -1496,6 +1654,9 @@ const reader = response.body.getReader(); const decoder = new TextDecoder('utf-8'); let assistantText = ''; + let roundToolCalls = []; + let collectedToolCalls = []; + let collectedToolResults = []; let sawFirstDelta = false; let focusShown = false; let searchApplied = false; @@ -1557,7 +1718,31 @@ try { const parsed = JSON.parse(clean); applySearchAttachment(parsed); - const delta = parsed?.choices?.[0]?.delta?.content; + if (Array.isArray(parsed['tool-results']) && parsed['tool-results'].length > 0) { + collectedToolResults = collectedToolResults.concat(normalizeToolResults(parsed['tool-results'])); + } + const choice = parsed?.choices?.[0]; + const delta = choice?.delta; + const finishReason = choice?.finish_reason; + const toolDelta = delta?.tool_calls; + if (toolDelta) { + roundToolCalls = mergeToolCalls(roundToolCalls, toolDelta); + } + if (delta?.function_call) { + roundToolCalls = mergeToolCalls(roundToolCalls, [{ + index: 0, + type: 'function', + function: delta.function_call + }]); + } + if (finishReason === 'tool_calls' || finishReason === 'function_call') { + const finalized = normalizeToolCalls(roundToolCalls); + if (finalized.length) { + collectedToolCalls = collectedToolCalls.concat(finalized); + } + roundToolCalls = []; + } + const contentDelta = delta?.content; if (delta) { if (!sawFirstDelta) { sawFirstDelta = true; @@ -1567,7 +1752,10 @@ onFirstToken(); } } - assistantText += delta; + if (contentDelta) { + assistantText += contentDelta; + assistantText = stripToolProtocolMarkup(assistantText); + } if (!thinkAutoClosed && hasThinkCloseOutsideFences(assistantText)) { thinkAutoClosed = true; if (assistantNode) { @@ -1637,7 +1825,22 @@ : explicitThinkOpen === 'false' ? false : (thinkAutoClosed && !thinkAutoCloseFired); - state.messages.push({ role: 'assistant', content: assistantText, thinkOpen: storedThinkOpen }); + const normalizedToolCalls = normalizeToolCalls(collectedToolCalls.concat(roundToolCalls)); + const assistantMessage = { + role: 'assistant', + content: assistantText, + thinkOpen: storedThinkOpen + }; + if (normalizedToolCalls.length) { + assistantMessage.tool_calls = normalizedToolCalls; + if (collectedToolResults.length) { + assistantMessage.tool_results = collectedToolResults; + } + assistantMessage.display = assistantText && assistantText.trim() + ? assistantText + : `Tool call: ${normalizedToolCalls.map(call => call?.function?.name || 'tool').join(', ')}`; + } + state.messages.push(assistantMessage); console.debug('[yacychat] response chars', assistantText.length, 'total ms', Math.round(performance.now() - requestStart)); state.assistantSeen = true; persistConversation(); @@ -1691,12 +1894,21 @@ } if (message.role === 'assistant') { if (textOnly || typeof content === 'string') { - content = stripThinkBlocks(String(content || '')); + content = stripToolProtocolMarkup(stripThinkBlocks(String(content || ''))); } else { content = stripThinkFromContent(content); } } const sanitized = { role: message.role, content }; + if (message.tool_calls) { + sanitized.tool_calls = message.tool_calls; + } + if (message.tool_call_id) { + sanitized.tool_call_id = message.tool_call_id; + } + if (message.name) { + sanitized.name = message.name; + } if (message.search === true) { sanitized.search = SEARCH_DEFAULTS.local; } else if (message.search === SEARCH_DEFAULTS.local || message.search === SEARCH_DEFAULTS.global) { @@ -1726,6 +1938,12 @@ return ''; } + function messageDisplayText(message) { + if (!message) return ''; + if (typeof message.display === 'string') return message.display; + return messageText(message.content); + } + function renderConversation(options = {}) { const { skipScroll = false } = options; closeAttachmentPopover(); @@ -1734,7 +1952,7 @@ for (let i = 0; i < state.messages.length; i++) { const msg = state.messages[i]; if (msg.role === 'system' && !state.showSystem) continue; - appendMessage(msg.role, messageText(msg.content), { skipScroll: true, skipVisibilityUpdate: true, messageIndex: i, attachment: msg.attachment }); + appendMessage(msg.role, messageDisplayText(msg), { skipScroll: true, skipVisibilityUpdate: true, messageIndex: i, attachment: msg.attachment, toolCalls: msg.tool_calls, toolResults: msg.tool_results }); } updateClearChatVisibility(); updateSystemToggleButton(); @@ -1743,6 +1961,119 @@ } } + function normalizeToolCalls(calls) { + if (!Array.isArray(calls)) return []; + const stamp = Date.now(); + return calls + .filter(Boolean) + .map((call, index) => { + const fn = call.function || {}; + return { + id: call.id || `toolcall_${stamp}_${index}`, + type: call.type || 'function', + function: { + name: fn.name || call.name || 'unknown', + arguments: typeof fn.arguments === 'string' ? fn.arguments : (call.arguments || '') + } + }; + }); + } + + function normalizeToolResults(results) { + if (!Array.isArray(results)) return []; + const stamp = Date.now(); + return results + .filter(Boolean) + .map((result, index) => ({ + id: result.id || result.tool_call_id || `toolres_${stamp}_${index}`, + tool_call_id: result.tool_call_id || result.id || '', + name: result.name || result?.function?.name || 'unknown', + content: typeof result.content === 'string' + ? result.content + : (result?.function?.content ? String(result.function.content) : '') + })); + } + + function mergeToolCalls(existing, deltaCalls) { + const result = Array.isArray(existing) ? [...existing] : []; + if (!Array.isArray(deltaCalls)) return result; + for (const delta of deltaCalls) { + if (!delta) continue; + const index = readToolCallIndex(delta); + if (index === null || index < 0) continue; + if (!result[index]) { + result[index] = { id: '', type: delta.type || 'function', function: { name: '', arguments: '' } }; + } + const target = result[index]; + if (delta.id) target.id = delta.id; + if (delta.type) target.type = delta.type; + if (delta.function?.name) target.function.name = delta.function.name; + if (typeof delta.function?.arguments === 'string' && delta.function.arguments.length > 0) { + target.function.arguments = (target.function.arguments || '') + delta.function.arguments; + } + } + return result; + } + + function readToolCallIndex(delta) { + if (!delta || typeof delta !== 'object' || !Object.prototype.hasOwnProperty.call(delta, 'index')) { + return null; + } + return Number.isInteger(delta.index) ? delta.index : null; + } + + function formatToolArguments(raw) { + if (typeof raw !== 'string' || !raw.trim()) return '(no arguments)'; + try { + return JSON.stringify(JSON.parse(raw), null, 2); + } catch (err) { + return raw; + } + } + + function createToolCallPanel(call, result, totalCount, index) { + const panel = document.createElement('div'); + panel.className = 'toolcalls-popover'; + panel.style.right = `${36 + (index * 28)}px`; + + const header = document.createElement('div'); + header.className = 'toolcalls-popover-header'; + header.textContent = `Tool ${index + 1}/${totalCount}`; + panel.appendChild(header); + + const list = document.createElement('div'); + list.className = 'toolcalls-list'; + const item = document.createElement('div'); + item.className = 'toolcall-item'; + + const fnName = call?.function?.name || call?.name || 'unknown'; + const name = document.createElement('div'); + name.className = 'toolcall-name'; + name.textContent = fnName; + item.appendChild(name); + + const args = document.createElement('pre'); + args.className = 'toolcall-args'; + args.textContent = formatToolArguments(call?.function?.arguments || call?.arguments || ''); + item.appendChild(args); + + if (result) { + const responseTitle = document.createElement('div'); + responseTitle.className = 'toolcall-result-title'; + responseTitle.textContent = 'Response'; + item.appendChild(responseTitle); + + const responseBody = document.createElement('pre'); + responseBody.className = 'toolcall-result'; + responseBody.textContent = formatToolArguments(result?.content || result?.function?.content || ''); + item.appendChild(responseBody); + } + + list.appendChild(item); + panel.appendChild(list); + return panel; + } + function persistConversation() { try { const messages = state.messages.map(msg => { diff --git a/source/net/yacy/ai/LLM.java b/source/net/yacy/ai/LLM.java index 7987cbb29..cf12265ef 100644 --- a/source/net/yacy/ai/LLM.java +++ b/source/net/yacy/ai/LLM.java @@ -1,5 +1,5 @@ /** - * LLMEndpoint + * LLM * Copyright 2024 by Michael Peter Christen * First released 17.05.2024 at https://yacy.net * diff --git a/source/net/yacy/ai/LLMSwarm.java b/source/net/yacy/ai/LLMSwarm.java index 9aa3286b2..67508c236 100644 --- a/source/net/yacy/ai/LLMSwarm.java +++ b/source/net/yacy/ai/LLMSwarm.java @@ -1,5 +1,5 @@ /** - * OllamaSwarm.java + * LLMSwarm.java * Copyright 2025 by Michael Peter Christen * First released 01.06.2025 at https://yacy.net * diff --git a/source/net/yacy/ai/RAGAugmentor.java b/source/net/yacy/ai/RAGAugmentor.java new file mode 100644 index 000000000..d1dbb18db --- /dev/null +++ b/source/net/yacy/ai/RAGAugmentor.java @@ -0,0 +1,614 @@ +/** + * RAGAugmentor + * Copyright 2026 by Michael Peter Christen + * First released 06.02.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.ai; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrException; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import net.yacy.cora.document.analysis.Classification; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; +import net.yacy.cora.federate.solr.SolrType; +import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector; +import net.yacy.cora.federate.yacy.CacheStrategy; +import net.yacy.cora.lod.vocabulary.Tagging; +import net.yacy.cora.protocol.ClientIdentification; +import net.yacy.cora.util.ConcurrentLog; +import net.yacy.kelondro.data.meta.URIMetadataNode; +import net.yacy.search.Switchboard; +import net.yacy.search.SwitchboardConstants; +import net.yacy.search.query.QueryGoal; +import net.yacy.search.query.QueryModifier; +import net.yacy.search.query.QueryParams; +import net.yacy.search.query.SearchEvent; +import net.yacy.search.query.SearchEventCache; +import net.yacy.search.ranking.RankingProfile; +import net.yacy.search.schema.CollectionSchema; +import net.yacy.search.snippet.TextSnippet; + +/** + * Utility methods that enrich prompts/responses with search-derived context for + * Retrieval-Augmented Generation (RAG). + * <p> + * This class provides: + * <ul> + * <li>local Solr-backed search result extraction</li> + * <li>global YaCy network search extraction</li> + * <li>markdown condensation of search results</li> + * <li>token intersection helpers for query boosting</li> + * <li>snippet scoring/selection utilities</li> + * </ul> + */ +public final class RAGAugmentor { + + /** + * Utility class; not instantiable. + */ + private RAGAugmentor() {} + + /** + * Executes local index search with default boost terms. + * + * @param query query string + * @param count max number of results + * @param includeSnippet include text snippet field in response objects + * @return JSON array with {@code url,title[,text]} entries + */ + public static JSONArray searchResults(String query, int count, final boolean includeSnippet) { + return searchResults(query, count, includeSnippet, new LinkedHashSet<>()); + } + + /** + * Executes local Solr search with optional dynamic boost terms. + * + * @param query query string + * @param count max number of results + * @param includeSnippet include text snippets from indexed text field + * @param boostTerms optional overlap terms used to bias ranking + * @return JSON array with normalized search result objects + */ + public static JSONArray searchResults(String query, int count, final boolean includeSnippet, final Set<String> boostTerms) { + final JSONArray results = new JSONArray(); + if (query == null || query.length() == 0 || count == 0) return results; + Switchboard sb = Switchboard.getSwitchboard(); + EmbeddedSolrConnector connector = sb.index.fulltext().getDefaultEmbeddedConnector(); + final SolrQuery params = new SolrQuery(); + // Base query and parser setup. + params.setQuery(query); + params.set("defType", "edismax"); + // Static field boosts favor title/headings over body text. + params.set("qf", + CollectionSchema.title.getSolrFieldName() + "^3 " + + CollectionSchema.text_t.getSolrFieldName() + "^1 " + + CollectionSchema.sku.getSolrFieldName() + "^0.5 " + + CollectionSchema.h1_txt.getSolrFieldName() + "^2"); + params.set("pf", + CollectionSchema.title.getSolrFieldName() + "^5 " + + CollectionSchema.text_t.getSolrFieldName() + "^2"); + final List<String> bqParts = new ArrayList<>(); + if (boostTerms != null && !boostTerms.isEmpty()) { + // Apply weak boosts to overlap terms, keeping lexical query dominant. + for (String term : boostTerms) { + if (term == null || term.isEmpty()) continue; + bqParts.add(CollectionSchema.title.getSolrFieldName() + ":" + term + "^0.5"); + bqParts.add(CollectionSchema.h1_txt.getSolrFieldName() + ":" + term + "^0.4"); + bqParts.add(CollectionSchema.text_t.getSolrFieldName() + ":" + term + "^0.2"); + } + } + // Slightly prefer archive/container file extensions in this use case. + bqParts.add("(" + CollectionSchema.url_file_ext_s.getSolrFieldName() + ":(zip rar 7z tar gz bz2 xz tgz))^0.1"); + params.set("bq", String.join(" ", bqParts)); + params.setRows(count); + params.setStart(0); + params.setFacet(false); + params.clearSorts(); + // Fetch only fields needed for tool / markdown rendering. + params.setFields( + CollectionSchema.sku.getSolrFieldName(), CollectionSchema.title.getSolrFieldName(), CollectionSchema.text_t.getSolrFieldName(), + CollectionSchema.description_txt.getSolrFieldName(), CollectionSchema.keywords.getSolrFieldName(), CollectionSchema.synonyms_sxt.getSolrFieldName(), + CollectionSchema.h1_txt.getSolrFieldName(), CollectionSchema.h2_txt.getSolrFieldName(), CollectionSchema.h3_txt.getSolrFieldName(), + CollectionSchema.h4_txt.getSolrFieldName(), CollectionSchema.h5_txt.getSolrFieldName(), CollectionSchema.h6_txt.getSolrFieldName() + ); + params.setIncludeScore(true); + params.set("df", CollectionSchema.text_t.getSolrFieldName()); + + try { + final SolrDocumentList sdl = connector.getDocumentListByParams(params); + Iterator<SolrDocument> i = sdl.iterator(); + while (i.hasNext()) { + try { + SolrDocument doc = i.next(); + final JSONObject result = new JSONObject(true); + String url = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName()); + result.put("url", url == null ? "" : url.trim()); + String title = getOneString(doc, CollectionSchema.title); + result.put("title", title == null ? "" : title.trim()); + if (includeSnippet) { + // Use indexed text body as quick snippet source. + String text = (String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName()); + result.put("text", limitSnippet(text == null ? "" : text.trim(), 2000)); + } + results.put(result); + } catch (JSONException e) { + } + } + return results; + } catch (SolrException | IOException e) { + return results; + } + } + + /** + * Renders search results as compact markdown context using local-search mode. + * + * @param query query string + * @param count max number of search rows + * @param global when true, use global YaCy search + * @return markdown context block + */ + public static String searchResultsAsMarkdown(String query, int count, boolean global) { + return searchResultsAsMarkdown(query, count, global, new LinkedHashSet<>()); + } + + /** + * Renders search results as compact markdown and applies snippet ranking to + * reduce noise. + * + * @param query query string + * @param count max number of search rows + * @param global when true, use global YaCy search; otherwise local Solr + * @param boostTerms optional local-search boost terms + * @return markdown formatted context used in downstream prompt augmentation + */ + public static String searchResultsAsMarkdown(String query, int count, boolean global, final Set<String> boostTerms) { + final long searchStart = System.currentTimeMillis(); + JSONArray searchResults = global ? searchResultsGlobal(query, count, true) : searchResults(query, count, true, boostTerms); + ConcurrentLog.info("RAGProxy", "searchResults=" + searchResults.length() + " global=" + global + " searchMs=" + (System.currentTimeMillis() - searchStart)); + StringBuilder sb = new StringBuilder(); + + // Convert raw rows into scoreable snippet candidates. + List<Snippet> results = new ArrayList<>(); + for (int i = 0; i < searchResults.length(); i++) { + try { + JSONObject r = searchResults.getJSONObject(i); + String title = r.optString("title", ""); + String url = r.optString("url", ""); + String text = r.optString("text", ""); + if (title.isEmpty()) title = url; + if (text.isEmpty()) text = title; + if (title.length() > 0 && text.length() > 0) { + Snippet snippet = new Snippet(query, text, url, title, 256); + if (snippet.getText().length() > 0) results.add(snippet); + } + } catch (JSONException e) {} + } + + // Lower score is better with the current tf-idf based chunk scorer. + results.sort(Comparator.comparingDouble(Snippet::getScore)); + // Keep top half to avoid overloading the model context window. + int limit = results.size() / 2; + if (results.size() > 0 && limit == 0) limit = 1; + for (int i = 0; i < limit; i++) { + Snippet snippet = results.get(i); + sb.append("## ").append(snippet.getTitle()).append("\n"); + sb.append(snippet.text).append("\n"); + if (snippet.getURL().length() > 0) sb.append("Source: ").append(snippet.getURL()).append("\n"); + sb.append("\n\n"); + } + + ConcurrentLog.info("RAGProxy", "markdownChars=" + sb.length() + " snippetCount=" + results.size()); + return sb.toString(); + } + + /** + * Executes a global/distributed YaCy search event and maps results into a + * compact JSON format. + * + * @param query query string + * @param count max number of results + * @param includeSnippet include snippet text when available + * @return JSON array with normalized result objects + */ + public static JSONArray searchResultsGlobal(String query, int count, final boolean includeSnippet) { + final JSONArray results = new JSONArray(); + if (query == null || query.length() == 0 || count == 0) return results; + final Switchboard sb = Switchboard.getSwitchboard(); + final RankingProfile ranking = sb.getRanking(); + final int timezoneOffset = 0; + final QueryModifier modifier = new QueryModifier(timezoneOffset); + // Parse modifiers and normalize effective query string. + String querystring = modifier.parse(query); + if (querystring.length() == 0) querystring = query == null ? "" : query.trim(); + if (querystring.length() == 0) return results; + final QueryGoal qg = new QueryGoal(querystring); + // Construct a standard text-domain global query. + final QueryParams theQuery = new QueryParams( + qg, + modifier, + 0, + "", + Classification.ContentDomain.TEXT, + "", + timezoneOffset, + new HashSet<Tagging.Metatag>(), + CacheStrategy.IFFRESH, + count, + 0, + ".*", + null, + null, + QueryParams.Searchdom.GLOBAL, + null, + true, + DigestURL.hosthashess(sb.getConfig("search.excludehosth", "")), + MultiProtocolURL.TLD_any_zone_filter, + null, + false, + sb.index, + ranking, + ClientIdentification.yacyIntranetCrawlerAgent.userAgent(), + 0.0d, + 0.0d, + 0.0d, + sb.getConfigSet("search.navigation")); + final SearchEvent theSearch = SearchEventCache.getEvent( + theQuery, + sb.peers, + sb.tables, + (sb.isRobinsonMode()) ? sb.clusterhashes : null, + false, + sb.loader, + (int) sb.getConfigLong( + SwitchboardConstants.REMOTESEARCH_MAXCOUNT_USER, + sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXCOUNT_DEFAULT, 10)), + sb.getConfigLong( + SwitchboardConstants.REMOTESEARCH_MAXTIME_USER, + sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000))); + final long timeout = sb.getConfigLong( + SwitchboardConstants.REMOTESEARCH_MAXTIME_USER, + sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000)); + // Wait until remote feeds are done (or timeout), then stabilize ordering. + waitForFeedingAndResort(theSearch, timeout); + for (int i = 0; i < count; i++) { + URIMetadataNode node = theSearch.oneResult(i, timeout); + if (node == null) break; + try { + final JSONObject result = new JSONObject(true); + result.put("url", node.urlstring()); + result.put("title", node.title()); + if (includeSnippet) { + // Prefer direct snippet; fall back to textSnippet, description, then text field. + String text = node.snippet(); + if (text == null || text.isEmpty()) { + TextSnippet snippet = node.textSnippet(); + if (snippet != null && snippet.exists() && !snippet.getErrorCode().fail()) text = snippet.getLineRaw(); + } + if (text == null || text.isEmpty()) text = firstFieldString(node.getFieldValue(CollectionSchema.description_txt.getSolrFieldName())); + if (text == null || text.isEmpty()) text = firstFieldString(node.getFieldValue(CollectionSchema.text_t.getSolrFieldName())); + result.put("text", limitSnippet(text == null ? "" : text.trim(), 2000)); + } + results.put(result); + } catch (JSONException e) { + } + } + return results; + } + + /** + * Uses an LLM list schema prompt to compute likely discriminative search + * terms for a user prompt. + * + * @param llm configured LLM backend + * @param model target model name + * @param prompt user prompt to analyze + * @return space-separated lowercase term list or {@code null} on failure + */ + public static String searchWordsForPrompt(LLM llm, String model, String prompt) { + final String question = prompt == null ? "" : prompt; + if (llm == null || model == null || model.isEmpty()) return null; + try { + LLM.Context context = new LLM.Context("\n\nYou may receive additional expert knowledge in the user prompt after a 'Additional Information' headline to enhance your knowledge. Use it only if applicable."); + context.addPrompt(question); + Set<String> singlewords = new LinkedHashSet<>(); + String[] a = LLM.stringsFromChat(llm.chat(model, context, LLM.listSchema, 200)); + if (a == null || a.length == 0) return null; + for (String s: a) { + if (s == null) continue; + // Flatten model output into unique lowercased tokens. + for (String t: s.split(" ")) if (!t.isEmpty()) singlewords.add(t.toLowerCase()); + } + if (singlewords.isEmpty()) return null; + StringBuilder query = new StringBuilder(); + for (String s: singlewords) query.append(s).append(' '); + String querys = query.toString().trim(); + if (querys.length() == 0) return null; + return querys; + } catch (IOException | JSONException e) { + e.printStackTrace(); + return null; + } + } + + /** + * Computes token overlap between original prompt and computed query terms. + * + * @param originalPrompt raw user prompt + * @param computedQuery generated query terms + * @param maxTerms max terms to keep; {@code <=0} means unlimited + * @return cleaned ordered overlap set + */ + public static Set<String> intersectTokens(String originalPrompt, String computedQuery, int maxTerms) { + Set<String> promptTerms = querySet(originalPrompt == null ? "" : originalPrompt); + Set<String> queryTerms = querySet(computedQuery == null ? "" : computedQuery); + Set<String> intersection = new LinkedHashSet<>(); + for (String term : promptTerms) { + if (!queryTerms.contains(term)) continue; + final String cleaned = cleanToken(term); + if (cleaned.isEmpty()) continue; + intersection.add(cleaned); + if (maxTerms > 0 && intersection.size() >= maxTerms) break; + } + return intersection; + } + + /** + * Splits text into sentence-aware chunks around a target max length. + * + * @param text source text + * @param len approximate chunk length + * @return ordered chunks + */ + public static List<String> slicer(String text, int len) { + List<String> result = new ArrayList<>(); + if (text == null || len <= 0) return result; + + int start = 0; + while (start < text.length()) { + int end = Math.min(start + len, text.length()); + // Extend to sentence boundary when possible. + while (end < text.length()) { + char ch = text.charAt(end - 1); + if ((ch == '.' || ch == '?' || ch == '!') && Character.isWhitespace(text.charAt(end))) break; + end++; + } + result.add(text.substring(start, end)); + start = end; + } + return result; + } + + /** + * Converts a query string into a normalized token set. + * + * @param query raw query text + * @return lowercase token set + */ + private static Set<String> querySet(String query) { + return Arrays.stream(query.trim().toLowerCase().split("\\s+")) + .map(String::toLowerCase) + .filter(word -> !word.isEmpty()) + .collect(Collectors.toSet()); + } + + /** + * Removes non-alphanumeric characters and enforces minimal token length. + * + * @param term source token + * @return cleaned lowercase token or empty string + */ + private static String cleanToken(String term) { + if (term == null) return ""; + String cleaned = term.replaceAll("[^A-Za-z0-9]", ""); + if (cleaned.length() < 2) return ""; + return cleaned.toLowerCase(); + } + + /** + * Truncates snippet text to a maximum character count. + * + * @param text input text + * @param maxChars limit + * @return truncated or original text + */ + private static String limitSnippet(String text, int maxChars) { + if (text == null) return ""; + if (maxChars <= 0 || text.length() <= maxChars) return text; + return text.substring(0, maxChars); + } + + /** + * Returns first non-null string from a field value that may be scalar or + * collection. + * + * @param value field value + * @return first string representation or empty string + */ + private static String firstFieldString(Object value) { + if (value == null) return ""; + if (value instanceof Collection) { + for (Object item : (Collection<?>) value) if (item != null) return item.toString(); + return ""; + } + return value.toString(); + } + + /** + * Waits for global search feeding completion and then resorts cached results. + * + * @param search active search event + * @param timeoutMs max wait time + */ + private static void waitForFeedingAndResort(SearchEvent search, long timeoutMs) { + if (search == null || timeoutMs <= 0) return; + final long end = System.currentTimeMillis() + timeoutMs; + while (!search.isFeedingFinished() && System.currentTimeMillis() < end) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + search.resortCachedResults(); + } + + /** + * Reads a single string value from a (possibly multivalued) Solr field. + * + * @param doc source Solr document + * @param field schema field descriptor + * @return first string value or empty string + */ + private static String getOneString(SolrDocument doc, CollectionSchema field) { + assert field.isMultiValued(); + assert field.getType() == SolrType.string || field.getType() == SolrType.text_general; + Object r = doc.getFieldValue(field.getSolrFieldName()); + if (r == null) return ""; + if (r instanceof ArrayList) { + return (String) ((ArrayList<?>) r).get(0); + } + return r.toString(); + } + + /** + * Represents one candidate snippet around a search result and stores its + * relevance score relative to the query. + */ + private static class Snippet { + private String text, url, title; + private double score; + + /** + * Scores text chunks and keeps the best chunk plus direct neighbors to + * preserve context continuity. + * + * @param query query text + * @param text source document/snippet text + * @param url source URL + * @param title source title + * @param maxChunkLength target chunk size + */ + public Snippet(String query, String text, String url, String title, int maxChunkLength) { + this.url = url; + this.title = title; + this.score = 0.0; + + if (text == null || text.isEmpty() || maxChunkLength <= 0 || query == null) { + this.text = ""; + return; + } + + List<String> chunks = slicer(text, maxChunkLength); + if (chunks.isEmpty()) { + this.text = ""; + return; + } + List<String> chunksLowerCase = new ArrayList<>(chunks.size()); + // Cache lowercase chunks so token comparisons are case-insensitive. + for (String chunk: chunks) chunksLowerCase.add(chunk.toLowerCase()); + + Set<String> queryWordSet = querySet(query); + if (queryWordSet.isEmpty()) { + this.text = ""; + return; + } + + int totalChunks = chunksLowerCase.size(); + Map<String, Double> idf = new HashMap<>(); + for (String word: queryWordSet) { + int docFreq = 0; + for (String chunk: chunksLowerCase) { + if (chunk.contains(word)) docFreq++; + } + // Smoothed IDF to avoid divide-by-zero and extreme values. + idf.put(word, Math.log((double) totalChunks / (docFreq + 1)) + 1); + } + + Map<Integer, Double> chunkScores = new HashMap<>(); + for (int i = 0; i < chunksLowerCase.size(); i++) { + String chunk = chunksLowerCase.get(i); + double score = 0.0; + Map<String, Integer> tf = new HashMap<>(); + + String[] wordsInChunk = chunk.split("\\s+"); + for (String w : wordsInChunk) { + String cleanWord = w.replaceAll("[.,!?;:]", ""); + if (cleanWord.length() > 0 && queryWordSet.contains(cleanWord)) { + tf.put(cleanWord, tf.getOrDefault(cleanWord, 0) + 1); + } + } + + for (String word: queryWordSet) { + int tfValue = tf.getOrDefault(word, 0); + double tfIdf = (double) tfValue * idf.getOrDefault(word, 1.0); + score += tfIdf; + } + chunkScores.put(i, score); + } + + int topChunkIndex = -1; + for (Map.Entry<Integer, Double> entry: chunkScores.entrySet()) { + // Keep best-scoring chunk index. + if (entry.getValue() > this.score) { + this.score = entry.getValue(); + topChunkIndex = entry.getKey(); + } + } + + if (topChunkIndex < 0) { + this.text = ""; + this.score = 0.0; + return; + } + + List<String> snippetChunks = new ArrayList<>(); + // Include neighboring chunks to reduce abrupt starts/ends. + if (topChunkIndex > 0) snippetChunks.add(chunks.get(topChunkIndex - 1)); + snippetChunks.add(chunks.get(topChunkIndex)); + if (topChunkIndex < chunks.size() - 1) snippetChunks.add(chunks.get(topChunkIndex + 1)); + this.text = String.join(" ", snippetChunks); + } + + public double getScore() { return this.score; } + public String getText() { return this.text; } + public String getURL() { return this.url; } + public String getTitle() { return this.title; } + } +} diff --git a/source/net/yacy/ai/ToolCallProtocol.java b/source/net/yacy/ai/ToolCallProtocol.java new file mode 100644 index 000000000..8f526beaa --- /dev/null +++ b/source/net/yacy/ai/ToolCallProtocol.java @@ -0,0 +1,634 @@ +/** + * ToolCallProtocol + * Copyright 2026 by Michael Peter Christen + * First released 06.02.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.ai; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.servlet.ServletOutputStream; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONTokener; + + +/** + * Implements the protocol glue for streamed chat completions that include tool + * calls. + * <p> + * Main responsibilities: + * <ul> + * <li>Collect partial tool-call fragments from streaming deltas.</li> + * <li>Merge fragments into complete tool calls (index-based merge).</li> + * <li>Execute tools once a tool turn is complete.</li> + * <li>Append assistant + tool messages back to the conversation.</li> + * <li>Start follow-up model rounds until no further tool call is requested.</li> + * </ul> + * <p> + * The class also enriches streamed lines with {@code tool-calls} and + * {@code tool-results} metadata so clients can display tool activity while the + * stream is forwarded. + */ +public final class ToolCallProtocol { + + /** + * Hard stop to prevent infinite tool loops (assistant requests tool calls + * repeatedly without converging to a final answer). + */ + private static final int MAX_TOOL_ROUNDS = 8; + + /** + * Utility class; not instantiable. + */ + private ToolCallProtocol() {} + + /** + * Builds a request body that is ready for tool-aware chat completion calls. + * <p> + * This method clones the input object, injects tool definitions and optionally + * enforces streaming. + * + * @param body original request body + * @param forceStream when true, sets {@code stream=true} in the cloned body + * @return prepared request body clone + */ + public static JSONObject prepareToolRequestBody(JSONObject body, boolean forceStream) { + try { + final JSONObject prepared = body == null ? new JSONObject(true) : new JSONObject(body.toString()); + if (forceStream) prepared.put("stream", true); + net.yacy.ai.ToolProvider.ensureTools(prepared); + return prepared; + } catch (JSONException e) { + final JSONObject fallback = new JSONObject(true); + net.yacy.ai.ToolProvider.ensureTools(fallback); + return fallback; + } + } + + /** + * Runs the full server-side tool-stream lifecycle for one chat request. + * <p> + * Lifecycle: + * <ol> + * <li>Prepare request body for tools.</li> + * <li>Send initial upstream chat request.</li> + * <li>Forward initial stream while collecting tool calls.</li> + * <li>If tool calls appeared, continue tool rounds until completion.</li> + * </ol> + * + * @param out downstream SSE output stream + * @param llm4Chat model config + * @param originalBody request body template + * @param messages mutable message history + * @param initialMetadata optional JSON fields to inject into the first JSON data line + * @return upstream HTTP status code for the initial request + * @throws IOException on network/stream/protocol errors + */ + public static int proxyToolLifecycle(ServletOutputStream out, LLM.LLMModel llm4Chat, JSONObject originalBody, JSONArray messages, JSONObject initialMetadata) throws IOException { + final JSONObject preparedBody = prepareToolRequestBody(originalBody, false); + final HttpURLConnection conn = openChatCompletionConnection(llm4Chat, preparedBody); + final int status = conn.getResponseCode(); + if (status == 200) { + handleInitialStreamAndContinue(out, conn, llm4Chat, preparedBody, messages, initialMetadata); + } + return status; + } + + /** + * Extracts streamed assistant content and tool-call deltas from one chat chunk. + * <p> + * Supports both modern {@code delta.tool_calls} and legacy + * {@code delta.function_call} formats. + * + * @param j streamed JSON chunk + * @param toolCalls mutable map collecting tool calls by index + * @param assistantContent buffer accumulating assistant text tokens + * @param sawToolCalls single-item boolean holder set to true once any tool + * call appears in this stream + */ + public static void captureToolCalls(JSONObject j, Map<Integer, ToolCall> toolCalls, StringBuilder assistantContent, boolean[] sawToolCalls) { + if (j == null) return; + // Only first choice is processed because streaming APIs typically emit one choice. + JSONArray choices = j.optJSONArray("choices"); + if (choices == null || choices.length() == 0) return; + JSONObject choice = choices.optJSONObject(0); + if (choice == null) return; + JSONObject delta = choice.optJSONObject("delta"); + if (delta == null) return; + + // Assistant text arrives token-by-token in streaming mode. + String content = delta.optString("content", null); + if (content != null) assistantContent.append(content); + + // Preferred modern tool-call format. + JSONArray toolDelta = delta.optJSONArray("tool_calls"); + if (toolDelta != null) { + sawToolCalls[0] = true; + mergeToolCalls(toolCalls, toolDelta); + } + + // Legacy compatibility: transform function_call into tool_calls-like shape. + JSONObject functionCall = delta.optJSONObject("function_call"); + if (functionCall != null) { + sawToolCalls[0] = true; + try { + JSONArray wrapper = new JSONArray(); + JSONObject entry = new JSONObject(true); + entry.put("index", 0); + entry.put("type", "function"); + entry.put("function", functionCall); + wrapper.put(entry); + mergeToolCalls(toolCalls, wrapper); + } catch (JSONException e) { + // ignore malformed legacy function_call chunk + } + } + } + + /** + * Forwards the initial upstream stream to the client while collecting tool + * calls, then executes follow-up tool rounds when needed. + * + * @param out downstream SSE output stream + * @param conn initial upstream connection + * @param llm4Chat model config for follow-up rounds + * @param originalBody request body template for follow-up rounds + * @param messages mutable conversation history + * @param initialMetadata optional metadata injected once into the first JSON line + * @throws IOException on I/O errors + */ + private static void handleInitialStreamAndContinue(ServletOutputStream out, HttpURLConnection conn, LLM.LLMModel llm4Chat, JSONObject originalBody, JSONArray messages, JSONObject initialMetadata) throws IOException { + final StringBuilder assistantContent = new StringBuilder(); + final Map<Integer, ToolCall> toolCalls = new HashMap<>(); + final boolean[] sawToolCalls = new boolean[]{false}; + boolean firstJsonDataLinePending = initialMetadata != null && initialMetadata.length() > 0; + + try (BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8))) { + String line; + while ((line = in.readLine()) != null) { + boolean isDoneLine = "data: [DONE]".equals(line.trim()) || "[DONE]".equals(line.trim()); + if (firstJsonDataLinePending) { + String patched = injectJsonMetadata(line, initialMetadata); + if (patched != null) { + line = patched; + firstJsonDataLinePending = false; + } + } + if (line.startsWith("data:")) { + int p = line.indexOf('{'); + if (p > 0) { + try { + JSONObject j = new JSONObject(new JSONTokener(line.substring(p))); + captureToolCalls(j, toolCalls, assistantContent, sawToolCalls); + } catch (JSONException e) { + // ignore malformed stream chunks and continue forwarding stream + } + } + } + if (!isDoneLine || !sawToolCalls[0]) { + out.println(line); + out.flush(); + } + } + } finally { + conn.disconnect(); + } + + if (sawToolCalls[0]) { + handleToolCallsAndContinue(out, llm4Chat, originalBody, messages, assistantContent.toString(), toolCalls); + } + } + + /** + * Runs the complete tool-turn loop after initial stream parsing detected tool + * calls. + * <p> + * Per round: + * <ol> + * <li>Append assistant tool request + tool response messages.</li> + * <li>Call the chat completion endpoint again with updated messages.</li> + * <li>Forward streamed lines to the client while capturing potential next + * tool calls.</li> + * <li>Stop when no more tool calls are requested or the round cap is hit.</li> + * </ol> + * + * @param out output stream to the client (SSE style lines) + * @param llm4Chat selected model configuration + * @param originalBody original request body template + * @param messages mutable conversation messages from original request + * @param assistantContent already streamed assistant text from current round + * @param toolCalls collected tool calls from current round + * @throws IOException when network I/O or JSON processing fails + */ + public static void handleToolCallsAndContinue(ServletOutputStream out, LLM.LLMModel llm4Chat, JSONObject originalBody, JSONArray messages, String assistantContent, Map<Integer, ToolCall> toolCalls) throws IOException { + try { + // Work on a copy so caller-owned message arrays are not modified unexpectedly. + JSONArray newMessages = new JSONArray(); + for (int i = 0; i < messages.length(); i++) { + newMessages.put(messages.get(i)); + } + String roundAssistantContent = assistantContent == null ? "" : assistantContent; + Map<Integer, ToolCall> roundToolCalls = new HashMap<>(toolCalls); + // Track total calls per tool across this whole tool-turn lifecycle. + Map<String, Integer> toolCallCounters = new HashMap<>(); + + for (int round = 0; round < MAX_TOOL_ROUNDS; round++) { + // Convert captured tool calls into assistant/tool messages and execute tools. + ToolRoundData roundData = appendAssistantAndToolMessages(newMessages, roundAssistantContent, roundToolCalls, toolCallCounters); + if (roundData == null || roundData.toolResults.length() == 0) { + // No executable tool calls; terminate stream cleanly. + out.println("data: [DONE]"); + out.flush(); + return; + } + + // Build follow-up completion request from original body template. + final JSONObject followup = prepareToolRequestBody(originalBody, true); + followup.put("messages", newMessages); + final HttpURLConnection followConn = openChatCompletionConnection(llm4Chat, followup); + if (followConn.getResponseCode() != 200) { + // Upstream error: close downstream stream instead of sending broken chunks. + out.println("data: [DONE]"); + out.flush(); + followConn.disconnect(); + return; + } + + // Collect data for the next potential tool round while forwarding this stream. + final StringBuilder nextAssistantContent = new StringBuilder(); + final Map<Integer, ToolCall> nextToolCalls = new HashMap<>(); + final boolean[] sawToolCalls = new boolean[]{false}; + + try (BufferedReader in = new BufferedReader(new InputStreamReader(followConn.getInputStream(), StandardCharsets.UTF_8))) { + String line; + boolean injectedToolResults = false; + while ((line = in.readLine()) != null) { + // Inject tool metadata once into outgoing stream so UI can show executed tools. + if (!injectedToolResults && roundData.toolResults.length() > 0) { + line = injectToolMetadata(line, roundData); + if (line != null && line.indexOf("\"tool-results\"") >= 0) { + injectedToolResults = true; + } + } + boolean isDoneLine = "data: [DONE]".equals(line.trim()) || "[DONE]".equals(line.trim()); + if (line.startsWith("data:")) { + int p = line.indexOf('{'); + if (p > 0) { + try { + JSONObject j = new JSONObject(new JSONTokener(line.substring(p))); + // Parse streamed deltas to capture possible next tool round. + captureToolCalls(j, nextToolCalls, nextAssistantContent, sawToolCalls); + } catch (JSONException e) { + // ignore malformed stream chunks and continue forwarding stream + } + } + } + // Hide interim [DONE] if another tool round is expected. + if (!isDoneLine || !sawToolCalls[0]) { + out.println(line); + out.flush(); + } + } + } finally { + followConn.disconnect(); + } + + // Ignore phantom rounds where stream signaled tool-calls but no concrete calls were parsed. + if (nextToolCalls.isEmpty()) sawToolCalls[0] = false; + // Final answer reached; caller already received forwarded stream lines. + if (!sawToolCalls[0]) return; + + // Continue with next round tool request that was found in follow-up stream. + roundAssistantContent = nextAssistantContent.toString(); + roundToolCalls = nextToolCalls; + } + // Safety fallback when round cap is hit. + out.println("data: [DONE]"); + out.flush(); + } catch (JSONException e) { + throw new IOException("JSON processing error in tool handling", e); + } + } + + /** + * Opens and sends one chat completion request. + * + * @param llm4Chat model config used for endpoint and auth + * @param requestBody request payload + * @return opened connection after request body has been sent + * @throws IOException on URI/network/write failures + */ + private static HttpURLConnection openChatCompletionConnection(LLM.LLMModel llm4Chat, JSONObject requestBody) throws IOException { + final URL url; + try { + url = new URI(llm4Chat.llm.hoststub + "/v1/chat/completions").toURL(); + } catch (URISyntaxException e) { + throw new IOException("Invalid chat completion URL: " + e.getMessage(), e); + } + final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("POST"); + conn.setRequestProperty("Content-Type", "application/json"); + if (!llm4Chat.llm.api_key.isEmpty()) { + conn.setRequestProperty("Authorization", "Bearer " + llm4Chat.llm.api_key); + } + conn.setDoOutput(true); + try (OutputStream os = conn.getOutputStream()) { + os.write(requestBody.toString().getBytes(StandardCharsets.UTF_8)); + os.flush(); + } + return conn; + } + + /** + * Merges incremental tool call fragments from a streaming delta into the + * in-progress tool call map. + * <p> + * Streaming responses may deliver one tool call over multiple chunks. This method + * merges fields by tool-call index: + * <ul> + * <li>{@code id}, {@code type}, and function {@code name} keep the latest non-empty value.</li> + * <li>function {@code arguments} are appended because they are often streamed in pieces.</li> + * </ul> + * + * @param toolCalls current merged tool calls, keyed by call index + * @param deltaCalls new delta chunk containing partial tool call objects + */ + private static void mergeToolCalls(Map<Integer, ToolCall> toolCalls, JSONArray deltaCalls) { + // Each array element can contain only a fragment of the final tool call. + for (int i = 0; i < deltaCalls.length(); i++) { + JSONObject delta = deltaCalls.optJSONObject(i); + if (delta == null) continue; + Integer index = readToolCallIndex(delta); + // Ignore chunks without a valid non-negative call index. + if (index == null || index.intValue() < 0) continue; + + // Reuse existing partial state for this index or start a new one. + ToolCall call = toolCalls.getOrDefault(index, new ToolCall()); + + // Scalar fields are replaced when a newer non-empty value arrives. + String id = delta.optString("id", null); + if (id != null && !id.isEmpty()) call.id = id; + String type = delta.optString("type", null); + if (type != null && !type.isEmpty()) call.type = type; + JSONObject fn = delta.optJSONObject("function"); + if (fn != null) { + String name = fn.optString("name", null); + if (name != null && !name.isEmpty()) call.name = name; + Object rawArgs = fn.opt("arguments"); + if (rawArgs instanceof String) { + String args = (String) rawArgs; + // Arguments are streamed as string fragments and must be concatenated. + if (!args.isEmpty()) call.arguments = (call.arguments == null ? "" : call.arguments) + args; + } + } + toolCalls.put(index, call); + } + } + + /** + * Appends one assistant message containing tool_calls plus corresponding tool + * role messages with execution results. + * <p> + * Only executable tool calls are included (must have a function name and valid + * JSON arguments if arguments are present). + * + * @param messages mutable conversation message array to append to + * @param assistantContent assistant text that accompanied tool call emission + * @param toolCalls merged tool calls keyed by index + * @param toolCallCounters cumulative per-tool call counters for this turn + * @return round data containing emitted {@code tool_calls} and tool execution + * results, or {@code null} when JSON assembly fails + */ + private static ToolRoundData appendAssistantAndToolMessages(JSONArray messages, String assistantContent, Map<Integer, ToolCall> toolCalls, Map<String, Integer> toolCallCounters) { + try { + // Keep original model call order stable by sorting call indices. + List<Integer> indices = new ArrayList<>(toolCalls.keySet()); + Collections.sort(indices); + JSONArray toolCallsArray = new JSONArray(); + JSONArray toolMessages = new JSONArray(); + JSONArray toolResults = new JSONArray(); + for (Integer idx : indices) { + ToolCall call = toolCalls.get(idx); + if (call == null) continue; + // Ensure required defaults for follow-up protocol compliance. + if (call.id == null || call.id.isEmpty()) call.id = "toolcall_" + idx + "_" + System.currentTimeMillis(); + if (call.type == null || call.type.isEmpty()) call.type = "function"; + if (!isExecutableToolCall(call)) continue; + // Enforce per-tool max calls for the complete tool turn lifecycle. + final String toolName = call.name == null ? "" : call.name.trim(); + final int maxCalls = net.yacy.ai.ToolProvider.maxCallsPerTurn(toolName); + final int usedCalls = toolCallCounters.getOrDefault(toolName, Integer.valueOf(0)).intValue(); + if (usedCalls >= maxCalls) continue; + + // Assistant message representation of requested tool call. + JSONObject toolCallJson = new JSONObject(true); + toolCallJson.put("id", call.id); + toolCallJson.put("type", call.type); + JSONObject fn = new JSONObject(true); + fn.put("name", call.name == null ? "" : call.name); + fn.put("arguments", call.arguments == null ? "" : call.arguments); + toolCallJson.put("function", fn); + toolCallsArray.put(toolCallJson); + + // Execute tool locally and create "tool" role response message. + String result = net.yacy.ai.ToolProvider.executeTool(call.name, call.arguments); + JSONObject toolMessage = new JSONObject(true); + toolMessage.put("role", "tool"); + toolMessage.put("tool_call_id", call.id); + toolMessage.put("name", call.name == null ? "" : call.name); + toolMessage.put("content", result); + toolMessages.put(toolMessage); + toolCallCounters.put(toolName, Integer.valueOf(usedCalls + 1)); + + // Separate compact result list for stream metadata injection. + JSONObject toolResult = new JSONObject(true); + toolResult.put("tool_call_id", call.id); + toolResult.put("name", call.name == null ? "" : call.name); + toolResult.put("content", result); + toolResults.put(toolResult); + } + + // If nothing is executable, leave the message history unchanged. + if (toolCallsArray.length() == 0) { + return new ToolRoundData(new JSONArray(), new JSONArray()); + } + + // Add assistant turn that requested tools. + JSONObject assistantMessage = new JSONObject(true); + assistantMessage.put("role", "assistant"); + assistantMessage.put("content", assistantContent == null ? "" : assistantContent); + assistantMessage.put("tool_calls", toolCallsArray); + messages.put(assistantMessage); + // Follow with synthetic tool result messages for next model round context. + for (int i = 0; i < toolMessages.length(); i++) messages.put(toolMessages.get(i)); + + return new ToolRoundData(toolCallsArray, toolResults); + } catch (JSONException e) { + return null; + } + } + + /** + * Injects tool metadata into one outbound SSE line if the line contains JSON + * data. + * <p> + * Metadata keys: + * <ul> + * <li>{@code tool-calls}: normalized tool call objects sent in the round.</li> + * <li>{@code tool-results}: local tool execution outputs.</li> + * </ul> + * + * @param line outbound line (typically prefixed with {@code data:}) + * @param roundData round metadata to inject + * @return original or augmented line + */ + private static String injectToolMetadata(String line, ToolRoundData roundData) { + if (line == null || roundData == null || roundData.toolResults == null || roundData.toolResults.length() == 0) return line; + if (!line.startsWith("data:")) return line; + int p = line.indexOf('{'); + if (p <= 0) return line; + try { + JSONObject j = new JSONObject(new JSONTokener(line.substring(p))); + // Do not overwrite if upstream already provided these fields. + if (!j.has("tool-calls") && roundData.toolCalls != null) j.put("tool-calls", roundData.toolCalls); + if (!j.has("tool-results")) j.put("tool-results", roundData.toolResults); + return line.substring(0, p) + j.toString(); + } catch (JSONException e) { + // Non-JSON data lines are forwarded unchanged. + return line; + } + } + + /** + * Injects arbitrary JSON metadata into one SSE {@code data: ...} JSON line. + * Existing keys are preserved. + * + * @param line outbound stream line + * @param metadata metadata object to inject + * @return augmented line, or {@code null} when the line is not a JSON data line + */ + private static String injectJsonMetadata(String line, JSONObject metadata) { + if (line == null || metadata == null || metadata.length() == 0) return null; + if (!line.startsWith("data:")) return null; + int p = line.indexOf('{'); + if (p <= 0) return null; + try { + JSONObject j = new JSONObject(new JSONTokener(line.substring(p))); + JSONArray names = metadata.names(); + if (names == null || names.length() == 0) return null; + for (int i = 0; i < names.length(); i++) { + String name = names.optString(i, null); + if (name == null || name.isEmpty()) continue; + if (!j.has(name)) j.put(name, metadata.get(name)); + } + return line.substring(0, p) + j.toString(); + } catch (JSONException e) { + return null; + } + } + + /** + * Reads the {@code index} field used to correlate streamed tool-call fragments. + * + * @param delta one tool-call delta object + * @return index as Integer when present and numeric, otherwise {@code null} + */ + private static Integer readToolCallIndex(JSONObject delta) { + if (delta == null || !delta.has("index")) return null; + Object raw = delta.opt("index"); + if (raw instanceof Number) return Integer.valueOf(((Number) raw).intValue()); + return null; + } + + /** + * Checks whether a merged tool call is executable locally. + * <p> + * Requirements: + * <ul> + * <li>non-empty tool/function name</li> + * <li>arguments omitted/empty or valid JSON object text</li> + * </ul> + * + * @param call merged tool call candidate + * @return true when tool execution should be attempted + */ + private static boolean isExecutableToolCall(ToolCall call) { + if (call == null || call.name == null || call.name.trim().isEmpty()) return false; + String args = call.arguments; + if (args == null || args.isEmpty()) return true; + try { + new JSONObject(args); + return true; + } catch (JSONException e) { + return false; + } + } + + /** + * Round-local metadata container used during tool loop execution. + */ + private static final class ToolRoundData { + /** Tool calls that were executed in the round. */ + final JSONArray toolCalls; + /** Tool outputs corresponding to {@link #toolCalls}. */ + final JSONArray toolResults; + + /** + * Builds a round data object with non-null arrays. + * + * @param toolCalls tool call descriptors + * @param toolResults tool execution outputs + */ + ToolRoundData(JSONArray toolCalls, JSONArray toolResults) { + this.toolCalls = toolCalls == null ? new JSONArray() : toolCalls; + this.toolResults = toolResults == null ? new JSONArray() : toolResults; + } + } + + /** + * Mutable tool-call assembly object built from streamed deltas. + * <p> + * Fields may be populated incrementally over multiple chunks. + */ + public static final class ToolCall { + /** Tool call id as emitted by model/provider. */ + public String id; + /** Tool call type, usually {@code function}. */ + public String type; + /** Tool/function name. */ + public String name; + /** JSON argument text (can be assembled from fragments). */ + public String arguments; + } +} diff --git a/source/net/yacy/ai/ToolHandler.java b/source/net/yacy/ai/ToolHandler.java new file mode 100644 index 000000000..9c278cf00 --- /dev/null +++ b/source/net/yacy/ai/ToolHandler.java @@ -0,0 +1,43 @@ +/** + * ToolHandler + * Copyright 2026 by Michael Peter Christen + * First released 06.02.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.ai; + +import org.json.JSONException; +import org.json.JSONObject; + +public interface ToolHandler { + + JSONObject definition() throws JSONException; + + String execute(String arguments); + + int maxCallsPerTurn(); + + public static String errorJson(String message) { + try { + JSONObject err = new JSONObject(true); + err.put("error", message == null ? "error" : message); + return err.toString(); + } catch (JSONException e) { + return "{\"error\":\"" + (message == null ? "error" : message.replace("\"", "'")) + "\"}"; + } + } +} diff --git a/source/net/yacy/ai/ToolProvider.java b/source/net/yacy/ai/ToolProvider.java new file mode 100644 index 000000000..070c6dd45 --- /dev/null +++ b/source/net/yacy/ai/ToolProvider.java @@ -0,0 +1,226 @@ +/** + * ToolProvider + * Copyright 2026 by Michael Peter Christen + * First released 06.02.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.ai; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import net.yacy.ai.tools.CalculatorTool; +import net.yacy.ai.tools.ChitChatTool; +import net.yacy.ai.tools.DateMathTool; +import net.yacy.ai.tools.DateTimeTool; +import net.yacy.ai.tools.HttpJsonTool; +import net.yacy.ai.tools.NumberParserTool; +import net.yacy.ai.tools.PromptToMermaidTool; +import net.yacy.ai.tools.SelfReflectTool; +import net.yacy.ai.tools.TableOpsTool; +import net.yacy.ai.tools.UnitConverterTool; +import net.yacy.ai.tools.WebFetchTool; +import net.yacy.ai.tools.WikipediaLinkCreatorTool; + +/** + * Central registry and dispatch utility for all built-in YaCy LLM tools. + * <p> + * Responsibilities: + * <ul> + * <li>Expose tool JSON schemas into outbound chat-completion requests.</li> + * <li>Resolve tool names to handlers.</li> + * <li>Execute tool calls by name with raw JSON argument text.</li> + * </ul> + */ +public final class ToolProvider { + + /** + * Ordered list of built-in tool handlers. Order is preserved when exposing + * definitions to providers. + */ + private static final List<ToolHandler> TOOLS = Arrays.asList( + new DateTimeTool(), + new DateMathTool(), + new CalculatorTool(), + new NumberParserTool(), + new UnitConverterTool(), + new WebFetchTool(), + new HttpJsonTool(), + new TableOpsTool(), + new SelfReflectTool(), + new ChitChatTool(), + new PromptToMermaidTool(), + new WikipediaLinkCreatorTool() + ); + + /** + * Lookup table for fast runtime dispatch by function/tool name. + */ + private static final Map<String, ToolHandler> TOOL_BY_NAME = buildToolIndex(TOOLS); + + /** + * Utility class; not instantiable. + */ + private ToolProvider() {} + + /** + * Ensures a chat request body contains all available tool definitions and a + * default tool selection mode. + * <p> + * Existing tool definitions are kept; missing ones are appended. + * + * @param body request body to mutate + */ + public static void ensureTools(JSONObject body) { + if (body == null) return; + try { + // Create "tools" array lazily when caller did not provide one. + JSONArray tools = body.optJSONArray("tools"); + if (tools == null) { + tools = new JSONArray(); + body.put("tools", tools); + } + // Merge registry definitions without duplicating by tool name. + for (ToolHandler tool : TOOLS) { + JSONObject definition = tool.definition(); + addToolDefinitionIfMissing(tools, definition); + } + // Providers usually expect explicit tool-choice mode. + if (!body.has("tool_choice")) { + body.put("tool_choice", "auto"); + } + } catch (JSONException e) { + // keep body unchanged if tool schema cannot be injected + } + } + + /** + * Dispatches one tool invocation by name. + * + * @param name tool/function name + * @param arguments raw JSON arguments string + * @return tool execution result as JSON string (or error JSON) + */ + public static String executeTool(String name, String arguments) { + if (name == null) return errorJson("Invalid tool call"); + ToolHandler tool = TOOL_BY_NAME.get(name); + if (tool == null) return errorJson("Unknown tool: " + name); + // Tool implementations are responsible for argument parsing/validation. + return tool.execute(arguments); + } + + /** + * Returns the configured maximum number of calls for a tool within one tool + * turn lifecycle. + * + * @param name tool/function name + * @return maximum call count, defaults to 1 for unknown/invalid values + */ + public static int maxCallsPerTurn(String name) { + if (name == null) return 1; + ToolHandler tool = TOOL_BY_NAME.get(name); + if (tool == null) return 1; + int max = tool.maxCallsPerTurn(); + return max <= 0 ? 1 : max; + } + + /** + * Builds an immutable map from tool name to handler. + * Invalid or unnamed definitions are skipped. + * + * @param tools source handlers + * @return unmodifiable name-indexed map + */ + private static Map<String, ToolHandler> buildToolIndex(List<ToolHandler> tools) { + Map<String, ToolHandler> byName = new LinkedHashMap<>(); + for (ToolHandler tool : tools) { + if (tool == null) continue; + try { + String name = extractToolName(tool.definition()); + if (name == null || name.isEmpty()) continue; + byName.put(name, tool); + } catch (JSONException e) { + // skip invalid tool definition + } + } + return Collections.unmodifiableMap(byName); + } + + /** + * Appends a tool definition if no existing tool with the same function name + * is already present. + * + * @param tools destination tool array in request body + * @param toolDefinition candidate definition + */ + private static void addToolDefinitionIfMissing(JSONArray tools, JSONObject toolDefinition) { + if (tools == null || toolDefinition == null) return; + try { + String name = extractToolName(toolDefinition); + if (name == null || name.isEmpty()) return; + // Dedupe by logical function name (not by full JSON string equality). + for (int i = 0; i < tools.length(); i++) { + JSONObject existing = tools.optJSONObject(i); + if (existing == null) continue; + String existingName = extractToolName(existing); + if (name.equals(existingName)) return; + } + tools.put(toolDefinition); + } catch (JSONException e) { + // ignore malformed tool schema + } + } + + /** + * Extracts the function name from a tool definition object. + * + * @param toolDefinition MCP/OpenAI-style tool definition + * @return trimmed function name or {@code null} + * @throws JSONException if JSON access fails unexpectedly + */ + private static String extractToolName(JSONObject toolDefinition) throws JSONException { + if (toolDefinition == null) return null; + JSONObject fn = toolDefinition.optJSONObject("function"); + if (fn == null) return null; + String name = fn.optString("name", ""); + return name == null ? null : name.trim(); + } + + /** + * Builds a minimal error JSON string while avoiding propagation of secondary + * JSON construction failures. + * + * @param message human-readable error message + * @return serialized JSON error object + */ + private static String errorJson(String message) { + try { + JSONObject err = new JSONObject(true); + err.put("error", message == null ? "error" : message); + return err.toString(); + } catch (JSONException e) { + return "{\"error\":\"" + (message == null ? "error" : message.replace("\"", "'")) + "\"}"; + } + } +} diff --git a/source/net/yacy/ai/tools/CalculatorTool.java b/source/net/yacy/ai/tools/CalculatorTool.java new file mode 100644 index 000000000..4c33d7194 --- /dev/null +++ b/source/net/yacy/ai/tools/CalculatorTool.java @@ -0,0 +1,358 @@ +/** + * CalculatorTool + * Copyright 2026 by Michael Peter Christen + * First released 06.02.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.ai.tools; + +import java.math.BigDecimal; +import java.math.MathContext; +import java.util.ArrayList; +import java.util.List; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import net.yacy.ai.ToolHandler; + +public class CalculatorTool implements ToolHandler { + + private static final String NAME = "calculator"; + + @Override + public JSONObject definition() throws JSONException { + JSONObject tool = new JSONObject(true); + tool.put("type", "function"); + JSONObject fn = new JSONObject(true); + fn.put("name", NAME); + fn.put("description", "Evaluate a mathematical formula. Supports operators (+,-,*,/,%,^), constants (pi,e,tau,phi), functions (sqrt,abs,ln,log,sin,cos,tan,asin,acos,atan,min,max,pow,root,...) and scientific notation. Use this as your calculator."); + JSONObject params = new JSONObject(true); + params.put("type", "object"); + JSONObject props = new JSONObject(true); + JSONObject formula = new JSONObject(true); + formula.put("type", "string"); + formula.put("description", "Formula to evaluate, e.g. (2+3)*4/5"); + props.put("formula", formula); + params.put("properties", props); + params.put("required", new JSONArray().put("formula")); + fn.put("parameters", params); + tool.put("function", fn); + return tool; + } + + + public int maxCallsPerTurn() { + return 10; + } + + + public String execute(String arguments) { + String formula; + try { + JSONObject obj = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments); + formula = obj.optString("formula", "").trim(); + } catch (JSONException e) { + return ToolHandler.errorJson("Invalid arguments JSON"); + } + if (formula == null || formula.isEmpty()) return ToolHandler.errorJson("Missing formula"); + try { + BigDecimal value = new ExpressionParser(formula).parse(); + JSONObject result = new JSONObject(true); + result.put("formula", formula); + result.put("result", value.stripTrailingZeros().toPlainString()); + return result.toString(); + } catch (Exception e) { + return ToolHandler.errorJson("Invalid formula: " + e.getMessage()); + } + } + + private static final class ExpressionParser { + private final String expression; + private int pos; + + private ExpressionParser(String expression) { + this.expression = expression == null ? "" : expression; + } + + private BigDecimal parse() { + BigDecimal value = parseExpression(); + skipWs(); + if (pos != expression.length()) throw new IllegalArgumentException("Unexpected token at " + pos); + return value; + } + + private BigDecimal parseExpression() { + BigDecimal value = parseTerm(); + while (true) { + skipWs(); + if (eat('+')) value = value.add(parseTerm()); + else if (eat('-')) value = value.subtract(parseTerm()); + else return value; + } + } + + private BigDecimal parseTerm() { + BigDecimal value = parseFactor(); + while (true) { + skipWs(); + if (eat('*')) value = value.multiply(parseFactor(), MathContext.DECIMAL64); + else if (eat('/')) value = value.divide(parseFactor(), MathContext.DECIMAL64); + else if (eat('%')) value = value.remainder(parseFactor(), MathContext.DECIMAL64); + else return value; + } + } + + private BigDecimal parseFactor() { + skipWs(); + if (eat('+')) return parseFactor(); + if (eat('-')) return parseFactor().negate(); + BigDecimal base; + if (eat('(')) { + base = parseExpression(); + if (!eat(')')) throw new IllegalArgumentException("Missing ')'"); + } else if (isIdentifierStart(currentChar())) { + String ident = parseIdentifier(); + if (eat('(')) { + List<BigDecimal> args = parseArguments(); + if (!eat(')')) throw new IllegalArgumentException("Missing ')' after function argument"); + base = applyFunction(ident, args); + } else { + base = resolveConstant(ident); + } + } else { + base = parseNumber(); + } + skipWs(); + if (eat('^')) { + BigDecimal exp = parseFactor(); + try { + int intExp = exp.intValueExact(); + return base.pow(intExp, MathContext.DECIMAL64); + } catch (ArithmeticException e) { + double b = base.doubleValue(); + double ex = exp.doubleValue(); + // Fractional powers are evaluated in double precision. + double result = Math.pow(b, ex); + if (Double.isNaN(result) || Double.isInfinite(result)) { + throw new IllegalArgumentException("Invalid exponentiation result"); + } + return new BigDecimal(result, MathContext.DECIMAL64); + } + } + return base; + } + + private BigDecimal parseNumber() { + skipWs(); + int start = pos; + boolean dot = false; + boolean exponent = false; + while (pos < expression.length()) { + char c = expression.charAt(pos); + if (c >= '0' && c <= '9') { + pos++; + continue; + } + if (c == '.' && !dot) { + dot = true; + pos++; + continue; + } + if ((c == 'e' || c == 'E') && !exponent) { + exponent = true; + pos++; + if (pos < expression.length()) { + char sign = expression.charAt(pos); + if (sign == '+' || sign == '-') pos++; + } + continue; + } + break; + } + if (start == pos) throw new IllegalArgumentException("Number expected at " + pos); + return new BigDecimal(expression.substring(start, pos), MathContext.DECIMAL64); + } + + private String parseIdentifier() { + skipWs(); + int start = pos; + if (!isIdentifierStart(currentChar())) throw new IllegalArgumentException("Identifier expected at " + pos); + pos++; + while (pos < expression.length()) { + char c = expression.charAt(pos); + if (Character.isLetterOrDigit(c) || c == '_' || c == '.') { + pos++; + continue; + } + break; + } + return expression.substring(start, pos).toLowerCase(); + } + + private List<BigDecimal> parseArguments() { + List<BigDecimal> args = new ArrayList<>(); + skipWs(); + if (currentChar() == ')') return args; + args.add(parseExpression()); + while (true) { + skipWs(); + if (!eat(',')) break; + args.add(parseExpression()); + } + return args; + } + + private BigDecimal applyFunction(String name, List<BigDecimal> args) { + String fn = shortName(name); + if ("sqrt".equals(fn)) return bd(Math.sqrt(nonNegative(arg(args, 0), "sqrt"))); + if ("cbrt".equals(fn)) return bd(Math.cbrt(arg(args, 0).doubleValue())); + if ("abs".equals(fn)) return arg(args, 0).abs(); + if ("floor".equals(fn)) return bd(Math.floor(arg(args, 0).doubleValue())); + if ("ceil".equals(fn)) return bd(Math.ceil(arg(args, 0).doubleValue())); + if ("round".equals(fn)) return bd(Math.rint(arg(args, 0).doubleValue())); + if ("trunc".equals(fn)) return bd(arg(args, 0).doubleValue() < 0d ? Math.ceil(arg(args, 0).doubleValue()) : Math.floor(arg(args, 0).doubleValue())); + + if ("exp".equals(fn)) return bd(Math.exp(arg(args, 0).doubleValue())); + if ("ln".equals(fn)) return bd(Math.log(positive(arg(args, 0), "ln"))); + if ("log".equals(fn)) { + if (args.size() == 1) return bd(Math.log10(positive(arg(args, 0), "log"))); + double value = positive(arg(args, 0), "log"); + double base = positive(arg(args, 1), "log base"); + if (base == 1d) throw new IllegalArgumentException("log base must not be 1"); + return bd(Math.log(value) / Math.log(base)); + } + if ("log10".equals(fn)) return bd(Math.log10(positive(arg(args, 0), "log10"))); + if ("log2".equals(fn)) return bd(Math.log(positive(arg(args, 0), "log2")) / Math.log(2d)); + + if ("sin".equals(fn)) return bd(Math.sin(arg(args, 0).doubleValue())); + if ("cos".equals(fn)) return bd(Math.cos(arg(args, 0).doubleValue())); + if ("tan".equals(fn)) return bd(Math.tan(arg(args, 0).doubleValue())); + if ("asin".equals(fn)) return bd(Math.asin(arg(args, 0).doubleValue())); + if ("acos".equals(fn)) return bd(Math.acos(arg(args, 0).doubleValue())); + if ("atan".equals(fn)) return bd(Math.atan(arg(args, 0).doubleValue())); + if ("atan2".equals(fn)) return bd(Math.atan2(arg(args, 0).doubleValue(), arg(args, 1).doubleValue())); + if ("sinh".equals(fn)) return bd(Math.sinh(arg(args, 0).doubleValue())); + if ("cosh".equals(fn)) return bd(Math.cosh(arg(args, 0).doubleValue())); + if ("tanh".equals(fn)) return bd(Math.tanh(arg(args, 0).doubleValue())); + + if ("deg".equals(fn) || "degrees".equals(fn) || "todeg".equals(fn)) return bd(Math.toDegrees(arg(args, 0).doubleValue())); + if ("rad".equals(fn) || "radians".equals(fn) || "torad".equals(fn)) return bd(Math.toRadians(arg(args, 0).doubleValue())); + + if ("pow".equals(fn)) return bd(Math.pow(arg(args, 0).doubleValue(), arg(args, 1).doubleValue())); + if ("root".equals(fn)) { + double n = arg(args, 1).doubleValue(); + if (n == 0d) throw new IllegalArgumentException("root degree must not be 0"); + return bd(Math.pow(arg(args, 0).doubleValue(), 1d / n)); + } + if ("mod".equals(fn)) return arg(args, 0).remainder(arg(args, 1), MathContext.DECIMAL64); + + if ("min".equals(fn)) return min(args); + if ("max".equals(fn)) return max(args); + if ("avg".equals(fn) || "mean".equals(fn)) return mean(args); + if ("sum".equals(fn)) return sum(args); + throw new IllegalArgumentException("Unknown function: " + name); + } + + private BigDecimal resolveConstant(String name) { + String n = shortName(name); + if ("pi".equals(n)) return new BigDecimal(Math.PI, MathContext.DECIMAL64); + if ("e".equals(n)) return new BigDecimal(Math.E, MathContext.DECIMAL64); + if ("tau".equals(n)) return new BigDecimal(2d * Math.PI, MathContext.DECIMAL64); + if ("phi".equals(n)) return new BigDecimal((1d + Math.sqrt(5d)) / 2d, MathContext.DECIMAL64); + throw new IllegalArgumentException("Unknown identifier: " + name); + } + + private String shortName(String name) { + if (name == null || name.isEmpty()) return ""; + int p = name.lastIndexOf('.'); + if (p < 0 || p >= name.length() - 1) return name; + return name.substring(p + 1); + } + + private BigDecimal arg(List<BigDecimal> args, int index) { + if (args == null || index < 0 || index >= args.size()) throw new IllegalArgumentException("Missing function argument at position " + (index + 1)); + return args.get(index); + } + + private double positive(BigDecimal value, String label) { + double v = value.doubleValue(); + if (v <= 0d) throw new IllegalArgumentException(label + " argument must be > 0"); + return v; + } + + private double nonNegative(BigDecimal value, String label) { + double v = value.doubleValue(); + if (v < 0d) throw new IllegalArgumentException(label + " argument must be >= 0"); + return v; + } + + private BigDecimal bd(double value) { + if (Double.isNaN(value) || Double.isInfinite(value)) { + throw new IllegalArgumentException("Invalid numeric result"); + } + return new BigDecimal(value, MathContext.DECIMAL64); + } + + private BigDecimal min(List<BigDecimal> args) { + if (args == null || args.isEmpty()) throw new IllegalArgumentException("min requires at least one argument"); + BigDecimal m = args.get(0); + for (int i = 1; i < args.size(); i++) if (args.get(i).compareTo(m) < 0) m = args.get(i); + return m; + } + + private BigDecimal max(List<BigDecimal> args) { + if (args == null || args.isEmpty()) throw new IllegalArgumentException("max requires at least one argument"); + BigDecimal m = args.get(0); + for (int i = 1; i < args.size(); i++) if (args.get(i).compareTo(m) > 0) m = args.get(i); + return m; + } + + private BigDecimal sum(List<BigDecimal> args) { + if (args == null || args.isEmpty()) throw new IllegalArgumentException("sum requires at least one argument"); + BigDecimal s = BigDecimal.ZERO; + for (BigDecimal v : args) s = s.add(v, MathContext.DECIMAL64); + return s; + } + + private BigDecimal mean(List<BigDecimal> args) { + return sum(args).divide(new BigDecimal(args.size()), MathContext.DECIMAL64); + } + + private char currentChar() { + if (pos < 0 || pos >= expression.length()) return '\0'; + return expression.charAt(pos); + } + + private boolean isIdentifierStart(char c) { + return Character.isLetter(c) || c == '_'; + } + + private void skipWs() { + while (pos < expression.length() && Character.isWhitespace(expression.charAt(pos))) pos++; + } + + private boolean eat(char c) { + if (pos < expression.length() && expression.charAt(pos) == c) { + pos++; + return true; + } + return false; + } + } +} diff --git a/source/net/yacy/ai/tools/ChitChatTool.java b/source/net/yacy/ai/tools/ChitChatTool.java new file mode 100644 index 000000000..8c0326068 --- /dev/null +++ b/source/net/yacy/ai/tools/ChitChatTool.java @@ -0,0 +1,134 @@ +/** + * ChitChatTool + * Copyright 2026 by Michael Peter Christen + * First released 06.02.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.ai.tools; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import net.yacy.ai.ToolHandler; + +public class ChitChatTool implements ToolHandler { + + private static final String NAME = "chitchat"; + private static final int DEFAULT_FACT_COUNT = 3; + private static final int MAX_FACT_COUNT = 5; + + @Override + public JSONObject definition() throws JSONException { + JSONObject tool = new JSONObject(true); + tool.put("type", "function"); + JSONObject fn = new JSONObject(true); + fn.put("name", NAME); + fn.put("description", "Provide friendly chitchat. Call me when the user just tests the chat, says hello, is unsure with the prompt or in case of profanity."); + + JSONObject params = new JSONObject(true); + params.put("type", "object"); + JSONObject props = new JSONObject(true); + + JSONObject tone = new JSONObject(true); + tone.put("type", "string"); + tone.put("description", "Optional tone hint, e.g. friendly, concise, playful."); + props.put("tone", tone); + + JSONObject factCount = new JSONObject(true); + factCount.put("type", "integer"); + factCount.put("description", "How many ability facts to include (1-5, default 3)."); + props.put("fact_count", factCount); + + params.put("properties", props); + params.put("required", new JSONArray()); + fn.put("parameters", params); + tool.put("function", fn); + return tool; + } + + + public int maxCallsPerTurn() { + return 1; + } + + + public String execute(String arguments) { + String tone = "friendly"; + int factCount = DEFAULT_FACT_COUNT; + if (arguments != null && !arguments.isEmpty()) { + try { + JSONObject args = new JSONObject(arguments); + tone = args.optString("tone", tone).trim(); + factCount = args.optInt("fact_count", factCount); + } catch (JSONException e) { + return ToolHandler.errorJson("Invalid arguments JSON"); + } + } + + int count = clamp(factCount, 1, MAX_FACT_COUNT); + List<String> selected = pickFacts(count); + String message = buildMessage(tone, selected); + + try { + JSONObject result = new JSONObject(true); + result.put("tool", NAME); + result.put("tone", tone); + result.put("message", message); + result.put("facts", new JSONArray(selected)); + return result.toString(); + } catch (JSONException e) { + return ToolHandler.errorJson("Failed to build chitchat response"); + } + } + + private static List<String> pickFacts(int count) { + List<String> facts = new ArrayList<>(SelfReflectTool.facts()); + if (facts.isEmpty()) return facts; + Collections.shuffle(facts, ThreadLocalRandom.current()); + int n = Math.min(count, facts.size()); + return new ArrayList<>(facts.subList(0, n)); + } + + private static String buildMessage(String tone, List<String> facts) { + StringBuilder sb = new StringBuilder(); + sb.append("Happy to chat. "); + sb.append(SelfReflectTool.purpose()); + if (tone != null && !tone.isEmpty()) { + sb.append(" Tone requested: ").append(tone).append(". "); + } else { + sb.append(" "); + } + sb.append("Here are a few things I can do: "); + for (int i = 0; i < facts.size(); i++) { + if (i > 0) sb.append(" "); + sb.append(i + 1).append(") ").append(facts.get(i)); + } + return sb.toString(); + } + + private static int clamp(int value, int min, int max) { + if (value < min) return min; + if (value > max) return max; + return value; + } +} diff --git a/source/net/yacy/ai/tools/DateMathTool.java b/source/net/yacy/ai/tools/DateMathTool.java new file mode 100644 index 000000000..bcd3e333b --- /dev/null +++ b/source/net/yacy/ai/tools/DateMathTool.java @@ -0,0 +1,214 @@ +/** + * DateMathTool + * Copyright 2026 by Michael Peter Christen + * First released 06.02.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.ai.tools; + +import java.time.DayOfWeek; +import java.time.Duration; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.YearMonth; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.time.temporal.ChronoUnit; +import java.util.Locale; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import net.yacy.ai.ToolHandler; + +public class DateMathTool implements ToolHandler { + + private static final String NAME = "date_math"; + + @Override + public JSONObject definition() throws JSONException { + JSONObject tool = new JSONObject(true); + tool.put("type", "function"); + JSONObject fn = new JSONObject(true); + fn.put("name", NAME); + fn.put("description", "Perform deterministic date/time operations: add, subtract, diff, start_of, end_of."); + + JSONObject params = new JSONObject(true); + params.put("type", "object"); + JSONObject props = new JSONObject(true); + props.put("operation", new JSONObject(true).put("type", "string").put("description", "add|subtract|diff|start_of|end_of")); + props.put("datetime", new JSONObject(true).put("type", "string").put("description", "Base datetime/date in ISO format.")); + props.put("other_datetime", new JSONObject(true).put("type", "string").put("description", "Second datetime for diff.")); + props.put("amount", new JSONObject(true).put("type", "integer").put("description", "Amount for add/subtract.")); + props.put("unit", new JSONObject(true).put("type", "string").put("description", "minute|hour|day|week|month|year")); + props.put("period", new JSONObject(true).put("type", "string").put("description", "day|week|month|year (for start_of/end_of).")); + props.put("timezone", new JSONObject(true).put("type", "string").put("description", "IANA zone; default system zone.")); + params.put("properties", props); + params.put("required", new JSONArray().put("operation")); + fn.put("parameters", params); + tool.put("function", fn); + return tool; + } + + + public int maxCallsPerTurn() { + return 1; + } + + + public String execute(String arguments) { + final JSONObject args; + try { + args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments); + } catch (JSONException e) { + return ToolHandler.errorJson("Invalid arguments JSON"); + } + + String op = args.optString("operation", "").trim().toLowerCase(Locale.ROOT); + if (op.isEmpty()) return ToolHandler.errorJson("Missing operation"); + + ZoneId zone = parseZone(args.optString("timezone", "")); + if (zone == null) return ToolHandler.errorJson("Invalid timezone"); + ZonedDateTime base = parseDateTime(args.optString("datetime", ""), zone); + if (base == null) base = ZonedDateTime.now(zone); + + try { + JSONObject out = new JSONObject(true); + out.put("tool", NAME); + out.put("operation", op); + out.put("timezone", zone.getId()); + out.put("input_datetime", base.toString()); + + if ("add".equals(op) || "subtract".equals(op)) { + int amount = args.optInt("amount", Integer.MIN_VALUE); + if (amount == Integer.MIN_VALUE) return ToolHandler.errorJson("Missing amount"); + String unit = normalizeUnit(args.optString("unit", "")); + if (unit == null) return ToolHandler.errorJson("Invalid unit"); + long signed = "subtract".equals(op) ? -((long) amount) : amount; + ZonedDateTime result = applyAdd(base, signed, unit); + out.put("result_datetime", result.toString()); + out.put("result_epoch_ms", result.toInstant().toEpochMilli()); + return out.toString(); + } + + if ("diff".equals(op)) { + ZonedDateTime other = parseDateTime(args.optString("other_datetime", ""), zone); + if (other == null) return ToolHandler.errorJson("Missing or invalid other_datetime"); + Duration d = Duration.between(base.toInstant(), other.toInstant()); + out.put("other_datetime", other.toString()); + out.put("difference_seconds", d.getSeconds()); + out.put("difference_minutes", d.toMinutes()); + out.put("difference_hours", d.toHours()); + out.put("difference_days", d.toDays()); + return out.toString(); + } + + if ("start_of".equals(op) || "end_of".equals(op)) { + String period = args.optString("period", "").trim().toLowerCase(Locale.ROOT); + if (period.isEmpty()) return ToolHandler.errorJson("Missing period"); + ZonedDateTime result = "start_of".equals(op) ? startOf(base, period) : endOf(base, period); + out.put("period", period); + out.put("result_datetime", result.toString()); + out.put("result_epoch_ms", result.toInstant().toEpochMilli()); + return out.toString(); + } + return ToolHandler.errorJson("Unsupported operation: " + op); + } catch (IllegalArgumentException | JSONException e) { + return ToolHandler.errorJson(e.getMessage()); + } + } + + private static ZoneId parseZone(String timezone) { + if (timezone == null || timezone.trim().isEmpty()) return ZoneId.systemDefault(); + try { + return ZoneId.of(timezone.trim()); + } catch (Exception e) { + return null; + } + } + + private static ZonedDateTime parseDateTime(String value, ZoneId zone) { + if (value == null || value.trim().isEmpty()) return null; + String v = value.trim(); + try { + return ZonedDateTime.parse(v); + } catch (DateTimeParseException e) {} + try { + LocalDateTime ldt = LocalDateTime.parse(v, DateTimeFormatter.ISO_LOCAL_DATE_TIME); + return ldt.atZone(zone); + } catch (DateTimeParseException e) {} + try { + LocalDate ld = LocalDate.parse(v, DateTimeFormatter.ISO_LOCAL_DATE); + return ld.atStartOfDay(zone); + } catch (DateTimeParseException e) {} + return null; + } + + private static String normalizeUnit(String unit) { + if (unit == null) return null; + String u = unit.trim().toLowerCase(Locale.ROOT); + if ("minutes".equals(u)) return "minute"; + if ("hours".equals(u)) return "hour"; + if ("days".equals(u)) return "day"; + if ("weeks".equals(u)) return "week"; + if ("months".equals(u)) return "month"; + if ("years".equals(u)) return "year"; + if ("minute".equals(u) || "hour".equals(u) || "day".equals(u) || "week".equals(u) || "month".equals(u) || "year".equals(u)) return u; + return null; + } + + private static ZonedDateTime applyAdd(ZonedDateTime dt, long amount, String unit) { + if ("minute".equals(unit)) return dt.plus(amount, ChronoUnit.MINUTES); + if ("hour".equals(unit)) return dt.plus(amount, ChronoUnit.HOURS); + if ("day".equals(unit)) return dt.plus(amount, ChronoUnit.DAYS); + if ("week".equals(unit)) return dt.plus(amount, ChronoUnit.WEEKS); + if ("month".equals(unit)) return dt.plus(amount, ChronoUnit.MONTHS); + if ("year".equals(unit)) return dt.plus(amount, ChronoUnit.YEARS); + throw new IllegalArgumentException("Invalid unit: " + unit); + } + + private static ZonedDateTime startOf(ZonedDateTime dt, String period) { + if ("day".equals(period)) { + return dt.toLocalDate().atStartOfDay(dt.getZone()); + } + if ("week".equals(period)) { + LocalDate date = dt.toLocalDate(); + while (date.getDayOfWeek() != DayOfWeek.MONDAY) date = date.minusDays(1); + return date.atStartOfDay(dt.getZone()); + } + if ("month".equals(period)) { + LocalDate first = YearMonth.from(dt).atDay(1); + return first.atStartOfDay(dt.getZone()); + } + if ("year".equals(period)) { + LocalDate first = LocalDate.of(dt.getYear(), 1, 1); + return first.atStartOfDay(dt.getZone()); + } + throw new IllegalArgumentException("Invalid period: " + period); + } + + private static ZonedDateTime endOf(ZonedDateTime dt, String period) { + if ("day".equals(period)) return startOf(dt, "day").plusDays(1).minusNanos(1); + if ("week".equals(period)) return startOf(dt, "week").plusWeeks(1).minusNanos(1); + if ("month".equals(period)) return startOf(dt, "month").plusMonths(1).minusNanos(1); + if ("year".equals(period)) return startOf(dt, "year").plusYears(1).minusNanos(1); + throw new IllegalArgumentException("Invalid period: " + period); + } +} diff --git a/source/net/yacy/ai/tools/DateTimeTool.java b/source/net/yacy/ai/tools/DateTimeTool.java new file mode 100644 index 000000000..d9ba10d59 --- /dev/null +++ b/source/net/yacy/ai/tools/DateTimeTool.java @@ -0,0 +1,112 @@ +/** + * DateTimeTool + * Copyright 2026 by Michael Peter Christen + * First released 06.02.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.ai.tools; + +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import net.yacy.ai.ToolHandler; + +public class DateTimeTool implements ToolHandler { + + private static final String NAME = "datetime"; + private static final DateTimeFormatter DATETIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + + @Override + public JSONObject definition() throws JSONException { + JSONObject tool = new JSONObject(true); + tool.put("type", "function"); + JSONObject fn = new JSONObject(true); + fn.put("name", NAME); + fn.put("description", "Get the current date and time. Optionally specify an IANA timezone."); + JSONObject params = new JSONObject(true); + params.put("type", "object"); + JSONObject props = new JSONObject(true); + JSONObject tz = new JSONObject(true); + tz.put("type", "string"); + tz.put("description", "IANA timezone name like America/New_York. Defaults to local time zone."); + props.put("timezone", tz); + params.put("properties", props); + params.put("required", new JSONArray()); + fn.put("parameters", params); + tool.put("function", fn); + return tool; + } + + + public int maxCallsPerTurn() { + return 1; + } + + + public String execute(String arguments) { + String timezone = null; + if (arguments != null && !arguments.isEmpty()) { + try { + JSONObject obj = new JSONObject(arguments); + timezone = obj.optString("timezone", null); + } catch (JSONException e) { + return ToolHandler.errorJson("Invalid arguments JSON"); + } + } + + ZoneId localZone = ZoneId.systemDefault(); + ZoneId zone = localZone; + String note = null; + if (timezone != null && !timezone.isEmpty()) { + try { + zone = ZoneId.of(timezone); + } catch (Exception e) { + zone = localZone; + note = "Invalid timezone \"" + timezone + "\". Used local timezone \"" + localZone.getId() + "\"."; + } + } + + ZonedDateTime now = ZonedDateTime.now(zone); + try { + JSONObject result = new JSONObject(true); + result.put("datetime", now.format(DATETIME_FORMAT)); + result.put("timezone", zone.getId()); + result.put("time_of_day", classifyTimeOfDay(now.getHour())); + result.put("utc_offset_minutes", now.getOffset().getTotalSeconds() / 60); + result.put("utc_datetime", Instant.now().toString()); + result.put("epoch_ms", System.currentTimeMillis()); + if (note != null) result.put("note", note); + return result.toString(); + } catch (JSONException e) { + return ToolHandler.errorJson("Failed to build datetime response"); + } + } + + private static String classifyTimeOfDay(int hour) { + if (hour < 0 || hour > 23) return "night"; + if (hour < 6) return "night"; + if (hour < 12) return "morning"; + if (hour < 18) return "afternoon"; + return "evening"; + } +} diff --git a/source/net/yacy/ai/tools/HttpJsonTool.java b/source/net/yacy/ai/tools/HttpJsonTool.java new file mode 100644 index 000000000..16251b3e5 --- /dev/null +++ b/source/net/yacy/ai/tools/HttpJsonTool.java @@ -0,0 +1,244 @@ +/** + * HttpJsonTool + * Copyright 2026 by Michael Peter Christen + * First released 06.02.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.ai.tools; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONTokener; + +import net.yacy.ai.ToolHandler; +import net.yacy.cora.document.id.DigestURL; + +public class HttpJsonTool implements ToolHandler { + + private static final String NAME = "http_json"; + private static final Set<String> ALLOWED_METHODS = new HashSet<>(Arrays.asList("GET", "POST", "PUT", "PATCH", "DELETE")); + private static final Set<String> BLOCKED_HEADERS = new HashSet<>(Arrays.asList("host", "content-length", "connection", "transfer-encoding")); + private static final int DEFAULT_TIMEOUT_MS = 15000; + private static final int MAX_TIMEOUT_MS = 60000; + private static final int DEFAULT_MAX_BYTES = 1_000_000; + private static final int MAX_ALLOWED_BYTES = 2_000_000; + + @Override + public JSONObject definition() throws JSONException { + JSONObject tool = new JSONObject(true); + tool.put("type", "function"); + JSONObject fn = new JSONObject(true); + fn.put("name", NAME); + fn.put("description", "Fetch an HTTP endpoint and return parsed JSON response."); + + JSONObject params = new JSONObject(true); + params.put("type", "object"); + JSONObject props = new JSONObject(true); + + JSONObject url = new JSONObject(true); + url.put("type", "string"); + url.put("description", "Absolute http or https URL."); + props.put("url", url); + + JSONObject method = new JSONObject(true); + method.put("type", "string"); + method.put("description", "HTTP method. Allowed: GET, POST, PUT, PATCH, DELETE."); + props.put("method", method); + + JSONObject headers = new JSONObject(true); + headers.put("type", "object"); + headers.put("description", "Optional request headers (string values)."); + props.put("headers", headers); + + JSONObject body = new JSONObject(true); + body.put("description", "Optional JSON body as object/array/string."); + props.put("body", body); + + JSONObject timeout = new JSONObject(true); + timeout.put("type", "integer"); + timeout.put("description", "Timeout in milliseconds (max 60000)."); + props.put("timeout_ms", timeout); + + JSONObject maxBytes = new JSONObject(true); + maxBytes.put("type", "integer"); + maxBytes.put("description", "Maximum response size in bytes (max 2000000)."); + props.put("max_bytes", maxBytes); + + params.put("properties", props); + params.put("required", new JSONArray().put("url")); + fn.put("parameters", params); + tool.put("function", fn); + return tool; + } + + + public int maxCallsPerTurn() { + return 1; + } + + + public String execute(String arguments) { + final JSONObject args; + try { + args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments); + } catch (JSONException e) { + return ToolHandler.errorJson("Invalid arguments JSON. don't try again"); + } + + final String urlRaw = args.optString("url", "").trim(); + if (urlRaw.isEmpty()) return ToolHandler.errorJson("Missing url. don't try again"); + + final DigestURL digestUrl; + try { + digestUrl = new DigestURL(urlRaw); + } catch (Exception e) { + return ToolHandler.errorJson("Invalid URL. don't try again"); + } + final String protocol = digestUrl.getProtocol(); + if (!"http".equalsIgnoreCase(protocol) && !"https".equalsIgnoreCase(protocol)) { + return ToolHandler.errorJson("Only http/https URLs are allowed. don't try again"); + } + + final String method = args.optString("method", "GET").trim().toUpperCase(); + if (!ALLOWED_METHODS.contains(method)) { + return ToolHandler.errorJson("Unsupported method " + method + ". don't try again"); + } + final int timeoutMs = clampPositive(args.optInt("timeout_ms", DEFAULT_TIMEOUT_MS), DEFAULT_TIMEOUT_MS, MAX_TIMEOUT_MS); + final int maxBytes = clampPositive(args.optInt("max_bytes", DEFAULT_MAX_BYTES), DEFAULT_MAX_BYTES, MAX_ALLOWED_BYTES); + final Object body = args.opt("body"); + final JSONObject headers = args.optJSONObject("headers"); + + HttpURLConnection conn = null; + try { + final URL url = new URI(digestUrl.toNormalform(true)).toURL(); + conn = (HttpURLConnection) url.openConnection(); + conn.setInstanceFollowRedirects(true); + conn.setRequestMethod(method); + conn.setConnectTimeout(timeoutMs); + conn.setReadTimeout(timeoutMs); + conn.setRequestProperty("Accept", "application/json"); + applyHeaders(conn, headers); + + if (body != null && !"GET".equals(method)) { + conn.setDoOutput(true); + if (conn.getRequestProperty("Content-Type") == null) { + conn.setRequestProperty("Content-Type", "application/json"); + } + byte[] payload = encodeBody(body); + try (OutputStream os = conn.getOutputStream()) { + os.write(payload); + } + } + + int status = conn.getResponseCode(); + String contentType = conn.getHeaderField("Content-Type"); + if (contentType == null) contentType = ""; + final InputStream stream = status >= 400 ? conn.getErrorStream() : conn.getInputStream(); + if (stream == null) { + return ToolHandler.errorJson("Empty response body. don't try again"); + } + final String text; + try (InputStream in = stream) { + text = new String(readLimited(in, maxBytes), StandardCharsets.UTF_8); + } + if (text.trim().isEmpty()) { + return ToolHandler.errorJson("Empty response body. don't try again"); + } + + final Object parsed; + try { + parsed = new JSONTokener(text).nextValue(); + } catch (JSONException e) { + return ToolHandler.errorJson("Response is not valid JSON. don't try again"); + } + if (!(parsed instanceof JSONObject) && !(parsed instanceof JSONArray)) { + return ToolHandler.errorJson("Response is not JSON object/array. don't try again"); + } + + JSONObject result = new JSONObject(true); + result.put("url", digestUrl.toNormalform(true)); + result.put("method", method); + result.put("status", status); + result.put("content_type", contentType); + result.put("json", parsed); + return result.toString(); + } catch (IOException e) { + return ToolHandler.errorJson("Fetch error: " + e.getMessage() + ". don't try again"); + } catch (URISyntaxException e) { + return ToolHandler.errorJson("Invalid URL syntax. don't try again"); + } catch (JSONException e) { + return ToolHandler.errorJson("Failed to build tool response. don't try again"); + } finally { + if (conn != null) conn.disconnect(); + } + } + + private static void applyHeaders(HttpURLConnection conn, JSONObject headers) { + if (conn == null || headers == null) return; + for (String key : headers.keySet()) { + if (key == null) continue; + final String normalized = key.trim(); + if (normalized.isEmpty()) continue; + if (BLOCKED_HEADERS.contains(normalized.toLowerCase())) continue; + final String value = headers.optString(key, ""); + conn.setRequestProperty(normalized, value); + } + } + + private static byte[] encodeBody(Object body) { + if (body == null) return new byte[0]; + if (body instanceof JSONObject) return ((JSONObject) body).toString().getBytes(StandardCharsets.UTF_8); + if (body instanceof JSONArray) return ((JSONArray) body).toString().getBytes(StandardCharsets.UTF_8); + if (body instanceof String) return ((String) body).getBytes(StandardCharsets.UTF_8); + return String.valueOf(body).getBytes(StandardCharsets.UTF_8); + } + + private static byte[] readLimited(InputStream in, int maxBytes) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int total = 0; + int read; + while ((read = in.read(buffer)) != -1) { + total += read; + if (total > maxBytes) { + throw new IOException("Response too large"); + } + out.write(buffer, 0, read); + } + return out.toByteArray(); + } + + private static int clampPositive(int value, int defaultValue, int maxValue) { + if (value <= 0) return defaultValue; + if (value > maxValue) return maxValue; + return value; + } +} diff --git a/source/net/yacy/ai/tools/NumberParserTool.java b/source/net/yacy/ai/tools/NumberParserTool.java new file mode 100644 index 000000000..9880de5e5 --- /dev/null +++ b/source/net/yacy/ai/tools/NumberParserTool.java @@ -0,0 +1,403 @@ +/** + * NumberParserTool + * Copyright 2026 by Michael Peter Christen + * First released 07.02.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.ai.tools; + +import java.math.BigDecimal; +import java.text.Normalizer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import net.yacy.ai.ToolHandler; + +public class NumberParserTool implements ToolHandler { + + private static final String NAME = "number_parser"; + + private enum Lang { + DE, EN, FR, ES, IT + } + + private static final class Lexicon { + final Map<String, Integer> small = new HashMap<>(); + final Map<String, Integer> scales = new HashMap<>(); + final String decimalMarker; + Lexicon(String decimalMarker) { + this.decimalMarker = decimalMarker; + } + } + + private static final Map<Lang, Lexicon> LEX = buildLexicons(); + + @Override + public JSONObject definition() throws JSONException { + JSONObject tool = new JSONObject(true); + tool.put("type", "function"); + JSONObject fn = new JSONObject(true); + fn.put("name", NAME); + fn.put("description", "Parse written number words into numeric values. Supports de/en/fr/es/it."); + + JSONObject params = new JSONObject(true); + params.put("type", "object"); + JSONObject props = new JSONObject(true); + + props.put("text", new JSONObject(true) + .put("type", "string") + .put("description", "Number words, e.g. einundzwanzig, five hundred seven, vingt-et-un.")); + + props.put("language", new JSONObject(true) + .put("type", "string") + .put("description", "Optional: de|en|fr|es|it|auto (default auto).")); + + props.put("strict", new JSONObject(true) + .put("type", "boolean") + .put("description", "If true (default), all tokens must be recognized as number words.")); + + params.put("properties", props); + params.put("required", new JSONArray().put("text")); + fn.put("parameters", params); + tool.put("function", fn); + return tool; + } + + @Override + public int maxCallsPerTurn() { + return 10; + } + + @Override + public String execute(String arguments) { + final JSONObject args; + try { + args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments); + } catch (JSONException e) { + return ToolHandler.errorJson("Invalid arguments JSON"); + } + + final String text = args.optString("text", "").trim(); + if (text.isEmpty()) return ToolHandler.errorJson("Missing text"); + + final String language = args.optString("language", "auto").trim().toLowerCase(Locale.ROOT); + final boolean strict = args.optBoolean("strict", true); + + ParseResult best = null; + Lang bestLang = null; + + if (!"auto".equals(language)) { + Lang lang = parseLang(language); + if (lang == null) return ToolHandler.errorJson("Invalid language. Use de|en|fr|es|it|auto"); + ParseResult r = parseWithLang(text, lang, strict); + if (r == null) return ToolHandler.errorJson("Could not parse number words"); + best = r; + bestLang = lang; + } else { + for (Lang lang : Lang.values()) { + ParseResult r = parseWithLang(text, lang, strict); + if (r == null) continue; + if (best == null || r.score > best.score) { + best = r; + bestLang = lang; + } + } + if (best == null) return ToolHandler.errorJson("Could not parse number words in supported languages"); + } + + try { + JSONObject out = new JSONObject(true); + out.put("tool", NAME); + out.put("text", text); + out.put("language", bestLang.name().toLowerCase(Locale.ROOT)); + out.put("value", best.value.stripTrailingZeros().toPlainString()); + return out.toString(); + } catch (JSONException e) { + return ToolHandler.errorJson("Failed to build number parser response"); + } + } + + private static Lang parseLang(String code) { + if ("de".equals(code)) return Lang.DE; + if ("en".equals(code)) return Lang.EN; + if ("fr".equals(code)) return Lang.FR; + if ("es".equals(code)) return Lang.ES; + if ("it".equals(code)) return Lang.IT; + return null; + } + + private static final class ParseResult { + final BigDecimal value; + final int score; + ParseResult(BigDecimal value, int score) { + this.value = value; + this.score = score; + } + } + + private static ParseResult parseWithLang(String input, Lang lang, boolean strict) { + String normalized = normalize(input); + List<String> tokens = tokenizeByLanguage(normalized, lang); + if (tokens.isEmpty()) return null; + + Lexicon lx = LEX.get(lang); + if (lx == null) return null; + + int decimalIndex = tokens.indexOf(lx.decimalMarker); + BigDecimal intPart; + int consumedInt; + if (decimalIndex >= 0) { + ParseSpan left = parseIntegerTokens(tokens.subList(0, decimalIndex), lang, strict); + if (left == null) return null; + intPart = new BigDecimal(left.value); + consumedInt = left.consumed; + + ParseFraction frac = parseFractionTokens(tokens.subList(decimalIndex + 1, tokens.size()), lang, strict); + if (frac == null) return null; + BigDecimal value = intPart.add(frac.value); + int score = consumedInt + frac.consumed + 1; + return new ParseResult(value, score); + } + + ParseSpan span = parseIntegerTokens(tokens, lang, strict); + if (span == null) return null; + return new ParseResult(new BigDecimal(span.value), span.consumed); + } + + private static final class ParseSpan { + final long value; + final int consumed; + ParseSpan(long value, int consumed) { + this.value = value; + this.consumed = consumed; + } + } + + private static ParseSpan parseIntegerTokens(List<String> tokens, Lang lang, boolean strict) { + if (tokens.isEmpty()) return new ParseSpan(0L, 0); + Lexicon lx = LEX.get(lang); + long total = 0L; + long current = 0L; + int consumed = 0; + + for (int i = 0; i < tokens.size(); i++) { + String t = tokens.get(i); + if (t == null || t.isEmpty() || "and".equals(t) || "et".equals(t) || "y".equals(t) || "und".equals(t) || "e".equals(t)) { + continue; + } + + if (isDigits(t)) { + long v = Long.parseLong(t); + current += v; + consumed++; + continue; + } + + // French special: quatre vingt => 80 + if (lang == Lang.FR && "quatre".equals(t) && i + 1 < tokens.size() && "vingt".equals(tokens.get(i + 1))) { + current += 80; + consumed += 2; + i++; + continue; + } + + Integer small = lx.small.get(t); + if (small != null) { + current += small.intValue(); + consumed++; + continue; + } + + if ("hundred".equals(t) || "hundert".equals(t) || "cent".equals(t) || "cien".equals(t) || "ciento".equals(t) || "cento".equals(t)) { + if (current == 0) current = 1; + current *= 100; + consumed++; + continue; + } + + Integer scale = lx.scales.get(t); + if (scale != null) { + if (current == 0) current = 1; + total += current * scale.longValue(); + current = 0; + consumed++; + continue; + } + + if (strict) return null; + } + return new ParseSpan(total + current, consumed); + } + + private static final class ParseFraction { + final BigDecimal value; + final int consumed; + ParseFraction(BigDecimal value, int consumed) { + this.value = value; + this.consumed = consumed; + } + } + + private static ParseFraction parseFractionTokens(List<String> tokens, Lang lang, boolean strict) { + if (tokens.isEmpty()) return new ParseFraction(BigDecimal.ZERO, 0); + Lexicon lx = LEX.get(lang); + StringBuilder digits = new StringBuilder(); + int consumed = 0; + for (String t : tokens) { + if (t == null || t.isEmpty()) continue; + if (isDigits(t)) { + digits.append(t); + consumed++; + continue; + } + Integer v = lx.small.get(t); + if (v != null && v.intValue() >= 0 && v.intValue() <= 9) { + digits.append(v.intValue()); + consumed++; + continue; + } + if (strict) return null; + } + if (digits.length() == 0) return new ParseFraction(BigDecimal.ZERO, consumed); + BigDecimal frac = new BigDecimal("0." + digits.toString()); + return new ParseFraction(frac, consumed); + } + + private static boolean isDigits(String t) { + if (t == null || t.isEmpty()) return false; + for (int i = 0; i < t.length(); i++) if (!Character.isDigit(t.charAt(i))) return false; + return true; + } + + private static String normalize(String s) { + String n = s == null ? "" : s.toLowerCase(Locale.ROOT); + n = n.replace('’', '\''); + n = n.replace('-', ' '); + n = n.replace("ß", "ss"); + n = Normalizer.normalize(n, Normalizer.Form.NFD).replaceAll("\\p{M}+", ""); + n = n.replaceAll("[^a-z0-9' ]", " "); + n = n.replaceAll("\\s+", " ").trim(); + return n; + } + + private static List<String> tokenizeByLanguage(String normalized, Lang lang) { + if (normalized.isEmpty()) return new ArrayList<>(); + String s = normalized; + + if (lang == Lang.DE) { + // Split common German compounding boundaries in one pass to avoid + // destructive re-replacements (e.g. "hundert" -> "... t"). + s = s.replaceAll("(millionen|million)", " million "); + s = s.replaceAll("(milliarden|milliarde)", " milliarde "); + s = s.replaceAll("(tausend)", " tausend "); + s = s.replaceAll("(hundert|hunder)", " hundert "); + s = s.replace("tausend", " tausend "); + // Split forms like "hundertundsieben" -> "hundert und sieben". + s = s.replaceAll("\\bund(ein|eine|eins|zwei|drei|vier|funf|fuenf|sechs|sieben|acht|neun|zehn|elf|zwolf|zwoelf|dreizehn|vierzehn|funfzehn|fuenfzehn|sechzehn|siebzehn|achtzehn|neunzehn|zwanzig|dreissig|dreiig|vierzig|funfzig|fuenfzig|sechzig|siebzig|achtzig|neunzig)\\b", "und $1"); + // Split only canonical DE one-and-tens compounds (e.g. einundzwanzig). + s = s.replaceAll("(ein|eine|zwei|drei|vier|funf|fuenf|sechs|sieben|acht|neun)und(zwanzig|dreissig|dreiig|vierzig|funfzig|fuenfzig|sechzig|siebzig|achtzig|neunzig)", "$1 und $2"); + s = s.replace("eins", "ein"); + } + + if (lang == Lang.ES) { + s = s.replaceAll("veinti([a-z]+)", "veinte $1"); + s = s.replaceAll("dieci([a-z]+)", "diez $1"); + } + + if (lang == Lang.IT) { + s = s.replaceAll("(venti|trenta|quaranta|cinquanta|sessanta|settanta|ottanta|novanta)([a-z]+)", "$1 $2"); + } + + if (lang == Lang.FR) { + s = s.replace("quatre vingt", "quatre vingt"); + } + + s = s.replaceAll("\\s+", " ").trim(); + if (s.isEmpty()) return new ArrayList<>(); + return new ArrayList<>(Arrays.asList(s.split(" "))); + } + + private static Map<Lang, Lexicon> buildLexicons() { + Map<Lang, Lexicon> map = new HashMap<>(); + + Lexicon de = new Lexicon("komma"); + putAll(de.small, + "null",0, "ein",1, "eine",1, "einen",1, "eins",1, "zwo",2, "zwei",2, "drei",3, "vier",4, "funf",5, "fuenf",5, "sechs",6, + "sieben",7, "acht",8, "neun",9, "zehn",10, "elf",11, "zwolf",12, "zwoelf",12, "dreizehn",13, "vierzehn",14, + "funfzehn",15, "fuenfzehn",15, "sechzehn",16, "siebzehn",17, "achtzehn",18, "neunzehn",19, + "zwanzig",20, "dreissig",30, "dreiig",30, "vierzig",40, "funfzig",50, "fuenfzig",50, "sechzig",60, "siebzig",70, + "achtzig",80, "neunzig",90); + putAll(de.scales, "tausend",1000, "million",1000000, "millionen",1000000, "milliarde",1000000000, "milliarden",1000000000); + map.put(Lang.DE, de); + + Lexicon en = new Lexicon("point"); + putAll(en.small, + "zero",0, "one",1, "two",2, "three",3, "four",4, "five",5, "six",6, "seven",7, "eight",8, "nine",9, + "ten",10, "eleven",11, "twelve",12, "thirteen",13, "fourteen",14, "fifteen",15, "sixteen",16, + "seventeen",17, "eighteen",18, "nineteen",19, + "twenty",20, "thirty",30, "forty",40, "fifty",50, "sixty",60, "seventy",70, "eighty",80, "ninety",90); + putAll(en.scales, "thousand",1000, "million",1000000, "billion",1000000000); + map.put(Lang.EN, en); + + Lexicon fr = new Lexicon("virgule"); + putAll(fr.small, + "zero",0, "un",1, "une",1, "deux",2, "trois",3, "quatre",4, "cinq",5, "six",6, "sept",7, "huit",8, "neuf",9, + "dix",10, "onze",11, "douze",12, "treize",13, "quatorze",14, "quinze",15, "seize",16, + "vingt",20, "trente",30, "quarante",40, "cinquante",50, "soixante",60, "soixantedix",70, + "quatrevingt",80, "quatrevingtdix",90); + putAll(fr.scales, "mille",1000, "million",1000000, "milliard",1000000000); + map.put(Lang.FR, fr); + + Lexicon es = new Lexicon("coma"); + putAll(es.small, + "cero",0, "uno",1, "una",1, "dos",2, "tres",3, "cuatro",4, "cinco",5, "seis",6, "siete",7, "ocho",8, "nueve",9, + "diez",10, "once",11, "doce",12, "trece",13, "catorce",14, "quince",15, "dieciseis",16, + "diecisiete",17, "dieciocho",18, "diecinueve",19, + "veinte",20, "treinta",30, "cuarenta",40, "cincuenta",50, "sesenta",60, "setenta",70, "ochenta",80, "noventa",90, + "cien",100, "ciento",100, "doscientos",200, "trescientos",300, "cuatrocientos",400, "quinientos",500, + "seiscientos",600, "setecientos",700, "ochocientos",800, "novecientos",900); + putAll(es.scales, "mil",1000, "millon",1000000, "millones",1000000, "milmillon",1000000000); + map.put(Lang.ES, es); + + Lexicon it = new Lexicon("virgola"); + putAll(it.small, + "zero",0, "uno",1, "una",1, "due",2, "tre",3, "quattro",4, "cinque",5, "sei",6, "sette",7, "otto",8, "nove",9, + "dieci",10, "undici",11, "dodici",12, "tredici",13, "quattordici",14, "quindici",15, "sedici",16, + "diciassette",17, "diciotto",18, "diciannove",19, + "venti",20, "trenta",30, "quaranta",40, "cinquanta",50, "sessanta",60, "settanta",70, "ottanta",80, "novanta",90, + "cento",100, "duecento",200, "trecento",300, "quattrocento",400, "cinquecento",500, + "seicento",600, "settecento",700, "ottocento",800, "novecento",900); + putAll(it.scales, "mille",1000, "mila",1000, "milione",1000000, "milioni",1000000, "miliardo",1000000000); + map.put(Lang.IT, it); + + return map; + } + + private static void putAll(Map<String, Integer> m, Object... kv) { + for (int i = 0; i + 1 < kv.length; i += 2) { + m.put(String.valueOf(kv[i]), Integer.valueOf(((Number) kv[i + 1]).intValue())); + } + } +} diff --git a/source/net/yacy/ai/tools/PromptToMermaidTool.java b/source/net/yacy/ai/tools/PromptToMermaidTool.java new file mode 100644 index 000000000..dc685f0d9 --- /dev/null +++ b/source/net/yacy/ai/tools/PromptToMermaidTool.java @@ -0,0 +1,314 @@ +/** + * PromptToMermaidTool + * Copyright 2026 by Michael Peter Christen + * First released 06.02.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.ai.tools; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.regex.Pattern; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONTokener; + +import net.yacy.ai.ToolHandler; +import net.yacy.ai.LLM; + +public class PromptToMermaidTool implements ToolHandler { + + private static final String NAME = "prompt_to_mermaid"; + //private static final List<String> DIAGRAM_TYPES = Arrays.asList("flowchart", "sequence", "graph"); + + private static final String SYSTEM_PROMPT = + "You convert natural-language process descriptions into Mermaid diagrams with deterministic, transparent behavior. " + + "Prefer explicit structure only from user text. Do not invent hidden steps. " + + "Support flowchart, sequence diagram, and graph only. " + + "If prompt is ambiguous or too vague, return fallback with plain-language summary and no forced diagram. " + + "Return ONLY JSON matching the requested fields."; + + private static final JSONObject RESPONSE_SCHEMA = buildResponseSchema(); + + private static final Pattern EMAIL_PATTERN = Pattern.compile("(?i)\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}\\b"); + private static final Pattern PHONE_PATTERN = Pattern.compile("\\b\\+?[0-9][0-9\\s().-]{6,}[0-9]\\b"); + + @Override + public JSONObject definition() throws JSONException { + JSONObject tool = new JSONObject(true); + tool.put("type", "function"); + JSONObject fn = new JSONObject(true); + fn.put("name", NAME); + fn.put("description", "Convert a user prompt into a Mermaid diagram with fallback summary when prompt is ambiguous."); + + JSONObject params = new JSONObject(true); + params.put("type", "object"); + JSONObject props = new JSONObject(true); + + JSONObject prompt = new JSONObject(true); + prompt.put("type", "string"); + prompt.put("description", "Natural-language diagram request."); + props.put("prompt", prompt); + + JSONObject preferredType = new JSONObject(true); + preferredType.put("type", "string"); + preferredType.put("description", "Optional hint: flowchart, sequence, graph."); + props.put("preferred_diagram_type", preferredType); + + params.put("properties", props); + params.put("required", new JSONArray().put("prompt")); + fn.put("parameters", params); + tool.put("function", fn); + return tool; + } + + + public int maxCallsPerTurn() { + return 1; + } + + + public String execute(String arguments) { + final JSONObject args; + try { + args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments); + } catch (JSONException e) { + return fallback("Invalid arguments JSON", "", null, Arrays.asList("Could not parse tool arguments")); + } + + String prompt = args.optString("prompt", "").trim(); + String preferredType = normalizeType(args.optString("preferred_diagram_type", null)); + if (prompt.isEmpty()) { + return fallback("Missing prompt", "", preferredType, Arrays.asList("No prompt text provided")); + } + + String redactedPrompt = redactSensitive(prompt); + if (isVague(redactedPrompt)) { + return fallback("Prompt is too vague for deterministic diagram generation", redactedPrompt, preferredType, + Arrays.asList("Add concrete steps, actors, or relationships", "Fallback used to avoid hallucinated structure")); + } + + LLM.LLMModel llmModel = LLM.llmFromUsage(LLM.LLMUsage.chat); + if (llmModel == null || llmModel.llm == null || llmModel.model == null || llmModel.model.isEmpty()) { + return fallback("No chat model configured for Mermaid generation", redactedPrompt, preferredType, + Arrays.asList("Configure a chat model in AI production models")); + } + + final String userPrompt = buildUserPrompt(redactedPrompt, preferredType); + try { + String raw = queryModel(llmModel, userPrompt, RESPONSE_SCHEMA); + JSONObject parsed = parseResponseObject(raw); + return normalizeResult(parsed, redactedPrompt, preferredType).toString(); + } catch (IOException e) { + try { + String raw = queryModel(llmModel, userPrompt, null); + JSONObject parsed = parseResponseObject(raw); + return normalizeResult(parsed, redactedPrompt, preferredType).toString(); + } catch (Exception retryError) { + return fallback("LLM generation failed", redactedPrompt, preferredType, + Arrays.asList("Model call failed: " + retryError.getMessage())); + } + } catch (Exception e) { + return fallback("Invalid model output", redactedPrompt, preferredType, + Arrays.asList("Failed to parse model output: " + e.getMessage())); + } + } + + private static String queryModel(LLM.LLMModel llmModel, String userPrompt, JSONObject schema) throws IOException { + try { + LLM.Context context = new LLM.Context(SYSTEM_PROMPT); + context.addPrompt(userPrompt); + return llmModel.llm.chat(llmModel.model, context, schema, 1200); + } catch (JSONException e) { + throw new IOException(e.getMessage(), e); + } + } + + private static JSONObject parseResponseObject(String raw) throws JSONException { + if (raw == null) throw new JSONException("Empty model response"); + String trimmed = raw.trim(); + if (trimmed.startsWith("```")) { + int firstNewline = trimmed.indexOf('\n'); + if (firstNewline >= 0) { + trimmed = trimmed.substring(firstNewline + 1); + } + int lastFence = trimmed.lastIndexOf("```"); + if (lastFence >= 0) { + trimmed = trimmed.substring(0, lastFence).trim(); + } + } + return new JSONObject(new JSONTokener(trimmed)); + } + + private static JSONObject normalizeResult(JSONObject parsed, String prompt, String preferredType) throws JSONException { + String status = parsed.optString("status", "fallback").trim().toLowerCase(Locale.ROOT); + if (!"success".equals(status) && !"fallback".equals(status)) status = "fallback"; + + String diagramType = normalizeType(parsed.optString("diagram_type", null)); + String mermaidCode = parsed.optString("mermaid_code", "").trim(); + String reasoning = parsed.optString("reasoning", "").trim(); + JSONArray warnings = parsed.optJSONArray("warnings"); + if (warnings == null) warnings = new JSONArray(); + + if (reasoning.isEmpty()) { + reasoning = "Generated from prompt using local structured conversion rules and LLM formatting."; + } + + if ("success".equals(status)) { + if (diagramType == null) { + status = "fallback"; + warnings.put("diagram_type missing or invalid"); + } + if (!isValidMermaid(mermaidCode, diagramType)) { + status = "fallback"; + warnings.put("Invalid Mermaid syntax for declared diagram_type"); + } + } + + if ("fallback".equals(status)) { + String fallbackReason = reasoning.isEmpty() ? "Prompt could not be deterministically mapped to a clear diagram." : reasoning; + return buildResult("fallback", null, "", fallbackReason, warnings, prompt, preferredType); + } + return buildResult("success", diagramType, mermaidCode, reasoning, warnings, prompt, preferredType); + } + + private static JSONObject buildResult(String status, String diagramType, String mermaidCode, String reasoning, + JSONArray warnings, String prompt, String preferredType) throws JSONException { + JSONObject out = new JSONObject(true); + out.put("status", status); + out.put("diagram_type", diagramType == null ? JSONObject.NULL : diagramType); + out.put("mermaid_code", mermaidCode == null ? "" : mermaidCode); + out.put("reasoning", reasoning == null ? "" : reasoning); + out.put("warnings", warnings == null ? new JSONArray() : warnings); + if (prompt != null && !prompt.isEmpty()) out.put("prompt_redacted", prompt); + if (preferredType != null) out.put("preferred_diagram_type", preferredType); + return out; + } + + private static boolean isValidMermaid(String code, String diagramType) { + if (code == null || code.trim().isEmpty() || diagramType == null) return false; + String text = code.trim().toLowerCase(Locale.ROOT); + if ("flowchart".equals(diagramType)) return text.startsWith("flowchart"); + if ("sequence".equals(diagramType)) return text.startsWith("sequencediagram"); + if ("graph".equals(diagramType)) return text.startsWith("graph"); + return false; + } + + private static String buildUserPrompt(String prompt, String preferredType) { + StringBuilder sb = new StringBuilder(); + sb.append("Convert the following user request into Mermaid.\n"); + sb.append("Requirements:\n"); + sb.append("- deterministic interpretation only\n"); + sb.append("- supported diagram types: flowchart, sequence, graph\n"); + sb.append("- if ambiguous, return fallback with summary reasoning\n"); + sb.append("- include warnings for assumptions\n"); + sb.append("- output JSON only with keys: status, diagram_type, mermaid_code, reasoning, warnings\n"); + if (preferredType != null) { + sb.append("- user preferred diagram type: ").append(preferredType).append("\n"); + } + sb.append("\nUser prompt:\n"); + sb.append(prompt); + return sb.toString(); + } + + private static boolean isVague(String prompt) { + if (prompt == null) return true; + String trimmed = prompt.trim(); + if (trimmed.length() < 24) return true; + String[] tokens = trimmed.split("\\s+"); + if (tokens.length < 6) return true; + String lower = trimmed.toLowerCase(Locale.ROOT); + return !(lower.contains("then") || lower.contains("first") || lower.contains("after") + || lower.contains("if") || lower.contains("when") || lower.contains("->") + || lower.contains("sequence") || lower.contains("workflow") || lower.contains("relationship")); + } + + private static String redactSensitive(String prompt) { + if (prompt == null) return ""; + String s = EMAIL_PATTERN.matcher(prompt).replaceAll("[EMAIL]"); + s = PHONE_PATTERN.matcher(s).replaceAll("[PHONE]"); + return s; + } + + private static String normalizeType(String value) { + if (value == null) return null; + String type = value.trim().toLowerCase(Locale.ROOT); + if (type.isEmpty()) return null; + if ("flowchart".equals(type) || "sequence".equals(type) || "graph".equals(type)) return type; + if ("sequence diagram".equals(type) || "sequencediagram".equals(type)) return "sequence"; + if ("flow".equals(type)) return "flowchart"; + return null; + } + + private static String fallback(String reason, String prompt, String preferredType, List<String> warnings) { + try { + JSONArray w = new JSONArray(); + if (warnings != null) { + for (String warning : warnings) { + if (warning != null && !warning.isEmpty()) w.put(warning); + } + } + return buildResult("fallback", null, "", + reason == null ? "Fallback used due to insufficient structure in prompt." : reason, + w, prompt, preferredType).toString(); + } catch (JSONException e) { + return ToolHandler.errorJson("Failed to build fallback response"); + } + } + + private static JSONObject buildResponseSchema() { + try { + JSONObject schema = new JSONObject(true); + schema.put("title", "PromptToMermaidResult"); + schema.put("type", "object"); + JSONObject properties = new JSONObject(true); + + JSONObject status = new JSONObject(true); + status.put("type", "string"); + status.put("enum", new JSONArray().put("success").put("fallback")); + properties.put("status", status); + + JSONObject diagramType = new JSONObject(true); + diagramType.put("type", "string"); + diagramType.put("enum", new JSONArray().put("flowchart").put("sequence").put("graph")); + properties.put("diagram_type", diagramType); + + JSONObject mermaid = new JSONObject(true); + mermaid.put("type", "string"); + properties.put("mermaid_code", mermaid); + + JSONObject reasoning = new JSONObject(true); + reasoning.put("type", "string"); + properties.put("reasoning", reasoning); + + JSONObject warnings = new JSONObject(true); + warnings.put("type", "array"); + warnings.put("items", new JSONObject(true).put("type", "string")); + properties.put("warnings", warnings); + + schema.put("properties", properties); + schema.put("required", new JSONArray().put("status").put("diagram_type").put("mermaid_code").put("reasoning").put("warnings")); + return schema; + } catch (JSONException e) { + return null; + } + } +} diff --git a/source/net/yacy/ai/tools/SelfReflectTool.java b/source/net/yacy/ai/tools/SelfReflectTool.java new file mode 100644 index 000000000..d1f99d099 --- /dev/null +++ b/source/net/yacy/ai/tools/SelfReflectTool.java @@ -0,0 +1,115 @@ +/** + * SelfReflectTool + * Copyright 2026 by Michael Peter Christen + * First released 06.02.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.ai.tools; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import net.yacy.ai.ToolHandler; + +public class SelfReflectTool implements ToolHandler { + + private static final String NAME = "self_reflect"; + + private static final String PURPOSE = + "YaCy AI aims to be the good one: privacy-aware, open, and useful AI that combines LLM responses with transparent tools and retrieval from YaCy search."; + + private static final List<String> FACTS = Collections.unmodifiableList(Arrays.asList( + "You can choose default dialog augmentation mode: no search, local search, or global search.", + "You can click the search button for the current message to attach search results as context without uploading a file.", + "You can run with no search and attach your own files instead, so you control exactly what context is sent.", + "You can attach PNG/JPG images and text files (.txt, .md, .tex), preview them, download them, or open them in a full view.", + "You can drag and drop supported files directly into the composer.", + "You can see transparency in action: search results are turned into virtual attachments so you can inspect which documents contributed to the answer. Click on the attachment button in your prompt to see the search answers!", + "You can see each tool call with wrench icons (click on it!) in assistant messages and open per-call details (arguments and tool response).", + "You can use deterministic tools such as datetime, calculator, webfetch, http_json, and table_ops to improve answer quality.", + "You can copy assistant answers, trim user prompts, and delete a user/assistant turn pair for iterative editing.", + "You can clear chat history, download the chat as JSON, and upload it again to continue later.", + "You can show or hide the system prompt in the interface to better understand chat behavior.", + "You can read markdown-formatted answers, including code blocks with syntax highlighting and collapsible model-thought sections when present.", + "You can use YaCy AI components beyond chat via standards-oriented integration points such as RAG proxy and MCP tooling." + )); + + private static final List<String> IDEAS = Collections.unmodifiableList(Arrays.asList( + "Transparency, not mystery: ranking, retrieval, and model use should be explainable and inspectable.", + "Autonomy, no dependency: avoid hidden data flows and centralized lock-in whenever possible.", + "Collaboration with user control: intelligence should grow from shared knowledge while users stay in charge.", + "Reproducibility: users should be able to choose models and verify outcomes.", + "Integration instead of isolation: AI should be part of the open web, not a wall around it.", + "Teach openly: communicate purpose and benefit so AI is understandable, not opaque or intimidating.", + "YaCy extends open-search principles into AI: openness, autonomy, and trust." + )); + + private static final String TALKS_URL = "https://www.youtube.com/orbiterlab"; + + @Override + public JSONObject definition() throws JSONException { + JSONObject tool = new JSONObject(true); + tool.put("type", "function"); + JSONObject fn = new JSONObject(true); + fn.put("name", NAME); + fn.put("description", "This is the help function. Explain what YaCy AI does and why it is built this way. Use when user asks about abilities, purpose, or philosophy."); + JSONObject params = new JSONObject(true); + params.put("type", "object"); + params.put("properties", new JSONObject(true)); + params.put("required", new JSONArray()); + fn.put("parameters", params); + tool.put("function", fn); + return tool; + } + + + public int maxCallsPerTurn() { + return 1; + } + + + public String execute(String arguments) { + try { + JSONObject result = new JSONObject(true); + result.put("tool", NAME); + result.put("purpose", PURPOSE); + result.put("facts", new JSONArray(FACTS)); + result.put("ideas", new JSONArray(IDEAS)); + result.put("talks_url", TALKS_URL); + return result.toString(); + } catch (JSONException e) { + return ToolHandler.errorJson("Failed to build self_reflect response"); + } + } + + public static List<String> facts() { + return FACTS; + } + + public static List<String> ideas() { + return IDEAS; + } + + public static String purpose() { + return PURPOSE; + } +} diff --git a/source/net/yacy/ai/tools/TableOpsTool.java b/source/net/yacy/ai/tools/TableOpsTool.java new file mode 100644 index 000000000..fe1b71699 --- /dev/null +++ b/source/net/yacy/ai/tools/TableOpsTool.java @@ -0,0 +1,285 @@ +/** + * TableOpsTool + * Copyright 2026 by Michael Peter Christen + * First released 06.02.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.ai.tools; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import net.yacy.ai.ToolHandler; + +public class TableOpsTool implements ToolHandler { + + private static final String NAME = "table_ops"; + + @Override + public JSONObject definition() throws JSONException { + JSONObject tool = new JSONObject(true); + tool.put("type", "function"); + JSONObject fn = new JSONObject(true); + fn.put("name", NAME); + fn.put("description", "Apply deterministic table operations to a JSON array of row objects."); + + JSONObject params = new JSONObject(true); + params.put("type", "object"); + JSONObject props = new JSONObject(true); + + JSONObject rows = new JSONObject(true); + rows.put("type", "array"); + rows.put("description", "Input table rows as an array of objects."); + props.put("rows", rows); + + JSONObject operations = new JSONObject(true); + operations.put("type", "array"); + operations.put("description", "Pipeline of operations: filter, sort, limit, project, group_count."); + props.put("operations", operations); + + params.put("properties", props); + params.put("required", new JSONArray().put("rows").put("operations")); + fn.put("parameters", params); + tool.put("function", fn); + return tool; + } + + + public int maxCallsPerTurn() { + return 3; + } + + + public String execute(String arguments) { + final JSONObject args; + try { + args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments); + } catch (JSONException e) { + return ToolHandler.errorJson("Invalid arguments JSON"); + } + + final JSONArray rowsArray = args.optJSONArray("rows"); + final JSONArray ops = args.optJSONArray("operations"); + if (rowsArray == null) return ToolHandler.errorJson("Missing rows"); + if (ops == null) return ToolHandler.errorJson("Missing operations"); + + try { + List<JSONObject> rows = toObjectRows(rowsArray); + for (int i = 0; i < ops.length(); i++) { + JSONObject op = ops.optJSONObject(i); + if (op == null) return ToolHandler.errorJson("Operation at index " + i + " must be an object"); + rows = applyOperation(rows, op); + } + JSONObject result = new JSONObject(true); + result.put("row_count", rows.size()); + result.put("rows", new JSONArray(rows)); + return result.toString(); + } catch (IllegalArgumentException | JSONException e) { + return ToolHandler.errorJson(e.getMessage()); + } + } + + private static List<JSONObject> toObjectRows(JSONArray array) { + List<JSONObject> rows = new ArrayList<>(); + for (int i = 0; i < array.length(); i++) { + JSONObject row = array.optJSONObject(i); + if (row == null) throw new IllegalArgumentException("Row at index " + i + " is not an object"); + rows.add(row); + } + return rows; + } + + private static List<JSONObject> applyOperation(List<JSONObject> rows, JSONObject op) { + String type = op.optString("type", "").trim().toLowerCase(); + if (type.isEmpty()) throw new IllegalArgumentException("Operation missing type"); + if ("filter".equals(type)) return applyFilter(rows, op); + if ("sort".equals(type)) return applySort(rows, op); + if ("limit".equals(type)) return applyLimit(rows, op); + if ("project".equals(type)) return applyProject(rows, op); + if ("group_count".equals(type)) return applyGroupCount(rows, op); + throw new IllegalArgumentException("Unsupported operation type: " + type); + } + + private static List<JSONObject> applyFilter(List<JSONObject> rows, JSONObject op) { + String field = requiredField(op, "field"); + String cmp = op.optString("op", "eq").trim().toLowerCase(); + Object expected = op.opt("value"); + List<JSONObject> result = new ArrayList<>(); + for (JSONObject row : rows) { + Object actual = row.opt(field); + if (matches(actual, cmp, expected)) result.add(row); + } + return result; + } + + private static List<JSONObject> applySort(List<JSONObject> rows, JSONObject op) { + String field = requiredField(op, "field"); + String order = op.optString("order", "asc").trim().toLowerCase(); + final int direction = "desc".equals(order) ? -1 : 1; + + List<JSONObject> copy = new ArrayList<>(rows); + Collections.sort(copy, new Comparator<JSONObject>() { + @Override + public int compare(JSONObject a, JSONObject b) { + Object va = a.opt(field); + Object vb = b.opt(field); + return direction * compareValues(va, vb); + } + }); + return copy; + } + + private static List<JSONObject> applyLimit(List<JSONObject> rows, JSONObject op) { + int count = op.optInt("count", -1); + int offset = op.optInt("offset", 0); + if (count < 0) throw new IllegalArgumentException("limit.count must be >= 0"); + if (offset < 0) throw new IllegalArgumentException("limit.offset must be >= 0"); + if (offset >= rows.size()) return new ArrayList<>(); + int end = Math.min(rows.size(), offset + count); + return new ArrayList<>(rows.subList(offset, end)); + } + + private static List<JSONObject> applyProject(List<JSONObject> rows, JSONObject op) { + JSONArray fields = op.optJSONArray("fields"); + if (fields == null || fields.length() == 0) throw new IllegalArgumentException("project.fields must be a non-empty array"); + List<String> keep = new ArrayList<>(); + for (int i = 0; i < fields.length(); i++) { + String f = fields.optString(i, "").trim(); + if (!f.isEmpty()) keep.add(f); + } + if (keep.isEmpty()) throw new IllegalArgumentException("project.fields must contain at least one field"); + + List<JSONObject> out = new ArrayList<>(); + for (JSONObject row : rows) { + JSONObject projected = new JSONObject(true); + for (String key : keep) { + if (row.has(key)) { + try { + projected.put(key, row.get(key)); + } catch (JSONException e) { + throw new IllegalArgumentException("Failed to project field: " + key); + } + } + } + out.add(projected); + } + return out; + } + + private static List<JSONObject> applyGroupCount(List<JSONObject> rows, JSONObject op) { + String field = requiredField(op, "field"); + String order = op.optString("order", "desc").trim().toLowerCase(); + final int direction = "asc".equals(order) ? 1 : -1; + + Map<String, Integer> counts = new LinkedHashMap<>(); + for (JSONObject row : rows) { + Object v = row.opt(field); + String key = v == null || v == JSONObject.NULL ? "null" : String.valueOf(v); + Integer cur = counts.get(key); + counts.put(key, cur == null ? 1 : cur + 1); + } + + List<JSONObject> grouped = new ArrayList<>(); + for (Map.Entry<String, Integer> entry : counts.entrySet()) { + try { + JSONObject g = new JSONObject(true); + g.put("key", entry.getKey()); + g.put("count", entry.getValue()); + grouped.add(g); + } catch (JSONException e) { + throw new IllegalArgumentException("Failed to build group_count result"); + } + } + Collections.sort(grouped, new Comparator<JSONObject>() { + @Override + public int compare(JSONObject a, JSONObject b) { + return direction * Integer.compare(a.optInt("count", 0), b.optInt("count", 0)); + } + }); + return grouped; + } + + private static String requiredField(JSONObject obj, String field) { + String value = obj.optString(field, "").trim(); + if (value.isEmpty()) throw new IllegalArgumentException("Missing " + field); + return value; + } + + private static boolean matches(Object actual, String cmp, Object expected) { + if ("eq".equals(cmp)) return valuesEqual(actual, expected); + if ("ne".equals(cmp)) return !valuesEqual(actual, expected); + if ("gt".equals(cmp)) return compareValues(actual, expected) > 0; + if ("gte".equals(cmp)) return compareValues(actual, expected) >= 0; + if ("lt".equals(cmp)) return compareValues(actual, expected) < 0; + if ("lte".equals(cmp)) return compareValues(actual, expected) <= 0; + if ("contains".equals(cmp)) { + if (actual == null || actual == JSONObject.NULL || expected == null || expected == JSONObject.NULL) return false; + if (actual instanceof JSONArray) return arrayContains((JSONArray) actual, expected); + return String.valueOf(actual).toLowerCase().contains(String.valueOf(expected).toLowerCase()); + } + if ("in".equals(cmp)) { + if (!(expected instanceof JSONArray)) return false; + return arrayContains((JSONArray) expected, actual); + } + throw new IllegalArgumentException("Unsupported filter op: " + cmp); + } + + private static boolean valuesEqual(Object a, Object b) { + if (a == b) return true; + if (a == null || a == JSONObject.NULL) return b == null || b == JSONObject.NULL; + if (b == null || b == JSONObject.NULL) return false; + Double na = asDouble(a); + Double nb = asDouble(b); + if (na != null && nb != null) return Double.compare(na, nb) == 0; + return String.valueOf(a).equals(String.valueOf(b)); + } + + private static int compareValues(Object a, Object b) { + if (a == b) return 0; + if (a == null || a == JSONObject.NULL) return -1; + if (b == null || b == JSONObject.NULL) return 1; + Double na = asDouble(a); + Double nb = asDouble(b); + if (na != null && nb != null) return Double.compare(na, nb); + return String.valueOf(a).compareToIgnoreCase(String.valueOf(b)); + } + + private static Double asDouble(Object value) { + if (value == null || value == JSONObject.NULL) return null; + if (value instanceof Number) return Double.valueOf(((Number) value).doubleValue()); + try { + return Double.valueOf(String.valueOf(value)); + } catch (NumberFormatException e) { + return null; + } + } + + private static boolean arrayContains(JSONArray array, Object wanted) { + for (int i = 0; i < array.length(); i++) { + if (valuesEqual(array.opt(i), wanted)) return true; + } + return false; + } +} diff --git a/source/net/yacy/ai/tools/UnitConverterTool.java b/source/net/yacy/ai/tools/UnitConverterTool.java new file mode 100644 index 000000000..dfa41d100 --- /dev/null +++ b/source/net/yacy/ai/tools/UnitConverterTool.java @@ -0,0 +1,200 @@ +/** + * UnitConverterTool + * Copyright 2026 by Michael Peter Christen + * First released 06.02.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.ai.tools; + +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import net.yacy.ai.ToolHandler; + +public class UnitConverterTool implements ToolHandler { + + private static final String NAME = "unit_converter"; + + private static final Map<String, Double> LENGTH_TO_M = new HashMap<>(); + private static final Map<String, Double> MASS_TO_KG = new HashMap<>(); + private static final Map<String, Double> VOLUME_TO_L = new HashMap<>(); + private static final Map<String, Double> SPEED_TO_MPS = new HashMap<>(); + + static { + LENGTH_TO_M.put("mm", 0.001d); + LENGTH_TO_M.put("cm", 0.01d); + LENGTH_TO_M.put("m", 1d); + LENGTH_TO_M.put("km", 1000d); + LENGTH_TO_M.put("in", 0.0254d); + LENGTH_TO_M.put("ft", 0.3048d); + LENGTH_TO_M.put("yd", 0.9144d); + LENGTH_TO_M.put("mi", 1609.344d); + + MASS_TO_KG.put("mg", 0.000001d); + MASS_TO_KG.put("g", 0.001d); + MASS_TO_KG.put("kg", 1d); + MASS_TO_KG.put("t", 1000d); + MASS_TO_KG.put("oz", 0.028349523125d); + MASS_TO_KG.put("lb", 0.45359237d); + + VOLUME_TO_L.put("ml", 0.001d); + VOLUME_TO_L.put("l", 1d); + VOLUME_TO_L.put("m3", 1000d); + VOLUME_TO_L.put("tsp", 0.00492892159375d); + VOLUME_TO_L.put("tbsp", 0.01478676478125d); + VOLUME_TO_L.put("cup", 0.2365882365d); + VOLUME_TO_L.put("pt", 0.473176473d); + VOLUME_TO_L.put("qt", 0.946352946d); + VOLUME_TO_L.put("gal", 3.785411784d); + + SPEED_TO_MPS.put("m/s", 1d); + SPEED_TO_MPS.put("km/h", 0.2777777777777778d); + SPEED_TO_MPS.put("mph", 0.44704d); + SPEED_TO_MPS.put("kn", 0.514444d); + } + + @Override + public JSONObject definition() throws JSONException { + JSONObject tool = new JSONObject(true); + tool.put("type", "function"); + JSONObject fn = new JSONObject(true); + fn.put("name", NAME); + fn.put("description", "Convert numeric values between common units (length, mass, temperature, volume, speed)."); + JSONObject params = new JSONObject(true); + params.put("type", "object"); + JSONObject props = new JSONObject(true); + props.put("value", new JSONObject(true).put("type", "number").put("description", "Numeric input value.")); + props.put("from", new JSONObject(true).put("type", "string").put("description", "Source unit, e.g. km, lb, C.")); + props.put("to", new JSONObject(true).put("type", "string").put("description", "Target unit, e.g. mi, kg, F.")); + params.put("properties", props); + params.put("required", new JSONArray().put("value").put("from").put("to")); + fn.put("parameters", params); + tool.put("function", fn); + return tool; + } + + + public int maxCallsPerTurn() { + return 10; + } + + + public String execute(String arguments) { + final JSONObject args; + try { + args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments); + } catch (JSONException e) { + return ToolHandler.errorJson("Invalid arguments JSON"); + } + if (!args.has("value")) return ToolHandler.errorJson("Missing value"); + double value = args.optDouble("value", Double.NaN); + if (Double.isNaN(value) || Double.isInfinite(value)) return ToolHandler.errorJson("Invalid value"); + String from = normalize(args.optString("from", "")); + String to = normalize(args.optString("to", "")); + if (from.isEmpty() || to.isEmpty()) return ToolHandler.errorJson("Missing from/to"); + + try { + ConversionResult result = convert(value, from, to); + JSONObject out = new JSONObject(true); + out.put("tool", NAME); + out.put("category", result.category); + out.put("value", value); + out.put("from", from); + out.put("to", to); + out.put("result", result.value); + return out.toString(); + } catch (IllegalArgumentException | JSONException e) { + return ToolHandler.errorJson(e.getMessage()); + } + } + + private static ConversionResult convert(double value, String from, String to) { + if (isTemp(from) && isTemp(to)) { + double c = toCelsius(value, from); + return new ConversionResult("temperature", fromCelsius(c, to)); + } + if (LENGTH_TO_M.containsKey(from) && LENGTH_TO_M.containsKey(to)) { + double m = value * LENGTH_TO_M.get(from); + return new ConversionResult("length", m / LENGTH_TO_M.get(to)); + } + if (MASS_TO_KG.containsKey(from) && MASS_TO_KG.containsKey(to)) { + double kg = value * MASS_TO_KG.get(from); + return new ConversionResult("mass", kg / MASS_TO_KG.get(to)); + } + if (VOLUME_TO_L.containsKey(from) && VOLUME_TO_L.containsKey(to)) { + double l = value * VOLUME_TO_L.get(from); + return new ConversionResult("volume", l / VOLUME_TO_L.get(to)); + } + if (SPEED_TO_MPS.containsKey(from) && SPEED_TO_MPS.containsKey(to)) { + double mps = value * SPEED_TO_MPS.get(from); + return new ConversionResult("speed", mps / SPEED_TO_MPS.get(to)); + } + throw new IllegalArgumentException("Unsupported conversion pair: " + from + " -> " + to); + } + + private static boolean isTemp(String unit) { + return "c".equals(unit) || "f".equals(unit) || "k".equals(unit); + } + + private static double toCelsius(double v, String from) { + if ("c".equals(from)) return v; + if ("f".equals(from)) return (v - 32d) * 5d / 9d; + if ("k".equals(from)) return v - 273.15d; + throw new IllegalArgumentException("Unsupported temperature unit: " + from); + } + + private static double fromCelsius(double c, String to) { + if ("c".equals(to)) return c; + if ("f".equals(to)) return (c * 9d / 5d) + 32d; + if ("k".equals(to)) return c + 273.15d; + throw new IllegalArgumentException("Unsupported temperature unit: " + to); + } + + private static String normalize(String s) { + if (s == null) return ""; + String t = s.trim().toLowerCase(Locale.ROOT); + if ("°c".equals(t) || "celsius".equals(t)) return "c"; + if ("°f".equals(t) || "fahrenheit".equals(t)) return "f"; + if ("kelvin".equals(t)) return "k"; + if ("meter".equals(t) || "metre".equals(t)) return "m"; + if ("kilometer".equals(t) || "kilometre".equals(t)) return "km"; + if ("mile".equals(t)) return "mi"; + if ("foot".equals(t)) return "ft"; + if ("inch".equals(t)) return "in"; + if ("pound".equals(t)) return "lb"; + if ("ounce".equals(t)) return "oz"; + if ("liter".equals(t) || "litre".equals(t)) return "l"; + if ("gallon".equals(t)) return "gal"; + if ("kph".equals(t)) return "km/h"; + if ("fps".equals(t)) return "ft/s"; + return t; + } + + private static final class ConversionResult { + final String category; + final double value; + ConversionResult(String category, double value) { + this.category = category; + this.value = value; + } + } +} diff --git a/source/net/yacy/ai/tools/WebFetchTool.java b/source/net/yacy/ai/tools/WebFetchTool.java new file mode 100644 index 000000000..e09669807 --- /dev/null +++ b/source/net/yacy/ai/tools/WebFetchTool.java @@ -0,0 +1,168 @@ +/** + * WebFetchTool + * Copyright 2026 by Michael Peter Christen + * First released 06.02.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.ai.tools; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Date; +import java.util.HashSet; + +import org.apache.http.Header; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import net.yacy.ai.ToolHandler; +import net.yacy.cora.document.analysis.Classification; +import net.yacy.cora.document.analysis.Classification.ContentDomain; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; +import net.yacy.cora.protocol.ClientIdentification; +import net.yacy.cora.protocol.HeaderFramework; +import net.yacy.cora.protocol.http.HTTPClient; +import net.yacy.document.Document; +import net.yacy.document.Parser; +import net.yacy.document.TextParser; +import net.yacy.document.VocabularyScraper; +import net.yacy.document.parser.html.TagValency; + +public class WebFetchTool implements ToolHandler { + + private static final String NAME = "webfetch"; + private static final int TOOL_WEBFETCH_MAX_BYTES = 2_000_000; + private static final int TOOL_WEBFETCH_MAX_CHARS = 12_000; + + @Override + public JSONObject definition() throws JSONException { + JSONObject tool = new JSONObject(true); + tool.put("type", "function"); + JSONObject fn = new JSONObject(true); + fn.put("name", NAME); + fn.put("description", "Fetch a URL and return text content. Parse HTML/documents into markdown-style text when possible."); + JSONObject params = new JSONObject(true); + params.put("type", "object"); + JSONObject props = new JSONObject(true); + JSONObject url = new JSONObject(true); + url.put("type", "string"); + url.put("description", "Absolute http or https URL to fetch."); + props.put("url", url); + params.put("properties", props); + params.put("required", new JSONArray().put("url")); + fn.put("parameters", params); + tool.put("function", fn); + return tool; + } + + + public int maxCallsPerTurn() { + return 3; + } + + + public String execute(String arguments) { + String urlRaw; + try { + JSONObject obj = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments); + urlRaw = obj.optString("url", "").trim(); + } catch (JSONException e) { + return ToolHandler.errorJson("Invalid arguments JSON. don't try again"); + } + if (urlRaw == null || urlRaw.isEmpty()) return ToolHandler.errorJson("Missing url. don't try again"); + + final DigestURL url; + try { + url = new DigestURL(urlRaw); + } catch (Exception e) { + return ToolHandler.errorJson("Invalid URL. don't try again"); + } + final String protocol = url.getProtocol(); + if (!"http".equalsIgnoreCase(protocol) && !"https".equalsIgnoreCase(protocol)) { + return ToolHandler.errorJson("Only http/https URLs are allowed. don't try again"); + } + + try (HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 30000)) { + byte[] content = client.GETbytes(url, null, null, TOOL_WEBFETCH_MAX_BYTES, false); + int status = client.getStatusCode(); + if (status < 200 || status >= 300) return ToolHandler.errorJson("HTTP status " + status + ". don't try again"); + if (content == null || content.length == 0) return ToolHandler.errorJson("Empty response body. don't try again"); + + String contentType = "application/octet-stream"; + Header ct = client.getHttpResponse() == null ? null : client.getHttpResponse().getFirstHeader(HeaderFramework.CONTENT_TYPE); + if (ct != null && ct.getValue() != null && !ct.getValue().isEmpty()) contentType = ct.getValue(); + int semicolon = contentType.indexOf(';'); + if (semicolon > 0) contentType = contentType.substring(0, semicolon).trim().toLowerCase(); + + final String ext = MultiProtocolURL.getFileExtension(url.getFileName()); + final ContentDomain mimeDomain = Classification.getContentDomainFromMime(contentType); + if (mimeDomain == ContentDomain.IMAGE || mimeDomain == ContentDomain.AUDIO || mimeDomain == ContentDomain.VIDEO + || Classification.isMediaExtension(ext)) { + return ToolHandler.errorJson("Media content is not supported. don't try again"); + } + + String text; + if (contentType.startsWith("text/plain") || contentType.startsWith("text/markdown") || "txt".equals(ext) || "md".equals(ext)) { + text = truncate(new String(content, StandardCharsets.UTF_8), TOOL_WEBFETCH_MAX_CHARS); + } else { + String supportError = TextParser.supports(url, contentType); + if (supportError == null) { + Document[] docs = TextParser.parseSource(url, contentType, "UTF-8", TagValency.EVAL, + new HashSet<String>(), new VocabularyScraper(), 0, 0, content, new Date()); + Document merged = Document.mergeDocuments(url, contentType, docs); + text = truncate(documentToMarkdown(merged), TOOL_WEBFETCH_MAX_CHARS); + } else if (contentType.startsWith("text/")) { + text = truncate(new String(content, StandardCharsets.UTF_8), TOOL_WEBFETCH_MAX_CHARS); + } else { + return ToolHandler.errorJson("Unsupported content type " + contentType + ". don't try again"); + } + } + + JSONObject result = new JSONObject(true); + result.put("url", url.toNormalform(true)); + result.put("content_type", contentType); + result.put("content", text == null ? "" : text); + return result.toString(); + } catch (IOException e) { + return ToolHandler.errorJson("Fetch error: " + e.getMessage() + ". don't try again"); + } catch (Parser.Failure e) { + return ToolHandler.errorJson("Parse error: " + e.getMessage() + ". don't try again"); + } catch (JSONException e) { + return ToolHandler.errorJson("Failed to build tool response. don't try again"); + } + } + + private static String documentToMarkdown(Document doc) { + if (doc == null) return ""; + StringBuilder sb = new StringBuilder(); + String title = doc.dc_title(); + if (title != null && !title.isEmpty()) { + sb.append("# ").append(title).append("\n\n"); + } + String text = doc.getTextString(); + if (text != null && !text.isEmpty()) sb.append(text); + return sb.toString(); + } + + private static String truncate(String text, int maxChars) { + if (text == null) return ""; + if (maxChars <= 0 || text.length() <= maxChars) return text; + return text.substring(0, maxChars); + } +} diff --git a/source/net/yacy/ai/tools/WikipediaLinkCreatorTool.java b/source/net/yacy/ai/tools/WikipediaLinkCreatorTool.java new file mode 100644 index 000000000..977d9574d --- /dev/null +++ b/source/net/yacy/ai/tools/WikipediaLinkCreatorTool.java @@ -0,0 +1,169 @@ +/** + * WikipediaLinkCreatorTool + * Copyright 2026 by Michael Peter Christen + * First released 06.02.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.ai.tools; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.regex.Pattern; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import net.yacy.ai.ToolHandler; + +public class WikipediaLinkCreatorTool implements ToolHandler { + + private static final String NAME = "wikipedia_link_creator"; + private static final Pattern LANG_PATTERN = Pattern.compile("^[a-z]{2,10}(-[a-z0-9]{2,8})?$"); + + @Override + public JSONObject definition() throws JSONException { + JSONObject tool = new JSONObject(true); + tool.put("type", "function"); + JSONObject fn = new JSONObject(true); + fn.put("name", NAME); + fn.put("description", "In case you don't know the answer correctly or you are unsure. Build a Wikipedia article URL from topic keywords so it can be fetched with webfetch."); + + JSONObject params = new JSONObject(true); + params.put("type", "object"); + JSONObject props = new JSONObject(true); + + JSONObject topic = new JSONObject(true); + topic.put("type", "string"); + topic.put("description", "Target Wikipedia article title/topic (preferred input)."); + props.put("topic", topic); + + JSONObject keywords = new JSONObject(true); + keywords.put("type", "array"); + keywords.put("description", "Optional keyword list from which a topic is composed."); + keywords.put("items", new JSONObject(true).put("type", "string")); + props.put("keywords", keywords); + + JSONObject language = new JSONObject(true); + language.put("type", "string"); + language.put("description", "Wikipedia language code (e.g. en, de, fr). Defaults to en."); + props.put("language", language); + + params.put("properties", props); + params.put("required", new JSONArray()); + fn.put("parameters", params); + tool.put("function", fn); + return tool; + } + + + public int maxCallsPerTurn() { + return 3; + } + + + public String execute(String arguments) { + final JSONObject args; + try { + args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments); + } catch (JSONException e) { + return ToolHandler.errorJson("Invalid arguments JSON"); + } + + String topic = normalizeTopic(args.optString("topic", "")); + JSONArray keywords = args.optJSONArray("keywords"); + String language = normalizeLanguage(args.optString("language", "en")); + List<String> warnings = new ArrayList<>(); + + if (topic.isEmpty() && keywords != null) { + topic = normalizeTopic(joinKeywords(keywords)); + if (!topic.isEmpty()) { + warnings.add("Used keywords to compose article topic"); + } + } + if (topic.isEmpty()) { + return ToolHandler.errorJson("Missing topic/keywords"); + } + if (language == null) { + language = "en"; + warnings.add("Invalid language code; defaulted to en"); + } + + String encodedTitle = encodeTitle(topic); + String url = "https://" + language + ".wikipedia.org/wiki/" + encodedTitle; + + try { + JSONObject result = new JSONObject(true); + result.put("tool", NAME); + result.put("topic_input", topic); + result.put("article_title", topic); + result.put("language", language); + result.put("url", url); + result.put("next_tool_hint", "Use webfetch with this URL to retrieve the article content."); + result.put("warnings", new JSONArray(warnings)); + return result.toString(); + } catch (JSONException e) { + return ToolHandler.errorJson("Failed to build wikipedia link response"); + } + } + + private static String joinKeywords(JSONArray keywords) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < keywords.length(); i++) { + String k = keywords.optString(i, "").trim(); + if (k.isEmpty()) continue; + if (sb.length() > 0) sb.append(' '); + sb.append(k); + } + return sb.toString(); + } + + private static String normalizeTopic(String topic) { + if (topic == null) return ""; + String t = topic.trim(); + while (t.contains(" ")) t = t.replace(" ", " "); + if (t.startsWith("https://") || t.startsWith("http://")) { + int wikiIdx = t.indexOf("/wiki/"); + if (wikiIdx >= 0) { + t = t.substring(wikiIdx + 6); + } + } + t = t.replace('_', ' ').trim(); + if (t.indexOf('#') >= 0) t = t.substring(0, t.indexOf('#')).trim(); + return t; + } + + private static String normalizeLanguage(String language) { + if (language == null) return "en"; + String l = language.trim().toLowerCase(Locale.ROOT); + if (l.isEmpty()) return "en"; + if (!LANG_PATTERN.matcher(l).matches()) return null; + return l; + } + + private static String encodeTitle(String title) { + try { + return URLEncoder.encode(title, StandardCharsets.UTF_8.name()).replace("+", "_"); + } catch (UnsupportedEncodingException e) { + return title.replace(' ', '_'); + } + } +} diff --git a/source/net/yacy/http/servlets/MCPSearchServlet.java b/source/net/yacy/http/servlets/MCPSearchServlet.java index ffca72aac..983ad24f7 100644 --- a/source/net/yacy/http/servlets/MCPSearchServlet.java +++ b/source/net/yacy/http/servlets/MCPSearchServlet.java @@ -20,6 +20,7 @@ package net.yacy.http.servlets; +import net.yacy.ai.RAGAugmentor; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.util.ConcurrentLog; @@ -58,6 +59,17 @@ public class MCPSearchServlet extends HttpServlet { private static final int DEFAULT_RESULT_COUNT = 10; private static final int MAX_RESULT_COUNT = 100; + /** + * Handles JSON-RPC requests for the MCP surface. + * <p> + * Accepts single request objects and batch arrays, supports notifications, and + * always returns UTF-8 JSON responses. + * + * @param request servlet request + * @param response servlet response + * @throws ServletException servlet errors + * @throws IOException I/O errors + */ @Override public void service(final ServletRequest request, final ServletResponse response) throws ServletException, IOException { request.setCharacterEncoding("UTF-8"); @@ -72,6 +84,7 @@ public class MCPSearchServlet extends HttpServlet { final Method reqMethod = Method.getMethod(hrequest.getMethod()); if (reqMethod == Method.OTHER) { + // Browsers perform CORS preflight with OPTIONS; answer quickly. hresponse.setStatus(HttpServletResponse.SC_OK); return; } @@ -85,10 +98,12 @@ public class MCPSearchServlet extends HttpServlet { parsed = tokener.nextValue(); } catch (JSONException e) { writeJsonResponse(hresponse, errorResponse(JSONObject.NULL, -32700, e.getMessage())); + return; } try { if (parsed instanceof JSONObject) { + // Support shorthand endpoint style: infer method from URI path. if (((JSONObject) parsed).optString("method", "").length() == 0) { String uri = hrequest.getRequestURI(); ((JSONObject) parsed).put("method", uri.substring(1)); @@ -100,6 +115,7 @@ public class MCPSearchServlet extends HttpServlet { hresponse.setStatus(HttpServletResponse.SC_NO_CONTENT); } } else if (parsed instanceof JSONArray) { + // Batch request mode. final JSONArray requestArray = (JSONArray) parsed; final JSONArray responseArray = new JSONArray(); for (int i = 0; i < requestArray.length(); i++) { @@ -124,6 +140,12 @@ public class MCPSearchServlet extends HttpServlet { } } + /** + * Routes one JSON-RPC request object to the corresponding MCP handler. + * + * @param requestObject parsed request + * @return response object or {@code null} for notifications + */ private JSONObject handleRequest(final JSONObject requestObject) { final Object id = requestObject.opt("id"); final String jsonrpc = requestObject.optString("jsonrpc", JSONRPC_VERSION); @@ -147,6 +169,7 @@ public class MCPSearchServlet extends HttpServlet { } if (id == JSONObject.NULL || id == null) { + // Notifications must not produce a response body. // Notification: acknowledge silently if ("ping".equals(method)) { return null; @@ -167,6 +190,13 @@ public class MCPSearchServlet extends HttpServlet { } } + /** + * Handles MCP initialize call and advertises protocol/capabilities. + * + * @param id JSON-RPC id + * @param params initialize params + * @return JSON-RPC success or error response + */ private JSONObject handleInitialize(final Object id, final JSONObject params) { try {log.info("MCPSearchServlet: initialize " + (params == null ? "" : params.toString(0)));} catch (JSONException e) {} final JSONObject result = new JSONObject(true); @@ -190,6 +220,13 @@ public class MCPSearchServlet extends HttpServlet { return successResponse(id, result); } + /** + * Handles MCP tool listing and returns schema for the single {@code search} + * tool. + * + * @param id JSON-RPC id + * @return JSON-RPC success or error response + */ private JSONObject handleToolsList(final Object id) { log.info("MCPSearchServlet: list " + (id == null ? "" : id.toString())); try { @@ -245,6 +282,13 @@ public class MCPSearchServlet extends HttpServlet { } } + /** + * Handles tool invocation for {@code search}. + * + * @param id JSON-RPC id + * @param params tool call params + * @return JSON-RPC success or error response + */ private JSONObject handleToolsCall(final Object id, final JSONObject params) { try {log.info("MCPSearchServlet: call " + (id == null ? "" : id.toString()) + " params: " + (params == null ? "" : params.toString(0)));} catch (JSONException e) {} if (params == null) { @@ -265,6 +309,7 @@ public class MCPSearchServlet extends HttpServlet { return errorResponse(id, -32602, "Tool argument 'query' must be a non-empty string"); } + // Sanitize and clamp result limit. int limit = arguments.optInt("limit", DEFAULT_RESULT_COUNT); if (limit <= 0) { limit = DEFAULT_RESULT_COUNT; @@ -272,8 +317,9 @@ public class MCPSearchServlet extends HttpServlet { limit = Math.min(limit, MAX_RESULT_COUNT); final boolean includeSnippet = arguments.optBoolean("include_snippet", true); + // Delegate the actual search to the RAG helper utility. JSONArray results; - results = RAGProxyServlet.searchResults(query, limit, includeSnippet); + results = RAGAugmentor.searchResults(query, limit, includeSnippet); try { final JSONObject payload = new JSONObject(true); @@ -294,6 +340,13 @@ public class MCPSearchServlet extends HttpServlet { } } + /** + * Reads full HTTP request body into a string. + * + * @param request servlet request + * @return request body (possibly empty) + * @throws IOException when reader access fails + */ private static String readBody(final ServletRequest request) throws IOException { final StringBuilder builder = new StringBuilder(); try (BufferedReader reader = request.getReader()) { @@ -305,6 +358,13 @@ public class MCPSearchServlet extends HttpServlet { return builder.toString(); } + /** + * Builds a JSON-RPC success object. + * + * @param id JSON-RPC id + * @param result result payload + * @return JSON-RPC success response + */ private static JSONObject successResponse(final Object id, final JSONObject result) { final JSONObject response = new JSONObject(true); try { @@ -318,6 +378,14 @@ public class MCPSearchServlet extends HttpServlet { return response; } + /** + * Builds a JSON-RPC error object. + * + * @param id JSON-RPC id + * @param code JSON-RPC error code + * @param message human-readable error message + * @return JSON-RPC error response + */ private static JSONObject errorResponse(final Object id, final int code, final String message) { final JSONObject response = new JSONObject(true); try { @@ -333,6 +401,13 @@ public class MCPSearchServlet extends HttpServlet { return response; } + /** + * Serializes and writes either a JSON object or JSON array response. + * + * @param response servlet response + * @param payload JSONObject or JSONArray + * @throws IOException when writing fails + */ private static void writeJsonResponse(final HttpServletResponse response, final Object payload) throws IOException { final String serialized = payload instanceof JSONObject ? ((JSONObject) payload).toString() : payload instanceof JSONArray ? ((JSONArray) payload).toString() : payload.toString(); diff --git a/source/net/yacy/http/servlets/RAGProxyServlet.java b/source/net/yacy/http/servlets/RAGProxyServlet.java index 840e5a564..c552b7387 100644 --- a/source/net/yacy/http/servlets/RAGProxyServlet.java +++ b/source/net/yacy/http/servlets/RAGProxyServlet.java @@ -22,31 +22,14 @@ package net.yacy.http.servlets; import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.net.HttpURLConnection; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.AbstractMap; import java.util.ArrayList; -import java.util.Arrays; import java.util.Base64; -import java.util.Collection; -import java.util.Comparator; import java.util.Deque; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.stream.Collectors; import javax.servlet.ServletException; import javax.servlet.ServletOutputStream; @@ -56,38 +39,17 @@ import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.SolrException; import org.apache.solr.servlet.cache.Method; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -import org.json.JSONTokener; import net.yacy.ai.LLM; -import net.yacy.cora.document.analysis.Classification; -import net.yacy.cora.document.id.DigestURL; -import net.yacy.cora.document.id.MultiProtocolURL; -import net.yacy.cora.federate.solr.SolrType; -import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector; -import net.yacy.cora.federate.yacy.CacheStrategy; -import net.yacy.cora.lod.vocabulary.Tagging; -import net.yacy.cora.protocol.ClientIdentification; +import net.yacy.ai.RAGAugmentor; +import net.yacy.ai.ToolCallProtocol; import net.yacy.cora.protocol.Domains; import net.yacy.cora.util.ConcurrentLog; -import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.search.Switchboard; -import net.yacy.search.SwitchboardConstants; -import net.yacy.search.query.QueryGoal; -import net.yacy.search.query.QueryModifier; -import net.yacy.search.query.QueryParams; -import net.yacy.search.query.SearchEvent; -import net.yacy.search.query.SearchEventCache; -import net.yacy.search.ranking.RankingProfile; -import net.yacy.search.schema.CollectionSchema; -import net.yacy.search.snippet.TextSnippet; /** * This class implements a Retrieval Augmented Generation ("RAG") proxy which @@ -109,7 +71,6 @@ public class RAGProxyServlet extends HttpServlet { private static final long serialVersionUID = 3411544789759643137L; public static final String LLM_SYSTEM_PROMPT_DEFAULT = "You are a smart and helpful chatbot. If possible, use friendly emojies."; - private static final String LLM_SYSTEM_PREFIX_DEFAULT = "\n\nYou may receive additional expert knowledge in the user prompt after a 'Additional Information' headline to enhance your knowledge. Use it only if applicable."; private static final String LLM_USER_PREFIX_DEFAULT = "\n\nAdditional Information:\n\nbelow you find a collection of texts that might be useful to generate a response. Do not discuss these documents, just use them to answer the question above.\n\n"; private static final String LLM_QUERY_GENERATOR_PREFIX_DEFAULT = "Make a list of search words with low document frequency for the following prompt; use a JSON Array: "; @@ -193,7 +154,6 @@ public class RAGProxyServlet extends HttpServlet { // get messages and prepare user message attachments JSONArray messages = bodyObject.optJSONArray("messages"); - final String systemPrefix = sb.getConfig("ai.llm-system-prefix", LLM_SYSTEM_PREFIX_DEFAULT); final String userPrefix = sb.getConfig("ai.llm-user-prefix", LLM_USER_PREFIX_DEFAULT); // debug @@ -206,26 +166,40 @@ public class RAGProxyServlet extends HttpServlet { userObject.attachAttachment(userPrefix); } } - UserObject userObject = new UserObject(messages.getJSONObject(messages.length() - 1)); - String user = userObject.getContentText(); // this is the latest prompt - final String userPrompt = user; - String ragMode = userObject.getSearchMode(); + UserObject userObject = null; + String user = ""; + String ragMode = "no"; + String userPrompt = ""; + int lastUserIndex = -1; + for (int i = messages.length() - 1; i >= 0; i--) { + JSONObject message = messages.getJSONObject(i); + if ("user".equals(message.optString("role", ""))) { + lastUserIndex = i; + break; + } + } + if (lastUserIndex >= 0) { + userObject = new UserObject(messages.getJSONObject(lastUserIndex)); + user = userObject.getContentText(); // this is the latest user prompt + userPrompt = user; + ragMode = userObject.getSearchMode(); + } ConcurrentLog.info("RAGProxy", "ragMode=" + ragMode + " userChars=" + (user == null ? 0 : user.length())); //List<DataURL> data_urls = userObject.getContentAttachments(); // this list is a copy of the content data_urls // RAG String searchResultQuery = ""; String searchResultMarkdown = ""; - if (!"no".equals(ragMode)) { + if (userObject != null && !"no".equals(ragMode)) { // modify system and user prompt here in bodyObject to enable RAG final String queryPrefix = sb.getConfig("ai.llm-query-generator-prefix", LLM_QUERY_GENERATOR_PREFIX_DEFAULT); final long queryStart = System.currentTimeMillis(); - searchResultQuery = this.searchWordsForPrompt(llm4tldr.llm, llm4tldr.model, queryPrefix + user); // might return null in case any error occurred + searchResultQuery = RAGAugmentor.searchWordsForPrompt(llm4tldr.llm, llm4tldr.model, queryPrefix + user); // might return null in case any error occurred if (searchResultQuery == null || searchResultQuery.length() == 0) searchResultQuery = user; // in case there is an error we simply search with the prompt final long queryElapsed = System.currentTimeMillis() - queryStart; - final Set<String> boostTerms = intersectTokens(userPrompt, searchResultQuery, 8); + final Set<String> boostTerms = RAGAugmentor.intersectTokens(userPrompt, searchResultQuery, 8); final long searchStart = System.currentTimeMillis(); - searchResultMarkdown = searchResultsAsMarkdown(searchResultQuery, 10, "global".equals(ragMode), boostTerms); + searchResultMarkdown = RAGAugmentor.searchResultsAsMarkdown(searchResultQuery, 10, "global".equals(ragMode), boostTerms); final long searchElapsed = System.currentTimeMillis() - searchStart; ConcurrentLog.info( "RAGProxy", @@ -236,76 +210,22 @@ public class RAGProxyServlet extends HttpServlet { userObject.setContentText(user); } - // write back modified bodyMap to body - body = bodyObject.toString(); - - // Open request to back-end service - final URL url = new URI(llm4Chat.llm.hoststub + "/v1/chat/completions").toURL(); - final HttpURLConnection conn = (HttpURLConnection) url.openConnection(); - conn.setRequestMethod("POST"); - conn.setRequestProperty("Content-Type", "application/json"); - if (!llm4Chat.llm.api_key.isEmpty()) { - conn.setRequestProperty("Authorization", "Bearer " + llm4Chat.llm.api_key); + JSONObject initialMetadata = null; + if (searchResultMarkdown.length() > 0) { + initialMetadata = new JSONObject(true); + initialMetadata.put("search-filename", "search_result_" + searchResultQuery.replace(' ', '_') + ".md"); + initialMetadata.put("search-text-base64", new String(Base64.getEncoder().encode(searchResultMarkdown.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8)); } - conn.setDoOutput(true); - - // write the body to back-end LLM - try (OutputStream os = conn.getOutputStream()) { - os.write(body.getBytes()); - os.flush(); - } // here we wait for the response from upstream - // write back response of the back-end service to the client; use status of - // backend-response - final int status = conn.getResponseCode(); - // String rmessage = conn.getResponseMessage(); + // ToolCallProtocol owns request preparation, initial stream handling and follow-up tool rounds. + final int status = ToolCallProtocol.proxyToolLifecycle(out, llm4Chat, bodyObject, messages, initialMetadata); hresponse.setStatus(status); - - if (status == 200) { - final BlockingQueue<String> inputQueue = new LinkedBlockingQueue<>(); - final String POISON = "POISON"; - Thread readerThread = new Thread(() -> { - // read the response of the back-end line-by-line and push it to a stack concurrently - try { - final BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream())); - String inputLine; - while ((inputLine = in.readLine()) != null) {inputQueue.put(inputLine);} - in.close(); - inputQueue.put(POISON); - } catch (IOException | InterruptedException e) { - } finally { - try {inputQueue.put(POISON);} catch (InterruptedException e) {} - } - }); - readerThread.start(); - - // read the stack line-by-line and write it to the client line-by-line - try { - String inputLine; - int count = 0; - while (!(inputLine = inputQueue.take()).equals(POISON)) { - if (count == 0 && searchResultMarkdown.length() > 0) { - // for the first line we modify the data line to integrate the search result as file - int p = inputLine.indexOf('{'); - if (p > 0) { - JSONObject j = new JSONObject(new JSONTokener(inputLine.substring(p))); - j.put("search-filename", "search_result_"+ searchResultQuery.replace(' ', '_') + ".md"); - j.put("search-text-base64", new String(Base64.getEncoder().encode(searchResultMarkdown.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8)); - inputLine = inputLine.substring(0, p) + j.toString(); - } - } - out.println(inputLine); // i.e. data: {"id":"chatcmpl-69","object":"chat.completion.chunk","created":1715908287,"model":"llama3:8b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"ߘ"},"finish_reason":null}]} - out.flush(); - count++; - } - } catch (InterruptedException e) {} - } out.close(); // close this here to end transmission - } catch (JSONException | URISyntaxException e) { + } catch (JSONException e) { throw new IOException(e.getMessage()); } } - + public final static class DataURL { private String mimetype; private byte[] data; @@ -482,495 +402,6 @@ public class RAGProxyServlet extends HttpServlet { } } } - - public static JSONArray searchResults(String query, int count, final boolean includeSnippet) { - return searchResults(query, count, includeSnippet, new LinkedHashSet<>()); - } - - public static JSONArray searchResults(String query, int count, final boolean includeSnippet, final Set<String> boostTerms) { - final JSONArray results = new JSONArray(); - if (query == null || query.length() == 0 || count == 0) return results; - Switchboard sb = Switchboard.getSwitchboard(); - EmbeddedSolrConnector connector = sb.index.fulltext().getDefaultEmbeddedConnector(); - // construct query - final SolrQuery params = new SolrQuery(); - params.setQuery(query); - params.set("defType", "edismax"); - params.set("qf", - CollectionSchema.title.getSolrFieldName() + "^3 " + - CollectionSchema.text_t.getSolrFieldName() + "^1 " + - CollectionSchema.sku.getSolrFieldName() + "^0.5 " + - CollectionSchema.h1_txt.getSolrFieldName() + "^2"); - params.set("pf", - CollectionSchema.title.getSolrFieldName() + "^5 " + - CollectionSchema.text_t.getSolrFieldName() + "^2"); - //params.set("mm", "2<75%"); // using mm is too strict; in many cases we don't get any hits - final List<String> bqParts = new ArrayList<>(); - if (boostTerms != null && !boostTerms.isEmpty()) { - for (String term : boostTerms) { - if (term == null || term.isEmpty()) continue; - bqParts.add(CollectionSchema.title.getSolrFieldName() + ":" + term + "^0.5"); - bqParts.add(CollectionSchema.h1_txt.getSolrFieldName() + ":" + term + "^0.4"); - bqParts.add(CollectionSchema.text_t.getSolrFieldName() + ":" + term + "^0.2"); - } - } - bqParts.add("(" + CollectionSchema.url_file_ext_s.getSolrFieldName() + ":(zip rar 7z tar gz bz2 xz tgz))^0.1"); - params.set("bq", String.join(" ", bqParts)); - params.setRows(count); - params.setStart(0); - params.setFacet(false); - params.clearSorts(); - params.setFields( - CollectionSchema.sku.getSolrFieldName(), CollectionSchema.title.getSolrFieldName(), CollectionSchema.text_t.getSolrFieldName(), - CollectionSchema.description_txt.getSolrFieldName(), CollectionSchema.keywords.getSolrFieldName(), CollectionSchema.synonyms_sxt.getSolrFieldName(), - CollectionSchema.h1_txt.getSolrFieldName(), CollectionSchema.h2_txt.getSolrFieldName(), CollectionSchema.h3_txt.getSolrFieldName(), - CollectionSchema.h4_txt.getSolrFieldName(), CollectionSchema.h5_txt.getSolrFieldName(), CollectionSchema.h6_txt.getSolrFieldName() - ); - params.setIncludeScore(true); - params.set("df", CollectionSchema.text_t.getSolrFieldName()); - - // query the server - try { - final SolrDocumentList sdl = connector.getDocumentListByParams(params); - Iterator<SolrDocument> i = sdl.iterator(); - while (i.hasNext()) { - try { - SolrDocument doc = i.next(); - final JSONObject result = new JSONObject(true); - String url = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName()); - result.put("url", url == null ? "" : url.trim()); - String title = getOneString(doc, CollectionSchema.title); - result.put("title", title == null ? "" : title.trim()); - if (includeSnippet) { - String text = (String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName()); - result.put("text", limitSnippet(text == null ? "" : text.trim(), 2000)); - } - results.put(result); - } catch (JSONException e) { - // skip this result - } - } - return results; - } catch (SolrException | IOException e) { - return results; - } - } - - public static String searchResultsAsMarkdown(String query, int count, boolean global) { - return searchResultsAsMarkdown(query, count, global, new LinkedHashSet<>()); - } - - public static String searchResultsAsMarkdown(String query, int count, boolean global, final Set<String> boostTerms) { - final long searchStart = System.currentTimeMillis(); - JSONArray searchResults = global ? searchResultsGlobal(query, count, true) : searchResults(query, count, true, boostTerms); - ConcurrentLog.info("RAGProxy", "searchResults=" + searchResults.length() + " global=" + global + " searchMs=" + (System.currentTimeMillis() - searchStart)); - StringBuilder sb = new StringBuilder(); - - // collect snippets - List<Snippet> results = new ArrayList<>(); - for (int i = 0; i < searchResults.length(); i++) { - try { - JSONObject r = searchResults.getJSONObject(i); - String title = r.optString("title", ""); - String url = r.optString("url", ""); - String text = r.optString("text", ""); - if (title.isEmpty()) title = url; - if (text.isEmpty()) text = title; - if (title.length() > 0 && text.length() > 0) { - Snippet snippet = new Snippet(query, text, url, title, 256); // we always compute a snippet because that gives us a hint if the query appears at all - if (snippet.getText().length() > 0) results.add(snippet); - } - } catch (JSONException e) {} - } - - // sort snippets again by score - results.sort(Comparator.comparingDouble(Snippet::getScore)); - - int limit = results.size() / 2; - if (results.size() > 0 && limit == 0) limit = 1; - for (int i = 0; i < limit; i++) { - Snippet snippet = results.get(i); - sb.append("## ").append(snippet.getTitle()).append("\n"); - sb.append(snippet.text).append("\n"); - if (snippet.getURL().length() > 0) sb.append("Source: ").append(snippet.getURL()).append("\n"); - sb.append("\n\n"); - } - - ConcurrentLog.info("RAGProxy", "markdownChars=" + sb.length() + " snippetCount=" + results.size()); - return sb.toString(); - } - - public static JSONArray searchResultsGlobal(String query, int count, final boolean includeSnippet) { - final JSONArray results = new JSONArray(); - if (query == null || query.length() == 0 || count == 0) return results; - final Switchboard sb = Switchboard.getSwitchboard(); - final RankingProfile ranking = sb.getRanking(); - final int timezoneOffset = 0; - final QueryModifier modifier = new QueryModifier(timezoneOffset); - String querystring = modifier.parse(query); - if (querystring.length() == 0) { - querystring = query == null ? "" : query.trim(); - } - if (querystring.length() == 0) return results; - final QueryGoal qg = new QueryGoal(querystring); - final QueryParams theQuery = new QueryParams( - qg, - modifier, - 0, - "", - Classification.ContentDomain.TEXT, - "", - timezoneOffset, - new HashSet<Tagging.Metatag>(), - CacheStrategy.IFFRESH, - count, - 0, - ".*", - null, - null, - QueryParams.Searchdom.GLOBAL, - null, - true, - DigestURL.hosthashess(sb.getConfig("search.excludehosth", "")), - MultiProtocolURL.TLD_any_zone_filter, - null, - false, - sb.index, - ranking, - ClientIdentification.yacyIntranetCrawlerAgent.userAgent(), - 0.0d, - 0.0d, - 0.0d, - sb.getConfigSet("search.navigation")); - final SearchEvent theSearch = SearchEventCache.getEvent( - theQuery, - sb.peers, - sb.tables, - (sb.isRobinsonMode()) ? sb.clusterhashes : null, - false, - sb.loader, - (int) sb.getConfigLong( - SwitchboardConstants.REMOTESEARCH_MAXCOUNT_USER, - sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXCOUNT_DEFAULT, 10)), - sb.getConfigLong( - SwitchboardConstants.REMOTESEARCH_MAXTIME_USER, - sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000))); - final long timeout = sb.getConfigLong( - SwitchboardConstants.REMOTESEARCH_MAXTIME_USER, - sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000)); - waitForFeedingAndResort(theSearch, timeout); - for (int i = 0; i < count; i++) { - URIMetadataNode node = theSearch.oneResult(i, timeout); - if (node == null) break; - try { - final JSONObject result = new JSONObject(true); - result.put("url", node.urlstring()); - result.put("title", node.title()); - if (includeSnippet) { - String text = node.snippet(); - if (text == null || text.isEmpty()) { - TextSnippet snippet = node.textSnippet(); - if (snippet != null && snippet.exists() && !snippet.getErrorCode().fail()) { - text = snippet.getLineRaw(); - } - } - if (text == null || text.isEmpty()) { - text = firstFieldString(node.getFieldValue(CollectionSchema.description_txt.getSolrFieldName())); - } - if (text == null || text.isEmpty()) { - text = firstFieldString(node.getFieldValue(CollectionSchema.text_t.getSolrFieldName())); - } - result.put("text", limitSnippet(text == null ? "" : text.trim(), 2000)); - } - results.put(result); - } catch (JSONException e) { - // skip this result - } - } - return results; - } - - private static String limitSnippet(String text, int maxChars) { - if (text == null) return ""; - if (maxChars <= 0 || text.length() <= maxChars) return text; - return text.substring(0, maxChars); - } - - private static String firstFieldString(Object value) { - if (value == null) return ""; - if (value instanceof Collection) { - for (Object item : (Collection<?>) value) { - if (item != null) return item.toString(); - } - return ""; - } - return value.toString(); - } - - private static void waitForFeedingAndResort(SearchEvent search, long timeoutMs) { - if (search == null || timeoutMs <= 0) return; - final long end = System.currentTimeMillis() + timeoutMs; - while (!search.isFeedingFinished() && System.currentTimeMillis() < end) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; - } - } - search.resortCachedResults(); - } - - - public static class Snippet { - - private String text, url, title; - private double score; - - /** - * Find a snippet inside a given text that contains most of the searched words plus some context. - * @param query a string with a search query; query words are separated by space - * @param text the text where we want to find the snippets - * @param maxChunkLength the maximum length of a single chunk; however the snippet is three times as this. - * @return one string containing the snippet. - */ - public Snippet(String query, String text, String url, String title, int maxChunkLength) { - this.url = url; - this.title = title; - this.score = 0.0; - - if (text == null || text.isEmpty() || maxChunkLength <= 0 || query == null) { - this.text = ""; - return; - } - - // Step 1: Slice text and make copy with lowercase version to support tf*idf computation - List<String> chunks = slicer(text, maxChunkLength); - if (chunks.isEmpty()) { - this.text = ""; - return; - } - List<String> chunksLowerCase = new ArrayList<>(chunks.size()); - for (String chunk: chunks) chunksLowerCase.add(chunk.toLowerCase()); - - // Step 2: Preprocess query - Set<String> queryWordSet = querySet(query); - if (queryWordSet.isEmpty()) { - this.text = ""; - return; - } - - // Step 3: Compute IDF - // IDF uses a logarithm because the information gain of rare words grows non-linearly; - // the log dampens extreme ratios (N/df), stabilizes TF-IDF values, and matches the - // information-theoretic definition of word informativeness. - int totalChunks = chunksLowerCase.size(); - Map<String, Double> idf = new HashMap<>(); - for (String word: queryWordSet) { - int docFreq = 0; - for (String chunk: chunksLowerCase) { - if (chunk.contains(word)) docFreq++; - } - idf.put(word, Math.log((double) totalChunks / (docFreq + 1)) + 1); - } - - // Step 4: Score chunks - Map<Integer, Double> chunkScores = new HashMap<>(); - for (int i = 0; i < chunksLowerCase.size(); i++) { - String chunk = chunksLowerCase.get(i); - double score = 0.0; - Map<String, Integer> tf = new HashMap<>(); // counts occurrence in query for each word in chunk - - // Extract words and clean - String[] wordsInChunk = chunk.split("\\s+"); - for (String w : wordsInChunk) { - String cleanWord = w.replaceAll("[.,!?;:]", ""); - if (cleanWord.length() > 0 && queryWordSet.contains(cleanWord)) { - tf.put(cleanWord, tf.getOrDefault(cleanWord, 0) + 1); - } - } - - // Sum TF-IDF - for (String word: queryWordSet) { - int tfValue = tf.getOrDefault(word, 0); - double tfIdf = (double) tfValue * idf.getOrDefault(word, 1.0); - score += tfIdf; - } - chunkScores.put(i, score); - } - - // Step 5: Find best chunk - int topChunkIndex = -1; - for (Map.Entry<Integer, Double> entry: chunkScores.entrySet()) { - if (entry.getValue() > this.score) { - this.score = entry.getValue(); - topChunkIndex = entry.getKey(); - } - } - - // if there is no best chunk, return an empty snippet - if (topChunkIndex < 0) { - this.text = ""; - this.score = 0.0; - return; - } - - // Step 6: Get 3-slice snippet - List<String> snippetChunks = new ArrayList<>(); - if (topChunkIndex > 0) { - snippetChunks.add(chunks.get(topChunkIndex - 1)); - } - snippetChunks.add(chunks.get(topChunkIndex)); - if (topChunkIndex < chunks.size() - 1) { - snippetChunks.add(chunks.get(topChunkIndex + 1)); - } - - // Step 7: Join - this.text = String.join(" ", snippetChunks); - } - - public double getScore() { - return this.score; - } - - public String getText() { - return this.text; - } - - public String getURL() { - return this.url; - } - - public String getTitle() { - return this.title; - } - } - - /** - * Creates slices of a given text. We want slices of average same size, - * but we want to prevent that cuts are made within sentences. - * @param text the given text - * @param len the minimum length of the wanted slices; actual slices may be longer - * @return a list of text slices - */ - public static List<String> slicer(String text, int len) { - List<String> result = new ArrayList<>(); - if (text == null || len <= 0) return result; - - int start = 0; - while (start < text.length()) { - int end = Math.min(start + len, text.length()); - - // Move end position further out until a a sentence end is found: - // look for sentence boundary: .!?, followed by whitespace char. - while (end < text.length()) { - char ch = text.charAt(end - 1); - if ((ch == '.' || ch == '?' || ch == '!') && Character.isWhitespace(text.charAt(end))) { - break; - } - end++; - } - result.add(text.substring(start, end)); - start = end; - } - - return result; - } - - private static String getOneString(SolrDocument doc, CollectionSchema field) { - assert field.isMultiValued(); - assert field.getType() == SolrType.string || field.getType() == SolrType.text_general; - Object r = doc.getFieldValue(field.getSolrFieldName()); - if (r == null) return ""; - if (r instanceof ArrayList) { - return (String) ((ArrayList<?>) r).get(0); - } - return r.toString(); - } - - private String searchWordsForPrompt(LLM llm, String model, String prompt) { - final String question = prompt == null ? "" : prompt; - if (llm == null || model == null || model.isEmpty()) { - return null; - } - try { - LLM.Context context = new LLM.Context(LLM_SYSTEM_PREFIX_DEFAULT); - context.addPrompt(question); - Set<String> singlewords = new LinkedHashSet<>(); - String[] a = LLM.stringsFromChat(llm.chat(model, context, LLM.listSchema, 200)); - if (a == null || a.length == 0) return null; - // unfortunately this might not be a single word per line but several words; we collect them all. - for (String s: a) { - if (s == null) continue; - for (String t: s.split(" ")) { - if (!t.isEmpty()) singlewords.add(t.toLowerCase()); - } - } - if (singlewords.isEmpty()) { - return null; - } - StringBuilder query = new StringBuilder(); - for (String s: singlewords) query.append(s).append(' '); - String querys = query.toString().trim(); - if (querys.length() == 0) return null; - return querys; - } catch (IOException | JSONException e) { - e.printStackTrace(); - return null; - } - } - - private static Set<String> querySet(String query) { - Set<String> queryWordSet = Arrays.stream(query.trim().toLowerCase().split("\\s+")) - .map(String::toLowerCase) - .filter(word -> !word.isEmpty()) - .collect(Collectors.toSet()); - return queryWordSet; - } - - private static Set<String> intersectTokens(String originalPrompt, String computedQuery, int maxTerms) { - Set<String> promptTerms = querySet(originalPrompt == null ? "" : originalPrompt); - Set<String> queryTerms = querySet(computedQuery == null ? "" : computedQuery); - Set<String> intersection = new LinkedHashSet<>(); - for (String term : promptTerms) { - if (!queryTerms.contains(term)) continue; - final String cleaned = cleanToken(term); - if (cleaned.isEmpty()) continue; - intersection.add(cleaned); - if (maxTerms > 0 && intersection.size() >= maxTerms) break; - } - return intersection; - } - - private static String cleanToken(String term) { - if (term == null) return ""; - String cleaned = term.replaceAll("[^A-Za-z0-9]", ""); - if (cleaned.length() < 2) return ""; - return cleaned.toLowerCase(); - } - - private static JSONObject responseLine(String payload) { - JSONObject j = new JSONObject(true); - try { - j.put("id", "log"); - j.put("object", "chat.completion.chunk"); - j.put("created", System.currentTimeMillis() / 1000); - j.put("model", "log"); - j.put("system_fingerprint", "YaCy"); - JSONArray choices = new JSONArray(); - JSONObject choice = new JSONObject(true); // {"index":0,"delta":{"role":"assistant","content":"ߘ" - choice.put("index", 0); - JSONObject delta = new JSONObject(true); - delta.put("role", "assistant"); - delta.put("content", payload); - choice.put("delta", delta); - choices.put(choice); - j.put("choices", choices); - // j.put("finish_reason", null); // this is problematic with the JSON library - } catch (JSONException e) { - } - return j; - } public static void pruneOldEntries(long now) { while (true) { |