summaryrefslogtreecommitdiff
path: root/source
diff options
context:
space:
mode:
authorMichael Peter Christen <mc@yacy.net>2026-02-07 10:51:54 +0100
committerMichael Peter Christen <mc@yacy.net>2026-02-07 10:51:54 +0100
commit16a91ab12ccb6ea414687f20ccbc3af53b4e95e1 (patch)
tree1bf59b2d8c2f2e38b6ac906d2d8443a244ab938f /source
parentfb7367ed5586f8730d7ddc578963027d48d1ef9c (diff)
added tool calling abilities to yacychat.html and added a tool handler to RAGProxyServlet; also provided a large set of basic tools to handle calculations, date understanding, unit conversion, web fetch, number parsing, self reflection and more
Diffstat (limited to 'source')
-rw-r--r--source/net/yacy/ai/LLM.java2
-rw-r--r--source/net/yacy/ai/LLMSwarm.java2
-rw-r--r--source/net/yacy/ai/RAGAugmentor.java614
-rw-r--r--source/net/yacy/ai/ToolCallProtocol.java634
-rw-r--r--source/net/yacy/ai/ToolHandler.java43
-rw-r--r--source/net/yacy/ai/ToolProvider.java226
-rw-r--r--source/net/yacy/ai/tools/CalculatorTool.java358
-rw-r--r--source/net/yacy/ai/tools/ChitChatTool.java134
-rw-r--r--source/net/yacy/ai/tools/DateMathTool.java214
-rw-r--r--source/net/yacy/ai/tools/DateTimeTool.java112
-rw-r--r--source/net/yacy/ai/tools/HttpJsonTool.java244
-rw-r--r--source/net/yacy/ai/tools/NumberParserTool.java403
-rw-r--r--source/net/yacy/ai/tools/PromptToMermaidTool.java314
-rw-r--r--source/net/yacy/ai/tools/SelfReflectTool.java115
-rw-r--r--source/net/yacy/ai/tools/TableOpsTool.java285
-rw-r--r--source/net/yacy/ai/tools/UnitConverterTool.java200
-rw-r--r--source/net/yacy/ai/tools/WebFetchTool.java168
-rw-r--r--source/net/yacy/ai/tools/WikipediaLinkCreatorTool.java169
-rw-r--r--source/net/yacy/http/servlets/MCPSearchServlet.java77
-rw-r--r--source/net/yacy/http/servlets/RAGProxyServlet.java635
20 files changed, 4344 insertions, 605 deletions
diff --git a/source/net/yacy/ai/LLM.java b/source/net/yacy/ai/LLM.java
index 7987cbb29..cf12265ef 100644
--- a/source/net/yacy/ai/LLM.java
+++ b/source/net/yacy/ai/LLM.java
@@ -1,5 +1,5 @@
/**
- * LLMEndpoint
+ * LLM
* Copyright 2024 by Michael Peter Christen
* First released 17.05.2024 at https://yacy.net
*
diff --git a/source/net/yacy/ai/LLMSwarm.java b/source/net/yacy/ai/LLMSwarm.java
index 9aa3286b2..67508c236 100644
--- a/source/net/yacy/ai/LLMSwarm.java
+++ b/source/net/yacy/ai/LLMSwarm.java
@@ -1,5 +1,5 @@
/**
- * OllamaSwarm.java
+ * LLMSwarm.java
* Copyright 2025 by Michael Peter Christen
* First released 01.06.2025 at https://yacy.net
*
diff --git a/source/net/yacy/ai/RAGAugmentor.java b/source/net/yacy/ai/RAGAugmentor.java
new file mode 100644
index 000000000..d1dbb18db
--- /dev/null
+++ b/source/net/yacy/ai/RAGAugmentor.java
@@ -0,0 +1,614 @@
+/**
+ * RAGAugmentor
+ * Copyright 2026 by Michael Peter Christen
+ * First released 06.02.2026 at https://yacy.net
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program in the file lgpl21.txt
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.ai;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import net.yacy.cora.document.analysis.Classification;
+import net.yacy.cora.document.id.DigestURL;
+import net.yacy.cora.document.id.MultiProtocolURL;
+import net.yacy.cora.federate.solr.SolrType;
+import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector;
+import net.yacy.cora.federate.yacy.CacheStrategy;
+import net.yacy.cora.lod.vocabulary.Tagging;
+import net.yacy.cora.protocol.ClientIdentification;
+import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.kelondro.data.meta.URIMetadataNode;
+import net.yacy.search.Switchboard;
+import net.yacy.search.SwitchboardConstants;
+import net.yacy.search.query.QueryGoal;
+import net.yacy.search.query.QueryModifier;
+import net.yacy.search.query.QueryParams;
+import net.yacy.search.query.SearchEvent;
+import net.yacy.search.query.SearchEventCache;
+import net.yacy.search.ranking.RankingProfile;
+import net.yacy.search.schema.CollectionSchema;
+import net.yacy.search.snippet.TextSnippet;
+
+/**
+ * Utility methods that enrich prompts/responses with search-derived context for
+ * Retrieval-Augmented Generation (RAG).
+ * <p>
+ * This class provides:
+ * <ul>
+ * <li>local Solr-backed search result extraction</li>
+ * <li>global YaCy network search extraction</li>
+ * <li>markdown condensation of search results</li>
+ * <li>token intersection helpers for query boosting</li>
+ * <li>snippet scoring/selection utilities</li>
+ * </ul>
+ */
+public final class RAGAugmentor {
+
+ /**
+ * Utility class; not instantiable.
+ */
+ private RAGAugmentor() {}
+
+ /**
+ * Executes local index search with default boost terms.
+ *
+ * @param query query string
+ * @param count max number of results
+ * @param includeSnippet include text snippet field in response objects
+ * @return JSON array with {@code url,title[,text]} entries
+ */
+ public static JSONArray searchResults(String query, int count, final boolean includeSnippet) {
+ return searchResults(query, count, includeSnippet, new LinkedHashSet<>());
+ }
+
+ /**
+ * Executes local Solr search with optional dynamic boost terms.
+ *
+ * @param query query string
+ * @param count max number of results
+ * @param includeSnippet include text snippets from indexed text field
+ * @param boostTerms optional overlap terms used to bias ranking
+ * @return JSON array with normalized search result objects
+ */
+ public static JSONArray searchResults(String query, int count, final boolean includeSnippet, final Set<String> boostTerms) {
+ final JSONArray results = new JSONArray();
+ if (query == null || query.length() == 0 || count == 0) return results;
+ Switchboard sb = Switchboard.getSwitchboard();
+ EmbeddedSolrConnector connector = sb.index.fulltext().getDefaultEmbeddedConnector();
+ final SolrQuery params = new SolrQuery();
+ // Base query and parser setup.
+ params.setQuery(query);
+ params.set("defType", "edismax");
+ // Static field boosts favor title/headings over body text.
+ params.set("qf",
+ CollectionSchema.title.getSolrFieldName() + "^3 " +
+ CollectionSchema.text_t.getSolrFieldName() + "^1 " +
+ CollectionSchema.sku.getSolrFieldName() + "^0.5 " +
+ CollectionSchema.h1_txt.getSolrFieldName() + "^2");
+ params.set("pf",
+ CollectionSchema.title.getSolrFieldName() + "^5 " +
+ CollectionSchema.text_t.getSolrFieldName() + "^2");
+ final List<String> bqParts = new ArrayList<>();
+ if (boostTerms != null && !boostTerms.isEmpty()) {
+ // Apply weak boosts to overlap terms, keeping lexical query dominant.
+ for (String term : boostTerms) {
+ if (term == null || term.isEmpty()) continue;
+ bqParts.add(CollectionSchema.title.getSolrFieldName() + ":" + term + "^0.5");
+ bqParts.add(CollectionSchema.h1_txt.getSolrFieldName() + ":" + term + "^0.4");
+ bqParts.add(CollectionSchema.text_t.getSolrFieldName() + ":" + term + "^0.2");
+ }
+ }
+ // Slightly prefer archive/container file extensions in this use case.
+ bqParts.add("(" + CollectionSchema.url_file_ext_s.getSolrFieldName() + ":(zip rar 7z tar gz bz2 xz tgz))^0.1");
+ params.set("bq", String.join(" ", bqParts));
+ params.setRows(count);
+ params.setStart(0);
+ params.setFacet(false);
+ params.clearSorts();
+ // Fetch only fields needed for tool / markdown rendering.
+ params.setFields(
+ CollectionSchema.sku.getSolrFieldName(), CollectionSchema.title.getSolrFieldName(), CollectionSchema.text_t.getSolrFieldName(),
+ CollectionSchema.description_txt.getSolrFieldName(), CollectionSchema.keywords.getSolrFieldName(), CollectionSchema.synonyms_sxt.getSolrFieldName(),
+ CollectionSchema.h1_txt.getSolrFieldName(), CollectionSchema.h2_txt.getSolrFieldName(), CollectionSchema.h3_txt.getSolrFieldName(),
+ CollectionSchema.h4_txt.getSolrFieldName(), CollectionSchema.h5_txt.getSolrFieldName(), CollectionSchema.h6_txt.getSolrFieldName()
+ );
+ params.setIncludeScore(true);
+ params.set("df", CollectionSchema.text_t.getSolrFieldName());
+
+ try {
+ final SolrDocumentList sdl = connector.getDocumentListByParams(params);
+ Iterator<SolrDocument> i = sdl.iterator();
+ while (i.hasNext()) {
+ try {
+ SolrDocument doc = i.next();
+ final JSONObject result = new JSONObject(true);
+ String url = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName());
+ result.put("url", url == null ? "" : url.trim());
+ String title = getOneString(doc, CollectionSchema.title);
+ result.put("title", title == null ? "" : title.trim());
+ if (includeSnippet) {
+ // Use indexed text body as quick snippet source.
+ String text = (String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName());
+ result.put("text", limitSnippet(text == null ? "" : text.trim(), 2000));
+ }
+ results.put(result);
+ } catch (JSONException e) {
+ }
+ }
+ return results;
+ } catch (SolrException | IOException e) {
+ return results;
+ }
+ }
+
+ /**
+ * Renders search results as compact markdown context using local-search mode.
+ *
+ * @param query query string
+ * @param count max number of search rows
+ * @param global when true, use global YaCy search
+ * @return markdown context block
+ */
+ public static String searchResultsAsMarkdown(String query, int count, boolean global) {
+ return searchResultsAsMarkdown(query, count, global, new LinkedHashSet<>());
+ }
+
+ /**
+ * Renders search results as compact markdown and applies snippet ranking to
+ * reduce noise.
+ *
+ * @param query query string
+ * @param count max number of search rows
+ * @param global when true, use global YaCy search; otherwise local Solr
+ * @param boostTerms optional local-search boost terms
+ * @return markdown formatted context used in downstream prompt augmentation
+ */
+ public static String searchResultsAsMarkdown(String query, int count, boolean global, final Set<String> boostTerms) {
+ final long searchStart = System.currentTimeMillis();
+ JSONArray searchResults = global ? searchResultsGlobal(query, count, true) : searchResults(query, count, true, boostTerms);
+ ConcurrentLog.info("RAGProxy", "searchResults=" + searchResults.length() + " global=" + global + " searchMs=" + (System.currentTimeMillis() - searchStart));
+ StringBuilder sb = new StringBuilder();
+
+ // Convert raw rows into scoreable snippet candidates.
+ List<Snippet> results = new ArrayList<>();
+ for (int i = 0; i < searchResults.length(); i++) {
+ try {
+ JSONObject r = searchResults.getJSONObject(i);
+ String title = r.optString("title", "");
+ String url = r.optString("url", "");
+ String text = r.optString("text", "");
+ if (title.isEmpty()) title = url;
+ if (text.isEmpty()) text = title;
+ if (title.length() > 0 && text.length() > 0) {
+ Snippet snippet = new Snippet(query, text, url, title, 256);
+ if (snippet.getText().length() > 0) results.add(snippet);
+ }
+ } catch (JSONException e) {}
+ }
+
+ // Lower score is better with the current tf-idf based chunk scorer.
+ results.sort(Comparator.comparingDouble(Snippet::getScore));
+ // Keep top half to avoid overloading the model context window.
+ int limit = results.size() / 2;
+ if (results.size() > 0 && limit == 0) limit = 1;
+ for (int i = 0; i < limit; i++) {
+ Snippet snippet = results.get(i);
+ sb.append("## ").append(snippet.getTitle()).append("\n");
+ sb.append(snippet.text).append("\n");
+ if (snippet.getURL().length() > 0) sb.append("Source: ").append(snippet.getURL()).append("\n");
+ sb.append("\n\n");
+ }
+
+ ConcurrentLog.info("RAGProxy", "markdownChars=" + sb.length() + " snippetCount=" + results.size());
+ return sb.toString();
+ }
+
+ /**
+ * Executes a global/distributed YaCy search event and maps results into a
+ * compact JSON format.
+ *
+ * @param query query string
+ * @param count max number of results
+ * @param includeSnippet include snippet text when available
+ * @return JSON array with normalized result objects
+ */
+ public static JSONArray searchResultsGlobal(String query, int count, final boolean includeSnippet) {
+ final JSONArray results = new JSONArray();
+ if (query == null || query.length() == 0 || count == 0) return results;
+ final Switchboard sb = Switchboard.getSwitchboard();
+ final RankingProfile ranking = sb.getRanking();
+ final int timezoneOffset = 0;
+ final QueryModifier modifier = new QueryModifier(timezoneOffset);
+ // Parse modifiers and normalize effective query string.
+ String querystring = modifier.parse(query);
+ if (querystring.length() == 0) querystring = query == null ? "" : query.trim();
+ if (querystring.length() == 0) return results;
+ final QueryGoal qg = new QueryGoal(querystring);
+ // Construct a standard text-domain global query.
+ final QueryParams theQuery = new QueryParams(
+ qg,
+ modifier,
+ 0,
+ "",
+ Classification.ContentDomain.TEXT,
+ "",
+ timezoneOffset,
+ new HashSet<Tagging.Metatag>(),
+ CacheStrategy.IFFRESH,
+ count,
+ 0,
+ ".*",
+ null,
+ null,
+ QueryParams.Searchdom.GLOBAL,
+ null,
+ true,
+ DigestURL.hosthashess(sb.getConfig("search.excludehosth", "")),
+ MultiProtocolURL.TLD_any_zone_filter,
+ null,
+ false,
+ sb.index,
+ ranking,
+ ClientIdentification.yacyIntranetCrawlerAgent.userAgent(),
+ 0.0d,
+ 0.0d,
+ 0.0d,
+ sb.getConfigSet("search.navigation"));
+ final SearchEvent theSearch = SearchEventCache.getEvent(
+ theQuery,
+ sb.peers,
+ sb.tables,
+ (sb.isRobinsonMode()) ? sb.clusterhashes : null,
+ false,
+ sb.loader,
+ (int) sb.getConfigLong(
+ SwitchboardConstants.REMOTESEARCH_MAXCOUNT_USER,
+ sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXCOUNT_DEFAULT, 10)),
+ sb.getConfigLong(
+ SwitchboardConstants.REMOTESEARCH_MAXTIME_USER,
+ sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000)));
+ final long timeout = sb.getConfigLong(
+ SwitchboardConstants.REMOTESEARCH_MAXTIME_USER,
+ sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000));
+ // Wait until remote feeds are done (or timeout), then stabilize ordering.
+ waitForFeedingAndResort(theSearch, timeout);
+ for (int i = 0; i < count; i++) {
+ URIMetadataNode node = theSearch.oneResult(i, timeout);
+ if (node == null) break;
+ try {
+ final JSONObject result = new JSONObject(true);
+ result.put("url", node.urlstring());
+ result.put("title", node.title());
+ if (includeSnippet) {
+ // Prefer direct snippet; fall back to textSnippet, description, then text field.
+ String text = node.snippet();
+ if (text == null || text.isEmpty()) {
+ TextSnippet snippet = node.textSnippet();
+ if (snippet != null && snippet.exists() && !snippet.getErrorCode().fail()) text = snippet.getLineRaw();
+ }
+ if (text == null || text.isEmpty()) text = firstFieldString(node.getFieldValue(CollectionSchema.description_txt.getSolrFieldName()));
+ if (text == null || text.isEmpty()) text = firstFieldString(node.getFieldValue(CollectionSchema.text_t.getSolrFieldName()));
+ result.put("text", limitSnippet(text == null ? "" : text.trim(), 2000));
+ }
+ results.put(result);
+ } catch (JSONException e) {
+ }
+ }
+ return results;
+ }
+
+ /**
+ * Uses an LLM list schema prompt to compute likely discriminative search
+ * terms for a user prompt.
+ *
+ * @param llm configured LLM backend
+ * @param model target model name
+ * @param prompt user prompt to analyze
+ * @return space-separated lowercase term list or {@code null} on failure
+ */
+ public static String searchWordsForPrompt(LLM llm, String model, String prompt) {
+ final String question = prompt == null ? "" : prompt;
+ if (llm == null || model == null || model.isEmpty()) return null;
+ try {
+ LLM.Context context = new LLM.Context("\n\nYou may receive additional expert knowledge in the user prompt after a 'Additional Information' headline to enhance your knowledge. Use it only if applicable.");
+ context.addPrompt(question);
+ Set<String> singlewords = new LinkedHashSet<>();
+ String[] a = LLM.stringsFromChat(llm.chat(model, context, LLM.listSchema, 200));
+ if (a == null || a.length == 0) return null;
+ for (String s: a) {
+ if (s == null) continue;
+ // Flatten model output into unique lowercased tokens.
+ for (String t: s.split(" ")) if (!t.isEmpty()) singlewords.add(t.toLowerCase());
+ }
+ if (singlewords.isEmpty()) return null;
+ StringBuilder query = new StringBuilder();
+ for (String s: singlewords) query.append(s).append(' ');
+ String querys = query.toString().trim();
+ if (querys.length() == 0) return null;
+ return querys;
+ } catch (IOException | JSONException e) {
+ e.printStackTrace();
+ return null;
+ }
+ }
+
+ /**
+ * Computes token overlap between original prompt and computed query terms.
+ *
+ * @param originalPrompt raw user prompt
+ * @param computedQuery generated query terms
+ * @param maxTerms max terms to keep; {@code <=0} means unlimited
+ * @return cleaned ordered overlap set
+ */
+ public static Set<String> intersectTokens(String originalPrompt, String computedQuery, int maxTerms) {
+ Set<String> promptTerms = querySet(originalPrompt == null ? "" : originalPrompt);
+ Set<String> queryTerms = querySet(computedQuery == null ? "" : computedQuery);
+ Set<String> intersection = new LinkedHashSet<>();
+ for (String term : promptTerms) {
+ if (!queryTerms.contains(term)) continue;
+ final String cleaned = cleanToken(term);
+ if (cleaned.isEmpty()) continue;
+ intersection.add(cleaned);
+ if (maxTerms > 0 && intersection.size() >= maxTerms) break;
+ }
+ return intersection;
+ }
+
+ /**
+ * Splits text into sentence-aware chunks around a target max length.
+ *
+ * @param text source text
+ * @param len approximate chunk length
+ * @return ordered chunks
+ */
+ public static List<String> slicer(String text, int len) {
+ List<String> result = new ArrayList<>();
+ if (text == null || len <= 0) return result;
+
+ int start = 0;
+ while (start < text.length()) {
+ int end = Math.min(start + len, text.length());
+ // Extend to sentence boundary when possible.
+ while (end < text.length()) {
+ char ch = text.charAt(end - 1);
+ if ((ch == '.' || ch == '?' || ch == '!') && Character.isWhitespace(text.charAt(end))) break;
+ end++;
+ }
+ result.add(text.substring(start, end));
+ start = end;
+ }
+ return result;
+ }
+
+ /**
+ * Converts a query string into a normalized token set.
+ *
+ * @param query raw query text
+ * @return lowercase token set
+ */
+ private static Set<String> querySet(String query) {
+ return Arrays.stream(query.trim().toLowerCase().split("\\s+"))
+ .map(String::toLowerCase)
+ .filter(word -> !word.isEmpty())
+ .collect(Collectors.toSet());
+ }
+
+ /**
+ * Removes non-alphanumeric characters and enforces minimal token length.
+ *
+ * @param term source token
+ * @return cleaned lowercase token or empty string
+ */
+ private static String cleanToken(String term) {
+ if (term == null) return "";
+ String cleaned = term.replaceAll("[^A-Za-z0-9]", "");
+ if (cleaned.length() < 2) return "";
+ return cleaned.toLowerCase();
+ }
+
+ /**
+ * Truncates snippet text to a maximum character count.
+ *
+ * @param text input text
+ * @param maxChars limit
+ * @return truncated or original text
+ */
+ private static String limitSnippet(String text, int maxChars) {
+ if (text == null) return "";
+ if (maxChars <= 0 || text.length() <= maxChars) return text;
+ return text.substring(0, maxChars);
+ }
+
+ /**
+ * Returns first non-null string from a field value that may be scalar or
+ * collection.
+ *
+ * @param value field value
+ * @return first string representation or empty string
+ */
+ private static String firstFieldString(Object value) {
+ if (value == null) return "";
+ if (value instanceof Collection) {
+ for (Object item : (Collection<?>) value) if (item != null) return item.toString();
+ return "";
+ }
+ return value.toString();
+ }
+
+ /**
+ * Waits for global search feeding completion and then resorts cached results.
+ *
+ * @param search active search event
+ * @param timeoutMs max wait time
+ */
+ private static void waitForFeedingAndResort(SearchEvent search, long timeoutMs) {
+ if (search == null || timeoutMs <= 0) return;
+ final long end = System.currentTimeMillis() + timeoutMs;
+ while (!search.isFeedingFinished() && System.currentTimeMillis() < end) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ break;
+ }
+ }
+ search.resortCachedResults();
+ }
+
+ /**
+ * Reads a single string value from a (possibly multivalued) Solr field.
+ *
+ * @param doc source Solr document
+ * @param field schema field descriptor
+ * @return first string value or empty string
+ */
+ private static String getOneString(SolrDocument doc, CollectionSchema field) {
+ assert field.isMultiValued();
+ assert field.getType() == SolrType.string || field.getType() == SolrType.text_general;
+ Object r = doc.getFieldValue(field.getSolrFieldName());
+ if (r == null) return "";
+ if (r instanceof ArrayList) {
+ return (String) ((ArrayList<?>) r).get(0);
+ }
+ return r.toString();
+ }
+
+ /**
+ * Represents one candidate snippet around a search result and stores its
+ * relevance score relative to the query.
+ */
+ private static class Snippet {
+ private String text, url, title;
+ private double score;
+
+ /**
+ * Scores text chunks and keeps the best chunk plus direct neighbors to
+ * preserve context continuity.
+ *
+ * @param query query text
+ * @param text source document/snippet text
+ * @param url source URL
+ * @param title source title
+ * @param maxChunkLength target chunk size
+ */
+ public Snippet(String query, String text, String url, String title, int maxChunkLength) {
+ this.url = url;
+ this.title = title;
+ this.score = 0.0;
+
+ if (text == null || text.isEmpty() || maxChunkLength <= 0 || query == null) {
+ this.text = "";
+ return;
+ }
+
+ List<String> chunks = slicer(text, maxChunkLength);
+ if (chunks.isEmpty()) {
+ this.text = "";
+ return;
+ }
+ List<String> chunksLowerCase = new ArrayList<>(chunks.size());
+ // Cache lowercase chunks so token comparisons are case-insensitive.
+ for (String chunk: chunks) chunksLowerCase.add(chunk.toLowerCase());
+
+ Set<String> queryWordSet = querySet(query);
+ if (queryWordSet.isEmpty()) {
+ this.text = "";
+ return;
+ }
+
+ int totalChunks = chunksLowerCase.size();
+ Map<String, Double> idf = new HashMap<>();
+ for (String word: queryWordSet) {
+ int docFreq = 0;
+ for (String chunk: chunksLowerCase) {
+ if (chunk.contains(word)) docFreq++;
+ }
+ // Smoothed IDF to avoid divide-by-zero and extreme values.
+ idf.put(word, Math.log((double) totalChunks / (docFreq + 1)) + 1);
+ }
+
+ Map<Integer, Double> chunkScores = new HashMap<>();
+ for (int i = 0; i < chunksLowerCase.size(); i++) {
+ String chunk = chunksLowerCase.get(i);
+ double score = 0.0;
+ Map<String, Integer> tf = new HashMap<>();
+
+ String[] wordsInChunk = chunk.split("\\s+");
+ for (String w : wordsInChunk) {
+ String cleanWord = w.replaceAll("[.,!?;:]", "");
+ if (cleanWord.length() > 0 && queryWordSet.contains(cleanWord)) {
+ tf.put(cleanWord, tf.getOrDefault(cleanWord, 0) + 1);
+ }
+ }
+
+ for (String word: queryWordSet) {
+ int tfValue = tf.getOrDefault(word, 0);
+ double tfIdf = (double) tfValue * idf.getOrDefault(word, 1.0);
+ score += tfIdf;
+ }
+ chunkScores.put(i, score);
+ }
+
+ int topChunkIndex = -1;
+ for (Map.Entry<Integer, Double> entry: chunkScores.entrySet()) {
+ // Keep best-scoring chunk index.
+ if (entry.getValue() > this.score) {
+ this.score = entry.getValue();
+ topChunkIndex = entry.getKey();
+ }
+ }
+
+ if (topChunkIndex < 0) {
+ this.text = "";
+ this.score = 0.0;
+ return;
+ }
+
+ List<String> snippetChunks = new ArrayList<>();
+ // Include neighboring chunks to reduce abrupt starts/ends.
+ if (topChunkIndex > 0) snippetChunks.add(chunks.get(topChunkIndex - 1));
+ snippetChunks.add(chunks.get(topChunkIndex));
+ if (topChunkIndex < chunks.size() - 1) snippetChunks.add(chunks.get(topChunkIndex + 1));
+ this.text = String.join(" ", snippetChunks);
+ }
+
+ public double getScore() { return this.score; }
+ public String getText() { return this.text; }
+ public String getURL() { return this.url; }
+ public String getTitle() { return this.title; }
+ }
+}
diff --git a/source/net/yacy/ai/ToolCallProtocol.java b/source/net/yacy/ai/ToolCallProtocol.java
new file mode 100644
index 000000000..8f526beaa
--- /dev/null
+++ b/source/net/yacy/ai/ToolCallProtocol.java
@@ -0,0 +1,634 @@
+/**
+ * ToolCallProtocol
+ * Copyright 2026 by Michael Peter Christen
+ * First released 06.02.2026 at https://yacy.net
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program in the file lgpl21.txt
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.ai;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.servlet.ServletOutputStream;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONTokener;
+
+
+/**
+ * Implements the protocol glue for streamed chat completions that include tool
+ * calls.
+ * <p>
+ * Main responsibilities:
+ * <ul>
+ * <li>Collect partial tool-call fragments from streaming deltas.</li>
+ * <li>Merge fragments into complete tool calls (index-based merge).</li>
+ * <li>Execute tools once a tool turn is complete.</li>
+ * <li>Append assistant + tool messages back to the conversation.</li>
+ * <li>Start follow-up model rounds until no further tool call is requested.</li>
+ * </ul>
+ * <p>
+ * The class also enriches streamed lines with {@code tool-calls} and
+ * {@code tool-results} metadata so clients can display tool activity while the
+ * stream is forwarded.
+ */
+public final class ToolCallProtocol {
+
+ /**
+ * Hard stop to prevent infinite tool loops (assistant requests tool calls
+ * repeatedly without converging to a final answer).
+ */
+ private static final int MAX_TOOL_ROUNDS = 8;
+
+ /**
+ * Utility class; not instantiable.
+ */
+ private ToolCallProtocol() {}
+
+ /**
+ * Builds a request body that is ready for tool-aware chat completion calls.
+ * <p>
+ * This method clones the input object, injects tool definitions and optionally
+ * enforces streaming.
+ *
+ * @param body original request body
+ * @param forceStream when true, sets {@code stream=true} in the cloned body
+ * @return prepared request body clone
+ */
+ public static JSONObject prepareToolRequestBody(JSONObject body, boolean forceStream) {
+ try {
+ final JSONObject prepared = body == null ? new JSONObject(true) : new JSONObject(body.toString());
+ if (forceStream) prepared.put("stream", true);
+ net.yacy.ai.ToolProvider.ensureTools(prepared);
+ return prepared;
+ } catch (JSONException e) {
+ final JSONObject fallback = new JSONObject(true);
+ net.yacy.ai.ToolProvider.ensureTools(fallback);
+ return fallback;
+ }
+ }
+
+ /**
+ * Runs the full server-side tool-stream lifecycle for one chat request.
+ * <p>
+ * Lifecycle:
+ * <ol>
+ * <li>Prepare request body for tools.</li>
+ * <li>Send initial upstream chat request.</li>
+ * <li>Forward initial stream while collecting tool calls.</li>
+ * <li>If tool calls appeared, continue tool rounds until completion.</li>
+ * </ol>
+ *
+ * @param out downstream SSE output stream
+ * @param llm4Chat model config
+ * @param originalBody request body template
+ * @param messages mutable message history
+ * @param initialMetadata optional JSON fields to inject into the first JSON data line
+ * @return upstream HTTP status code for the initial request
+ * @throws IOException on network/stream/protocol errors
+ */
+ public static int proxyToolLifecycle(ServletOutputStream out, LLM.LLMModel llm4Chat, JSONObject originalBody, JSONArray messages, JSONObject initialMetadata) throws IOException {
+ final JSONObject preparedBody = prepareToolRequestBody(originalBody, false);
+ final HttpURLConnection conn = openChatCompletionConnection(llm4Chat, preparedBody);
+ final int status = conn.getResponseCode();
+ if (status == 200) {
+ handleInitialStreamAndContinue(out, conn, llm4Chat, preparedBody, messages, initialMetadata);
+ }
+ return status;
+ }
+
+ /**
+ * Extracts streamed assistant content and tool-call deltas from one chat chunk.
+ * <p>
+ * Supports both modern {@code delta.tool_calls} and legacy
+ * {@code delta.function_call} formats.
+ *
+ * @param j streamed JSON chunk
+ * @param toolCalls mutable map collecting tool calls by index
+ * @param assistantContent buffer accumulating assistant text tokens
+ * @param sawToolCalls single-item boolean holder set to true once any tool
+ * call appears in this stream
+ */
+ public static void captureToolCalls(JSONObject j, Map<Integer, ToolCall> toolCalls, StringBuilder assistantContent, boolean[] sawToolCalls) {
+ if (j == null) return;
+ // Only first choice is processed because streaming APIs typically emit one choice.
+ JSONArray choices = j.optJSONArray("choices");
+ if (choices == null || choices.length() == 0) return;
+ JSONObject choice = choices.optJSONObject(0);
+ if (choice == null) return;
+ JSONObject delta = choice.optJSONObject("delta");
+ if (delta == null) return;
+
+ // Assistant text arrives token-by-token in streaming mode.
+ String content = delta.optString("content", null);
+ if (content != null) assistantContent.append(content);
+
+ // Preferred modern tool-call format.
+ JSONArray toolDelta = delta.optJSONArray("tool_calls");
+ if (toolDelta != null) {
+ sawToolCalls[0] = true;
+ mergeToolCalls(toolCalls, toolDelta);
+ }
+
+ // Legacy compatibility: transform function_call into tool_calls-like shape.
+ JSONObject functionCall = delta.optJSONObject("function_call");
+ if (functionCall != null) {
+ sawToolCalls[0] = true;
+ try {
+ JSONArray wrapper = new JSONArray();
+ JSONObject entry = new JSONObject(true);
+ entry.put("index", 0);
+ entry.put("type", "function");
+ entry.put("function", functionCall);
+ wrapper.put(entry);
+ mergeToolCalls(toolCalls, wrapper);
+ } catch (JSONException e) {
+ // ignore malformed legacy function_call chunk
+ }
+ }
+ }
+
+ /**
+ * Forwards the initial upstream stream to the client while collecting tool
+ * calls, then executes follow-up tool rounds when needed.
+ *
+ * @param out downstream SSE output stream
+ * @param conn initial upstream connection
+ * @param llm4Chat model config for follow-up rounds
+ * @param originalBody request body template for follow-up rounds
+ * @param messages mutable conversation history
+ * @param initialMetadata optional metadata injected once into the first JSON line
+ * @throws IOException on I/O errors
+ */
+ private static void handleInitialStreamAndContinue(ServletOutputStream out, HttpURLConnection conn, LLM.LLMModel llm4Chat, JSONObject originalBody, JSONArray messages, JSONObject initialMetadata) throws IOException {
+ final StringBuilder assistantContent = new StringBuilder();
+ final Map<Integer, ToolCall> toolCalls = new HashMap<>();
+ final boolean[] sawToolCalls = new boolean[]{false};
+ boolean firstJsonDataLinePending = initialMetadata != null && initialMetadata.length() > 0;
+
+ try (BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8))) {
+ String line;
+ while ((line = in.readLine()) != null) {
+ boolean isDoneLine = "data: [DONE]".equals(line.trim()) || "[DONE]".equals(line.trim());
+ if (firstJsonDataLinePending) {
+ String patched = injectJsonMetadata(line, initialMetadata);
+ if (patched != null) {
+ line = patched;
+ firstJsonDataLinePending = false;
+ }
+ }
+ if (line.startsWith("data:")) {
+ int p = line.indexOf('{');
+ if (p > 0) {
+ try {
+ JSONObject j = new JSONObject(new JSONTokener(line.substring(p)));
+ captureToolCalls(j, toolCalls, assistantContent, sawToolCalls);
+ } catch (JSONException e) {
+ // ignore malformed stream chunks and continue forwarding stream
+ }
+ }
+ }
+ if (!isDoneLine || !sawToolCalls[0]) {
+ out.println(line);
+ out.flush();
+ }
+ }
+ } finally {
+ conn.disconnect();
+ }
+
+ if (sawToolCalls[0]) {
+ handleToolCallsAndContinue(out, llm4Chat, originalBody, messages, assistantContent.toString(), toolCalls);
+ }
+ }
+
+ /**
+ * Runs the complete tool-turn loop after initial stream parsing detected tool
+ * calls.
+ * <p>
+ * Per round:
+ * <ol>
+ * <li>Append assistant tool request + tool response messages.</li>
+ * <li>Call the chat completion endpoint again with updated messages.</li>
+ * <li>Forward streamed lines to the client while capturing potential next
+ * tool calls.</li>
+ * <li>Stop when no more tool calls are requested or the round cap is hit.</li>
+ * </ol>
+ *
+ * @param out output stream to the client (SSE style lines)
+ * @param llm4Chat selected model configuration
+ * @param originalBody original request body template
+ * @param messages mutable conversation messages from original request
+ * @param assistantContent already streamed assistant text from current round
+ * @param toolCalls collected tool calls from current round
+ * @throws IOException when network I/O or JSON processing fails
+ */
+ public static void handleToolCallsAndContinue(ServletOutputStream out, LLM.LLMModel llm4Chat, JSONObject originalBody, JSONArray messages, String assistantContent, Map<Integer, ToolCall> toolCalls) throws IOException {
+ try {
+ // Work on a copy so caller-owned message arrays are not modified unexpectedly.
+ JSONArray newMessages = new JSONArray();
+ for (int i = 0; i < messages.length(); i++) {
+ newMessages.put(messages.get(i));
+ }
+ String roundAssistantContent = assistantContent == null ? "" : assistantContent;
+ Map<Integer, ToolCall> roundToolCalls = new HashMap<>(toolCalls);
+ // Track total calls per tool across this whole tool-turn lifecycle.
+ Map<String, Integer> toolCallCounters = new HashMap<>();
+
+ for (int round = 0; round < MAX_TOOL_ROUNDS; round++) {
+ // Convert captured tool calls into assistant/tool messages and execute tools.
+ ToolRoundData roundData = appendAssistantAndToolMessages(newMessages, roundAssistantContent, roundToolCalls, toolCallCounters);
+ if (roundData == null || roundData.toolResults.length() == 0) {
+ // No executable tool calls; terminate stream cleanly.
+ out.println("data: [DONE]");
+ out.flush();
+ return;
+ }
+
+ // Build follow-up completion request from original body template.
+ final JSONObject followup = prepareToolRequestBody(originalBody, true);
+ followup.put("messages", newMessages);
+ final HttpURLConnection followConn = openChatCompletionConnection(llm4Chat, followup);
+ if (followConn.getResponseCode() != 200) {
+ // Upstream error: close downstream stream instead of sending broken chunks.
+ out.println("data: [DONE]");
+ out.flush();
+ followConn.disconnect();
+ return;
+ }
+
+ // Collect data for the next potential tool round while forwarding this stream.
+ final StringBuilder nextAssistantContent = new StringBuilder();
+ final Map<Integer, ToolCall> nextToolCalls = new HashMap<>();
+ final boolean[] sawToolCalls = new boolean[]{false};
+
+ try (BufferedReader in = new BufferedReader(new InputStreamReader(followConn.getInputStream(), StandardCharsets.UTF_8))) {
+ String line;
+ boolean injectedToolResults = false;
+ while ((line = in.readLine()) != null) {
+ // Inject tool metadata once into outgoing stream so UI can show executed tools.
+ if (!injectedToolResults && roundData.toolResults.length() > 0) {
+ line = injectToolMetadata(line, roundData);
+ if (line != null && line.indexOf("\"tool-results\"") >= 0) {
+ injectedToolResults = true;
+ }
+ }
+ boolean isDoneLine = "data: [DONE]".equals(line.trim()) || "[DONE]".equals(line.trim());
+ if (line.startsWith("data:")) {
+ int p = line.indexOf('{');
+ if (p > 0) {
+ try {
+ JSONObject j = new JSONObject(new JSONTokener(line.substring(p)));
+ // Parse streamed deltas to capture possible next tool round.
+ captureToolCalls(j, nextToolCalls, nextAssistantContent, sawToolCalls);
+ } catch (JSONException e) {
+ // ignore malformed stream chunks and continue forwarding stream
+ }
+ }
+ }
+ // Hide interim [DONE] if another tool round is expected.
+ if (!isDoneLine || !sawToolCalls[0]) {
+ out.println(line);
+ out.flush();
+ }
+ }
+ } finally {
+ followConn.disconnect();
+ }
+
+ // Ignore phantom rounds where stream signaled tool-calls but no concrete calls were parsed.
+ if (nextToolCalls.isEmpty()) sawToolCalls[0] = false;
+ // Final answer reached; caller already received forwarded stream lines.
+ if (!sawToolCalls[0]) return;
+
+ // Continue with next round tool request that was found in follow-up stream.
+ roundAssistantContent = nextAssistantContent.toString();
+ roundToolCalls = nextToolCalls;
+ }
+ // Safety fallback when round cap is hit.
+ out.println("data: [DONE]");
+ out.flush();
+ } catch (JSONException e) {
+ throw new IOException("JSON processing error in tool handling", e);
+ }
+ }
+
+ /**
+ * Opens and sends one chat completion request.
+ *
+ * @param llm4Chat model config used for endpoint and auth
+ * @param requestBody request payload
+ * @return opened connection after request body has been sent
+ * @throws IOException on URI/network/write failures
+ */
+ private static HttpURLConnection openChatCompletionConnection(LLM.LLMModel llm4Chat, JSONObject requestBody) throws IOException {
+ final URL url;
+ try {
+ url = new URI(llm4Chat.llm.hoststub + "/v1/chat/completions").toURL();
+ } catch (URISyntaxException e) {
+ throw new IOException("Invalid chat completion URL: " + e.getMessage(), e);
+ }
+ final HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+ conn.setRequestMethod("POST");
+ conn.setRequestProperty("Content-Type", "application/json");
+ if (!llm4Chat.llm.api_key.isEmpty()) {
+ conn.setRequestProperty("Authorization", "Bearer " + llm4Chat.llm.api_key);
+ }
+ conn.setDoOutput(true);
+ try (OutputStream os = conn.getOutputStream()) {
+ os.write(requestBody.toString().getBytes(StandardCharsets.UTF_8));
+ os.flush();
+ }
+ return conn;
+ }
+
+ /**
+ * Merges incremental tool call fragments from a streaming delta into the
+ * in-progress tool call map.
+ * <p>
+ * Streaming responses may deliver one tool call over multiple chunks. This method
+ * merges fields by tool-call index:
+ * <ul>
+ * <li>{@code id}, {@code type}, and function {@code name} keep the latest non-empty value.</li>
+ * <li>function {@code arguments} are appended because they are often streamed in pieces.</li>
+ * </ul>
+ *
+ * @param toolCalls current merged tool calls, keyed by call index
+ * @param deltaCalls new delta chunk containing partial tool call objects
+ */
+ private static void mergeToolCalls(Map<Integer, ToolCall> toolCalls, JSONArray deltaCalls) {
+ // Each array element can contain only a fragment of the final tool call.
+ for (int i = 0; i < deltaCalls.length(); i++) {
+ JSONObject delta = deltaCalls.optJSONObject(i);
+ if (delta == null) continue;
+ Integer index = readToolCallIndex(delta);
+ // Ignore chunks without a valid non-negative call index.
+ if (index == null || index.intValue() < 0) continue;
+
+ // Reuse existing partial state for this index or start a new one.
+ ToolCall call = toolCalls.getOrDefault(index, new ToolCall());
+
+ // Scalar fields are replaced when a newer non-empty value arrives.
+ String id = delta.optString("id", null);
+ if (id != null && !id.isEmpty()) call.id = id;
+ String type = delta.optString("type", null);
+ if (type != null && !type.isEmpty()) call.type = type;
+ JSONObject fn = delta.optJSONObject("function");
+ if (fn != null) {
+ String name = fn.optString("name", null);
+ if (name != null && !name.isEmpty()) call.name = name;
+ Object rawArgs = fn.opt("arguments");
+ if (rawArgs instanceof String) {
+ String args = (String) rawArgs;
+ // Arguments are streamed as string fragments and must be concatenated.
+ if (!args.isEmpty()) call.arguments = (call.arguments == null ? "" : call.arguments) + args;
+ }
+ }
+ toolCalls.put(index, call);
+ }
+ }
+
+ /**
+ * Appends one assistant message containing tool_calls plus corresponding tool
+ * role messages with execution results.
+ * <p>
+ * Only executable tool calls are included (must have a function name and valid
+ * JSON arguments if arguments are present).
+ *
+ * @param messages mutable conversation message array to append to
+ * @param assistantContent assistant text that accompanied tool call emission
+ * @param toolCalls merged tool calls keyed by index
+ * @param toolCallCounters cumulative per-tool call counters for this turn
+ * @return round data containing emitted {@code tool_calls} and tool execution
+ * results, or {@code null} when JSON assembly fails
+ */
+ private static ToolRoundData appendAssistantAndToolMessages(JSONArray messages, String assistantContent, Map<Integer, ToolCall> toolCalls, Map<String, Integer> toolCallCounters) {
+ try {
+ // Keep original model call order stable by sorting call indices.
+ List<Integer> indices = new ArrayList<>(toolCalls.keySet());
+ Collections.sort(indices);
+ JSONArray toolCallsArray = new JSONArray();
+ JSONArray toolMessages = new JSONArray();
+ JSONArray toolResults = new JSONArray();
+ for (Integer idx : indices) {
+ ToolCall call = toolCalls.get(idx);
+ if (call == null) continue;
+ // Ensure required defaults for follow-up protocol compliance.
+ if (call.id == null || call.id.isEmpty()) call.id = "toolcall_" + idx + "_" + System.currentTimeMillis();
+ if (call.type == null || call.type.isEmpty()) call.type = "function";
+ if (!isExecutableToolCall(call)) continue;
+ // Enforce per-tool max calls for the complete tool turn lifecycle.
+ final String toolName = call.name == null ? "" : call.name.trim();
+ final int maxCalls = net.yacy.ai.ToolProvider.maxCallsPerTurn(toolName);
+ final int usedCalls = toolCallCounters.getOrDefault(toolName, Integer.valueOf(0)).intValue();
+ if (usedCalls >= maxCalls) continue;
+
+ // Assistant message representation of requested tool call.
+ JSONObject toolCallJson = new JSONObject(true);
+ toolCallJson.put("id", call.id);
+ toolCallJson.put("type", call.type);
+ JSONObject fn = new JSONObject(true);
+ fn.put("name", call.name == null ? "" : call.name);
+ fn.put("arguments", call.arguments == null ? "" : call.arguments);
+ toolCallJson.put("function", fn);
+ toolCallsArray.put(toolCallJson);
+
+ // Execute tool locally and create "tool" role response message.
+ String result = net.yacy.ai.ToolProvider.executeTool(call.name, call.arguments);
+ JSONObject toolMessage = new JSONObject(true);
+ toolMessage.put("role", "tool");
+ toolMessage.put("tool_call_id", call.id);
+ toolMessage.put("name", call.name == null ? "" : call.name);
+ toolMessage.put("content", result);
+ toolMessages.put(toolMessage);
+ toolCallCounters.put(toolName, Integer.valueOf(usedCalls + 1));
+
+ // Separate compact result list for stream metadata injection.
+ JSONObject toolResult = new JSONObject(true);
+ toolResult.put("tool_call_id", call.id);
+ toolResult.put("name", call.name == null ? "" : call.name);
+ toolResult.put("content", result);
+ toolResults.put(toolResult);
+ }
+
+ // If nothing is executable, leave the message history unchanged.
+ if (toolCallsArray.length() == 0) {
+ return new ToolRoundData(new JSONArray(), new JSONArray());
+ }
+
+ // Add assistant turn that requested tools.
+ JSONObject assistantMessage = new JSONObject(true);
+ assistantMessage.put("role", "assistant");
+ assistantMessage.put("content", assistantContent == null ? "" : assistantContent);
+ assistantMessage.put("tool_calls", toolCallsArray);
+ messages.put(assistantMessage);
+ // Follow with synthetic tool result messages for next model round context.
+ for (int i = 0; i < toolMessages.length(); i++) messages.put(toolMessages.get(i));
+
+ return new ToolRoundData(toolCallsArray, toolResults);
+ } catch (JSONException e) {
+ return null;
+ }
+ }
+
+ /**
+ * Injects tool metadata into one outbound SSE line if the line contains JSON
+ * data.
+ * <p>
+ * Metadata keys:
+ * <ul>
+ * <li>{@code tool-calls}: normalized tool call objects sent in the round.</li>
+ * <li>{@code tool-results}: local tool execution outputs.</li>
+ * </ul>
+ *
+ * @param line outbound line (typically prefixed with {@code data:})
+ * @param roundData round metadata to inject
+ * @return original or augmented line
+ */
+ private static String injectToolMetadata(String line, ToolRoundData roundData) {
+ if (line == null || roundData == null || roundData.toolResults == null || roundData.toolResults.length() == 0) return line;
+ if (!line.startsWith("data:")) return line;
+ int p = line.indexOf('{');
+ if (p <= 0) return line;
+ try {
+ JSONObject j = new JSONObject(new JSONTokener(line.substring(p)));
+ // Do not overwrite if upstream already provided these fields.
+ if (!j.has("tool-calls") && roundData.toolCalls != null) j.put("tool-calls", roundData.toolCalls);
+ if (!j.has("tool-results")) j.put("tool-results", roundData.toolResults);
+ return line.substring(0, p) + j.toString();
+ } catch (JSONException e) {
+ // Non-JSON data lines are forwarded unchanged.
+ return line;
+ }
+ }
+
+ /**
+ * Injects arbitrary JSON metadata into one SSE {@code data: ...} JSON line.
+ * Existing keys are preserved.
+ *
+ * @param line outbound stream line
+ * @param metadata metadata object to inject
+ * @return augmented line, or {@code null} when the line is not a JSON data line
+ */
+ private static String injectJsonMetadata(String line, JSONObject metadata) {
+ if (line == null || metadata == null || metadata.length() == 0) return null;
+ if (!line.startsWith("data:")) return null;
+ int p = line.indexOf('{');
+ if (p <= 0) return null;
+ try {
+ JSONObject j = new JSONObject(new JSONTokener(line.substring(p)));
+ JSONArray names = metadata.names();
+ if (names == null || names.length() == 0) return null;
+ for (int i = 0; i < names.length(); i++) {
+ String name = names.optString(i, null);
+ if (name == null || name.isEmpty()) continue;
+ if (!j.has(name)) j.put(name, metadata.get(name));
+ }
+ return line.substring(0, p) + j.toString();
+ } catch (JSONException e) {
+ return null;
+ }
+ }
+
+ /**
+ * Reads the {@code index} field used to correlate streamed tool-call fragments.
+ *
+ * @param delta one tool-call delta object
+ * @return index as Integer when present and numeric, otherwise {@code null}
+ */
+ private static Integer readToolCallIndex(JSONObject delta) {
+ if (delta == null || !delta.has("index")) return null;
+ Object raw = delta.opt("index");
+ if (raw instanceof Number) return Integer.valueOf(((Number) raw).intValue());
+ return null;
+ }
+
+ /**
+ * Checks whether a merged tool call is executable locally.
+ * <p>
+ * Requirements:
+ * <ul>
+ * <li>non-empty tool/function name</li>
+ * <li>arguments omitted/empty or valid JSON object text</li>
+ * </ul>
+ *
+ * @param call merged tool call candidate
+ * @return true when tool execution should be attempted
+ */
+ private static boolean isExecutableToolCall(ToolCall call) {
+ if (call == null || call.name == null || call.name.trim().isEmpty()) return false;
+ String args = call.arguments;
+ if (args == null || args.isEmpty()) return true;
+ try {
+ new JSONObject(args);
+ return true;
+ } catch (JSONException e) {
+ return false;
+ }
+ }
+
+ /**
+ * Round-local metadata container used during tool loop execution.
+ */
+ private static final class ToolRoundData {
+ /** Tool calls that were executed in the round. */
+ final JSONArray toolCalls;
+ /** Tool outputs corresponding to {@link #toolCalls}. */
+ final JSONArray toolResults;
+
+ /**
+ * Builds a round data object with non-null arrays.
+ *
+ * @param toolCalls tool call descriptors
+ * @param toolResults tool execution outputs
+ */
+ ToolRoundData(JSONArray toolCalls, JSONArray toolResults) {
+ this.toolCalls = toolCalls == null ? new JSONArray() : toolCalls;
+ this.toolResults = toolResults == null ? new JSONArray() : toolResults;
+ }
+ }
+
+ /**
+ * Mutable tool-call assembly object built from streamed deltas.
+ * <p>
+ * Fields may be populated incrementally over multiple chunks.
+ */
+ public static final class ToolCall {
+ /** Tool call id as emitted by model/provider. */
+ public String id;
+ /** Tool call type, usually {@code function}. */
+ public String type;
+ /** Tool/function name. */
+ public String name;
+ /** JSON argument text (can be assembled from fragments). */
+ public String arguments;
+ }
+}
diff --git a/source/net/yacy/ai/ToolHandler.java b/source/net/yacy/ai/ToolHandler.java
new file mode 100644
index 000000000..9c278cf00
--- /dev/null
+++ b/source/net/yacy/ai/ToolHandler.java
@@ -0,0 +1,43 @@
+/**
+ * ToolHandler
+ * Copyright 2026 by Michael Peter Christen
+ * First released 06.02.2026 at https://yacy.net
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program in the file lgpl21.txt
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.ai;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+
+public interface ToolHandler {
+
+ JSONObject definition() throws JSONException;
+
+ String execute(String arguments);
+
+ int maxCallsPerTurn();
+
+ public static String errorJson(String message) {
+ try {
+ JSONObject err = new JSONObject(true);
+ err.put("error", message == null ? "error" : message);
+ return err.toString();
+ } catch (JSONException e) {
+ return "{\"error\":\"" + (message == null ? "error" : message.replace("\"", "'")) + "\"}";
+ }
+ }
+}
diff --git a/source/net/yacy/ai/ToolProvider.java b/source/net/yacy/ai/ToolProvider.java
new file mode 100644
index 000000000..070c6dd45
--- /dev/null
+++ b/source/net/yacy/ai/ToolProvider.java
@@ -0,0 +1,226 @@
+/**
+ * ToolProvider
+ * Copyright 2026 by Michael Peter Christen
+ * First released 06.02.2026 at https://yacy.net
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program in the file lgpl21.txt
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.ai;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import net.yacy.ai.tools.CalculatorTool;
+import net.yacy.ai.tools.ChitChatTool;
+import net.yacy.ai.tools.DateMathTool;
+import net.yacy.ai.tools.DateTimeTool;
+import net.yacy.ai.tools.HttpJsonTool;
+import net.yacy.ai.tools.NumberParserTool;
+import net.yacy.ai.tools.PromptToMermaidTool;
+import net.yacy.ai.tools.SelfReflectTool;
+import net.yacy.ai.tools.TableOpsTool;
+import net.yacy.ai.tools.UnitConverterTool;
+import net.yacy.ai.tools.WebFetchTool;
+import net.yacy.ai.tools.WikipediaLinkCreatorTool;
+
+/**
+ * Central registry and dispatch utility for all built-in YaCy LLM tools.
+ * <p>
+ * Responsibilities:
+ * <ul>
+ * <li>Expose tool JSON schemas into outbound chat-completion requests.</li>
+ * <li>Resolve tool names to handlers.</li>
+ * <li>Execute tool calls by name with raw JSON argument text.</li>
+ * </ul>
+ */
+public final class ToolProvider {
+
+ /**
+ * Ordered list of built-in tool handlers. Order is preserved when exposing
+ * definitions to providers.
+ */
+ private static final List<ToolHandler> TOOLS = Arrays.asList(
+ new DateTimeTool(),
+ new DateMathTool(),
+ new CalculatorTool(),
+ new NumberParserTool(),
+ new UnitConverterTool(),
+ new WebFetchTool(),
+ new HttpJsonTool(),
+ new TableOpsTool(),
+ new SelfReflectTool(),
+ new ChitChatTool(),
+ new PromptToMermaidTool(),
+ new WikipediaLinkCreatorTool()
+ );
+
+ /**
+ * Lookup table for fast runtime dispatch by function/tool name.
+ */
+ private static final Map<String, ToolHandler> TOOL_BY_NAME = buildToolIndex(TOOLS);
+
+ /**
+ * Utility class; not instantiable.
+ */
+ private ToolProvider() {}
+
+ /**
+ * Ensures a chat request body contains all available tool definitions and a
+ * default tool selection mode.
+ * <p>
+ * Existing tool definitions are kept; missing ones are appended.
+ *
+ * @param body request body to mutate
+ */
+ public static void ensureTools(JSONObject body) {
+ if (body == null) return;
+ try {
+ // Create "tools" array lazily when caller did not provide one.
+ JSONArray tools = body.optJSONArray("tools");
+ if (tools == null) {
+ tools = new JSONArray();
+ body.put("tools", tools);
+ }
+ // Merge registry definitions without duplicating by tool name.
+ for (ToolHandler tool : TOOLS) {
+ JSONObject definition = tool.definition();
+ addToolDefinitionIfMissing(tools, definition);
+ }
+ // Providers usually expect explicit tool-choice mode.
+ if (!body.has("tool_choice")) {
+ body.put("tool_choice", "auto");
+ }
+ } catch (JSONException e) {
+ // keep body unchanged if tool schema cannot be injected
+ }
+ }
+
+ /**
+ * Dispatches one tool invocation by name.
+ *
+ * @param name tool/function name
+ * @param arguments raw JSON arguments string
+ * @return tool execution result as JSON string (or error JSON)
+ */
+ public static String executeTool(String name, String arguments) {
+ if (name == null) return errorJson("Invalid tool call");
+ ToolHandler tool = TOOL_BY_NAME.get(name);
+ if (tool == null) return errorJson("Unknown tool: " + name);
+ // Tool implementations are responsible for argument parsing/validation.
+ return tool.execute(arguments);
+ }
+
+ /**
+ * Returns the configured maximum number of calls for a tool within one tool
+ * turn lifecycle.
+ *
+ * @param name tool/function name
+ * @return maximum call count, defaults to 1 for unknown/invalid values
+ */
+ public static int maxCallsPerTurn(String name) {
+ if (name == null) return 1;
+ ToolHandler tool = TOOL_BY_NAME.get(name);
+ if (tool == null) return 1;
+ int max = tool.maxCallsPerTurn();
+ return max <= 0 ? 1 : max;
+ }
+
+ /**
+ * Builds an immutable map from tool name to handler.
+ * Invalid or unnamed definitions are skipped.
+ *
+ * @param tools source handlers
+ * @return unmodifiable name-indexed map
+ */
+ private static Map<String, ToolHandler> buildToolIndex(List<ToolHandler> tools) {
+ Map<String, ToolHandler> byName = new LinkedHashMap<>();
+ for (ToolHandler tool : tools) {
+ if (tool == null) continue;
+ try {
+ String name = extractToolName(tool.definition());
+ if (name == null || name.isEmpty()) continue;
+ byName.put(name, tool);
+ } catch (JSONException e) {
+ // skip invalid tool definition
+ }
+ }
+ return Collections.unmodifiableMap(byName);
+ }
+
+ /**
+ * Appends a tool definition if no existing tool with the same function name
+ * is already present.
+ *
+ * @param tools destination tool array in request body
+ * @param toolDefinition candidate definition
+ */
+ private static void addToolDefinitionIfMissing(JSONArray tools, JSONObject toolDefinition) {
+ if (tools == null || toolDefinition == null) return;
+ try {
+ String name = extractToolName(toolDefinition);
+ if (name == null || name.isEmpty()) return;
+ // Dedupe by logical function name (not by full JSON string equality).
+ for (int i = 0; i < tools.length(); i++) {
+ JSONObject existing = tools.optJSONObject(i);
+ if (existing == null) continue;
+ String existingName = extractToolName(existing);
+ if (name.equals(existingName)) return;
+ }
+ tools.put(toolDefinition);
+ } catch (JSONException e) {
+ // ignore malformed tool schema
+ }
+ }
+
+ /**
+ * Extracts the function name from a tool definition object.
+ *
+ * @param toolDefinition MCP/OpenAI-style tool definition
+ * @return trimmed function name or {@code null}
+ * @throws JSONException if JSON access fails unexpectedly
+ */
+ private static String extractToolName(JSONObject toolDefinition) throws JSONException {
+ if (toolDefinition == null) return null;
+ JSONObject fn = toolDefinition.optJSONObject("function");
+ if (fn == null) return null;
+ String name = fn.optString("name", "");
+ return name == null ? null : name.trim();
+ }
+
+ /**
+ * Builds a minimal error JSON string while avoiding propagation of secondary
+ * JSON construction failures.
+ *
+ * @param message human-readable error message
+ * @return serialized JSON error object
+ */
+ private static String errorJson(String message) {
+ try {
+ JSONObject err = new JSONObject(true);
+ err.put("error", message == null ? "error" : message);
+ return err.toString();
+ } catch (JSONException e) {
+ return "{\"error\":\"" + (message == null ? "error" : message.replace("\"", "'")) + "\"}";
+ }
+ }
+}
diff --git a/source/net/yacy/ai/tools/CalculatorTool.java b/source/net/yacy/ai/tools/CalculatorTool.java
new file mode 100644
index 000000000..4c33d7194
--- /dev/null
+++ b/source/net/yacy/ai/tools/CalculatorTool.java
@@ -0,0 +1,358 @@
+/**
+ * CalculatorTool
+ * Copyright 2026 by Michael Peter Christen
+ * First released 06.02.2026 at https://yacy.net
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program in the file lgpl21.txt
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.ai.tools;
+
+import java.math.BigDecimal;
+import java.math.MathContext;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import net.yacy.ai.ToolHandler;
+
+public class CalculatorTool implements ToolHandler {
+
+ private static final String NAME = "calculator";
+
+ @Override
+ public JSONObject definition() throws JSONException {
+ JSONObject tool = new JSONObject(true);
+ tool.put("type", "function");
+ JSONObject fn = new JSONObject(true);
+ fn.put("name", NAME);
+ fn.put("description", "Evaluate a mathematical formula. Supports operators (+,-,*,/,%,^), constants (pi,e,tau,phi), functions (sqrt,abs,ln,log,sin,cos,tan,asin,acos,atan,min,max,pow,root,...) and scientific notation. Use this as your calculator.");
+ JSONObject params = new JSONObject(true);
+ params.put("type", "object");
+ JSONObject props = new JSONObject(true);
+ JSONObject formula = new JSONObject(true);
+ formula.put("type", "string");
+ formula.put("description", "Formula to evaluate, e.g. (2+3)*4/5");
+ props.put("formula", formula);
+ params.put("properties", props);
+ params.put("required", new JSONArray().put("formula"));
+ fn.put("parameters", params);
+ tool.put("function", fn);
+ return tool;
+ }
+
+
+ public int maxCallsPerTurn() {
+ return 10;
+ }
+
+
+ public String execute(String arguments) {
+ String formula;
+ try {
+ JSONObject obj = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments);
+ formula = obj.optString("formula", "").trim();
+ } catch (JSONException e) {
+ return ToolHandler.errorJson("Invalid arguments JSON");
+ }
+ if (formula == null || formula.isEmpty()) return ToolHandler.errorJson("Missing formula");
+ try {
+ BigDecimal value = new ExpressionParser(formula).parse();
+ JSONObject result = new JSONObject(true);
+ result.put("formula", formula);
+ result.put("result", value.stripTrailingZeros().toPlainString());
+ return result.toString();
+ } catch (Exception e) {
+ return ToolHandler.errorJson("Invalid formula: " + e.getMessage());
+ }
+ }
+
+ private static final class ExpressionParser {
+ private final String expression;
+ private int pos;
+
+ private ExpressionParser(String expression) {
+ this.expression = expression == null ? "" : expression;
+ }
+
+ private BigDecimal parse() {
+ BigDecimal value = parseExpression();
+ skipWs();
+ if (pos != expression.length()) throw new IllegalArgumentException("Unexpected token at " + pos);
+ return value;
+ }
+
+ private BigDecimal parseExpression() {
+ BigDecimal value = parseTerm();
+ while (true) {
+ skipWs();
+ if (eat('+')) value = value.add(parseTerm());
+ else if (eat('-')) value = value.subtract(parseTerm());
+ else return value;
+ }
+ }
+
+ private BigDecimal parseTerm() {
+ BigDecimal value = parseFactor();
+ while (true) {
+ skipWs();
+ if (eat('*')) value = value.multiply(parseFactor(), MathContext.DECIMAL64);
+ else if (eat('/')) value = value.divide(parseFactor(), MathContext.DECIMAL64);
+ else if (eat('%')) value = value.remainder(parseFactor(), MathContext.DECIMAL64);
+ else return value;
+ }
+ }
+
+ private BigDecimal parseFactor() {
+ skipWs();
+ if (eat('+')) return parseFactor();
+ if (eat('-')) return parseFactor().negate();
+ BigDecimal base;
+ if (eat('(')) {
+ base = parseExpression();
+ if (!eat(')')) throw new IllegalArgumentException("Missing ')'");
+ } else if (isIdentifierStart(currentChar())) {
+ String ident = parseIdentifier();
+ if (eat('(')) {
+ List<BigDecimal> args = parseArguments();
+ if (!eat(')')) throw new IllegalArgumentException("Missing ')' after function argument");
+ base = applyFunction(ident, args);
+ } else {
+ base = resolveConstant(ident);
+ }
+ } else {
+ base = parseNumber();
+ }
+ skipWs();
+ if (eat('^')) {
+ BigDecimal exp = parseFactor();
+ try {
+ int intExp = exp.intValueExact();
+ return base.pow(intExp, MathContext.DECIMAL64);
+ } catch (ArithmeticException e) {
+ double b = base.doubleValue();
+ double ex = exp.doubleValue();
+ // Fractional powers are evaluated in double precision.
+ double result = Math.pow(b, ex);
+ if (Double.isNaN(result) || Double.isInfinite(result)) {
+ throw new IllegalArgumentException("Invalid exponentiation result");
+ }
+ return new BigDecimal(result, MathContext.DECIMAL64);
+ }
+ }
+ return base;
+ }
+
+ private BigDecimal parseNumber() {
+ skipWs();
+ int start = pos;
+ boolean dot = false;
+ boolean exponent = false;
+ while (pos < expression.length()) {
+ char c = expression.charAt(pos);
+ if (c >= '0' && c <= '9') {
+ pos++;
+ continue;
+ }
+ if (c == '.' && !dot) {
+ dot = true;
+ pos++;
+ continue;
+ }
+ if ((c == 'e' || c == 'E') && !exponent) {
+ exponent = true;
+ pos++;
+ if (pos < expression.length()) {
+ char sign = expression.charAt(pos);
+ if (sign == '+' || sign == '-') pos++;
+ }
+ continue;
+ }
+ break;
+ }
+ if (start == pos) throw new IllegalArgumentException("Number expected at " + pos);
+ return new BigDecimal(expression.substring(start, pos), MathContext.DECIMAL64);
+ }
+
+ private String parseIdentifier() {
+ skipWs();
+ int start = pos;
+ if (!isIdentifierStart(currentChar())) throw new IllegalArgumentException("Identifier expected at " + pos);
+ pos++;
+ while (pos < expression.length()) {
+ char c = expression.charAt(pos);
+ if (Character.isLetterOrDigit(c) || c == '_' || c == '.') {
+ pos++;
+ continue;
+ }
+ break;
+ }
+ return expression.substring(start, pos).toLowerCase();
+ }
+
+ private List<BigDecimal> parseArguments() {
+ List<BigDecimal> args = new ArrayList<>();
+ skipWs();
+ if (currentChar() == ')') return args;
+ args.add(parseExpression());
+ while (true) {
+ skipWs();
+ if (!eat(',')) break;
+ args.add(parseExpression());
+ }
+ return args;
+ }
+
+ private BigDecimal applyFunction(String name, List<BigDecimal> args) {
+ String fn = shortName(name);
+ if ("sqrt".equals(fn)) return bd(Math.sqrt(nonNegative(arg(args, 0), "sqrt")));
+ if ("cbrt".equals(fn)) return bd(Math.cbrt(arg(args, 0).doubleValue()));
+ if ("abs".equals(fn)) return arg(args, 0).abs();
+ if ("floor".equals(fn)) return bd(Math.floor(arg(args, 0).doubleValue()));
+ if ("ceil".equals(fn)) return bd(Math.ceil(arg(args, 0).doubleValue()));
+ if ("round".equals(fn)) return bd(Math.rint(arg(args, 0).doubleValue()));
+ if ("trunc".equals(fn)) return bd(arg(args, 0).doubleValue() < 0d ? Math.ceil(arg(args, 0).doubleValue()) : Math.floor(arg(args, 0).doubleValue()));
+
+ if ("exp".equals(fn)) return bd(Math.exp(arg(args, 0).doubleValue()));
+ if ("ln".equals(fn)) return bd(Math.log(positive(arg(args, 0), "ln")));
+ if ("log".equals(fn)) {
+ if (args.size() == 1) return bd(Math.log10(positive(arg(args, 0), "log")));
+ double value = positive(arg(args, 0), "log");
+ double base = positive(arg(args, 1), "log base");
+ if (base == 1d) throw new IllegalArgumentException("log base must not be 1");
+ return bd(Math.log(value) / Math.log(base));
+ }
+ if ("log10".equals(fn)) return bd(Math.log10(positive(arg(args, 0), "log10")));
+ if ("log2".equals(fn)) return bd(Math.log(positive(arg(args, 0), "log2")) / Math.log(2d));
+
+ if ("sin".equals(fn)) return bd(Math.sin(arg(args, 0).doubleValue()));
+ if ("cos".equals(fn)) return bd(Math.cos(arg(args, 0).doubleValue()));
+ if ("tan".equals(fn)) return bd(Math.tan(arg(args, 0).doubleValue()));
+ if ("asin".equals(fn)) return bd(Math.asin(arg(args, 0).doubleValue()));
+ if ("acos".equals(fn)) return bd(Math.acos(arg(args, 0).doubleValue()));
+ if ("atan".equals(fn)) return bd(Math.atan(arg(args, 0).doubleValue()));
+ if ("atan2".equals(fn)) return bd(Math.atan2(arg(args, 0).doubleValue(), arg(args, 1).doubleValue()));
+ if ("sinh".equals(fn)) return bd(Math.sinh(arg(args, 0).doubleValue()));
+ if ("cosh".equals(fn)) return bd(Math.cosh(arg(args, 0).doubleValue()));
+ if ("tanh".equals(fn)) return bd(Math.tanh(arg(args, 0).doubleValue()));
+
+ if ("deg".equals(fn) || "degrees".equals(fn) || "todeg".equals(fn)) return bd(Math.toDegrees(arg(args, 0).doubleValue()));
+ if ("rad".equals(fn) || "radians".equals(fn) || "torad".equals(fn)) return bd(Math.toRadians(arg(args, 0).doubleValue()));
+
+ if ("pow".equals(fn)) return bd(Math.pow(arg(args, 0).doubleValue(), arg(args, 1).doubleValue()));
+ if ("root".equals(fn)) {
+ double n = arg(args, 1).doubleValue();
+ if (n == 0d) throw new IllegalArgumentException("root degree must not be 0");
+ return bd(Math.pow(arg(args, 0).doubleValue(), 1d / n));
+ }
+ if ("mod".equals(fn)) return arg(args, 0).remainder(arg(args, 1), MathContext.DECIMAL64);
+
+ if ("min".equals(fn)) return min(args);
+ if ("max".equals(fn)) return max(args);
+ if ("avg".equals(fn) || "mean".equals(fn)) return mean(args);
+ if ("sum".equals(fn)) return sum(args);
+ throw new IllegalArgumentException("Unknown function: " + name);
+ }
+
+ private BigDecimal resolveConstant(String name) {
+ String n = shortName(name);
+ if ("pi".equals(n)) return new BigDecimal(Math.PI, MathContext.DECIMAL64);
+ if ("e".equals(n)) return new BigDecimal(Math.E, MathContext.DECIMAL64);
+ if ("tau".equals(n)) return new BigDecimal(2d * Math.PI, MathContext.DECIMAL64);
+ if ("phi".equals(n)) return new BigDecimal((1d + Math.sqrt(5d)) / 2d, MathContext.DECIMAL64);
+ throw new IllegalArgumentException("Unknown identifier: " + name);
+ }
+
+ private String shortName(String name) {
+ if (name == null || name.isEmpty()) return "";
+ int p = name.lastIndexOf('.');
+ if (p < 0 || p >= name.length() - 1) return name;
+ return name.substring(p + 1);
+ }
+
+ private BigDecimal arg(List<BigDecimal> args, int index) {
+ if (args == null || index < 0 || index >= args.size()) throw new IllegalArgumentException("Missing function argument at position " + (index + 1));
+ return args.get(index);
+ }
+
+ private double positive(BigDecimal value, String label) {
+ double v = value.doubleValue();
+ if (v <= 0d) throw new IllegalArgumentException(label + " argument must be > 0");
+ return v;
+ }
+
+ private double nonNegative(BigDecimal value, String label) {
+ double v = value.doubleValue();
+ if (v < 0d) throw new IllegalArgumentException(label + " argument must be >= 0");
+ return v;
+ }
+
+ private BigDecimal bd(double value) {
+ if (Double.isNaN(value) || Double.isInfinite(value)) {
+ throw new IllegalArgumentException("Invalid numeric result");
+ }
+ return new BigDecimal(value, MathContext.DECIMAL64);
+ }
+
+ private BigDecimal min(List<BigDecimal> args) {
+ if (args == null || args.isEmpty()) throw new IllegalArgumentException("min requires at least one argument");
+ BigDecimal m = args.get(0);
+ for (int i = 1; i < args.size(); i++) if (args.get(i).compareTo(m) < 0) m = args.get(i);
+ return m;
+ }
+
+ private BigDecimal max(List<BigDecimal> args) {
+ if (args == null || args.isEmpty()) throw new IllegalArgumentException("max requires at least one argument");
+ BigDecimal m = args.get(0);
+ for (int i = 1; i < args.size(); i++) if (args.get(i).compareTo(m) > 0) m = args.get(i);
+ return m;
+ }
+
+ private BigDecimal sum(List<BigDecimal> args) {
+ if (args == null || args.isEmpty()) throw new IllegalArgumentException("sum requires at least one argument");
+ BigDecimal s = BigDecimal.ZERO;
+ for (BigDecimal v : args) s = s.add(v, MathContext.DECIMAL64);
+ return s;
+ }
+
+ private BigDecimal mean(List<BigDecimal> args) {
+ return sum(args).divide(new BigDecimal(args.size()), MathContext.DECIMAL64);
+ }
+
+ private char currentChar() {
+ if (pos < 0 || pos >= expression.length()) return '\0';
+ return expression.charAt(pos);
+ }
+
+ private boolean isIdentifierStart(char c) {
+ return Character.isLetter(c) || c == '_';
+ }
+
+ private void skipWs() {
+ while (pos < expression.length() && Character.isWhitespace(expression.charAt(pos))) pos++;
+ }
+
+ private boolean eat(char c) {
+ if (pos < expression.length() && expression.charAt(pos) == c) {
+ pos++;
+ return true;
+ }
+ return false;
+ }
+ }
+}
diff --git a/source/net/yacy/ai/tools/ChitChatTool.java b/source/net/yacy/ai/tools/ChitChatTool.java
new file mode 100644
index 000000000..8c0326068
--- /dev/null
+++ b/source/net/yacy/ai/tools/ChitChatTool.java
@@ -0,0 +1,134 @@
+/**
+ * ChitChatTool
+ * Copyright 2026 by Michael Peter Christen
+ * First released 06.02.2026 at https://yacy.net
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program in the file lgpl21.txt
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.ai.tools;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ThreadLocalRandom;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import net.yacy.ai.ToolHandler;
+
+public class ChitChatTool implements ToolHandler {
+
+ private static final String NAME = "chitchat";
+ private static final int DEFAULT_FACT_COUNT = 3;
+ private static final int MAX_FACT_COUNT = 5;
+
+ @Override
+ public JSONObject definition() throws JSONException {
+ JSONObject tool = new JSONObject(true);
+ tool.put("type", "function");
+ JSONObject fn = new JSONObject(true);
+ fn.put("name", NAME);
+ fn.put("description", "Provide friendly chitchat. Call me when the user just tests the chat, says hello, is unsure with the prompt or in case of profanity.");
+
+ JSONObject params = new JSONObject(true);
+ params.put("type", "object");
+ JSONObject props = new JSONObject(true);
+
+ JSONObject tone = new JSONObject(true);
+ tone.put("type", "string");
+ tone.put("description", "Optional tone hint, e.g. friendly, concise, playful.");
+ props.put("tone", tone);
+
+ JSONObject factCount = new JSONObject(true);
+ factCount.put("type", "integer");
+ factCount.put("description", "How many ability facts to include (1-5, default 3).");
+ props.put("fact_count", factCount);
+
+ params.put("properties", props);
+ params.put("required", new JSONArray());
+ fn.put("parameters", params);
+ tool.put("function", fn);
+ return tool;
+ }
+
+
+ public int maxCallsPerTurn() {
+ return 1;
+ }
+
+
+ public String execute(String arguments) {
+ String tone = "friendly";
+ int factCount = DEFAULT_FACT_COUNT;
+ if (arguments != null && !arguments.isEmpty()) {
+ try {
+ JSONObject args = new JSONObject(arguments);
+ tone = args.optString("tone", tone).trim();
+ factCount = args.optInt("fact_count", factCount);
+ } catch (JSONException e) {
+ return ToolHandler.errorJson("Invalid arguments JSON");
+ }
+ }
+
+ int count = clamp(factCount, 1, MAX_FACT_COUNT);
+ List<String> selected = pickFacts(count);
+ String message = buildMessage(tone, selected);
+
+ try {
+ JSONObject result = new JSONObject(true);
+ result.put("tool", NAME);
+ result.put("tone", tone);
+ result.put("message", message);
+ result.put("facts", new JSONArray(selected));
+ return result.toString();
+ } catch (JSONException e) {
+ return ToolHandler.errorJson("Failed to build chitchat response");
+ }
+ }
+
+ private static List<String> pickFacts(int count) {
+ List<String> facts = new ArrayList<>(SelfReflectTool.facts());
+ if (facts.isEmpty()) return facts;
+ Collections.shuffle(facts, ThreadLocalRandom.current());
+ int n = Math.min(count, facts.size());
+ return new ArrayList<>(facts.subList(0, n));
+ }
+
+ private static String buildMessage(String tone, List<String> facts) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Happy to chat. ");
+ sb.append(SelfReflectTool.purpose());
+ if (tone != null && !tone.isEmpty()) {
+ sb.append(" Tone requested: ").append(tone).append(". ");
+ } else {
+ sb.append(" ");
+ }
+ sb.append("Here are a few things I can do: ");
+ for (int i = 0; i < facts.size(); i++) {
+ if (i > 0) sb.append(" ");
+ sb.append(i + 1).append(") ").append(facts.get(i));
+ }
+ return sb.toString();
+ }
+
+ private static int clamp(int value, int min, int max) {
+ if (value < min) return min;
+ if (value > max) return max;
+ return value;
+ }
+}
diff --git a/source/net/yacy/ai/tools/DateMathTool.java b/source/net/yacy/ai/tools/DateMathTool.java
new file mode 100644
index 000000000..bcd3e333b
--- /dev/null
+++ b/source/net/yacy/ai/tools/DateMathTool.java
@@ -0,0 +1,214 @@
+/**
+ * DateMathTool
+ * Copyright 2026 by Michael Peter Christen
+ * First released 06.02.2026 at https://yacy.net
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program in the file lgpl21.txt
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.ai.tools;
+
+import java.time.DayOfWeek;
+import java.time.Duration;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.YearMonth;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+import java.time.temporal.ChronoUnit;
+import java.util.Locale;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import net.yacy.ai.ToolHandler;
+
+public class DateMathTool implements ToolHandler {
+
+ private static final String NAME = "date_math";
+
+ @Override
+ public JSONObject definition() throws JSONException {
+ JSONObject tool = new JSONObject(true);
+ tool.put("type", "function");
+ JSONObject fn = new JSONObject(true);
+ fn.put("name", NAME);
+ fn.put("description", "Perform deterministic date/time operations: add, subtract, diff, start_of, end_of.");
+
+ JSONObject params = new JSONObject(true);
+ params.put("type", "object");
+ JSONObject props = new JSONObject(true);
+ props.put("operation", new JSONObject(true).put("type", "string").put("description", "add|subtract|diff|start_of|end_of"));
+ props.put("datetime", new JSONObject(true).put("type", "string").put("description", "Base datetime/date in ISO format."));
+ props.put("other_datetime", new JSONObject(true).put("type", "string").put("description", "Second datetime for diff."));
+ props.put("amount", new JSONObject(true).put("type", "integer").put("description", "Amount for add/subtract."));
+ props.put("unit", new JSONObject(true).put("type", "string").put("description", "minute|hour|day|week|month|year"));
+ props.put("period", new JSONObject(true).put("type", "string").put("description", "day|week|month|year (for start_of/end_of)."));
+ props.put("timezone", new JSONObject(true).put("type", "string").put("description", "IANA zone; default system zone."));
+ params.put("properties", props);
+ params.put("required", new JSONArray().put("operation"));
+ fn.put("parameters", params);
+ tool.put("function", fn);
+ return tool;
+ }
+
+
+ public int maxCallsPerTurn() {
+ return 1;
+ }
+
+
+ public String execute(String arguments) {
+ final JSONObject args;
+ try {
+ args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments);
+ } catch (JSONException e) {
+ return ToolHandler.errorJson("Invalid arguments JSON");
+ }
+
+ String op = args.optString("operation", "").trim().toLowerCase(Locale.ROOT);
+ if (op.isEmpty()) return ToolHandler.errorJson("Missing operation");
+
+ ZoneId zone = parseZone(args.optString("timezone", ""));
+ if (zone == null) return ToolHandler.errorJson("Invalid timezone");
+ ZonedDateTime base = parseDateTime(args.optString("datetime", ""), zone);
+ if (base == null) base = ZonedDateTime.now(zone);
+
+ try {
+ JSONObject out = new JSONObject(true);
+ out.put("tool", NAME);
+ out.put("operation", op);
+ out.put("timezone", zone.getId());
+ out.put("input_datetime", base.toString());
+
+ if ("add".equals(op) || "subtract".equals(op)) {
+ int amount = args.optInt("amount", Integer.MIN_VALUE);
+ if (amount == Integer.MIN_VALUE) return ToolHandler.errorJson("Missing amount");
+ String unit = normalizeUnit(args.optString("unit", ""));
+ if (unit == null) return ToolHandler.errorJson("Invalid unit");
+ long signed = "subtract".equals(op) ? -((long) amount) : amount;
+ ZonedDateTime result = applyAdd(base, signed, unit);
+ out.put("result_datetime", result.toString());
+ out.put("result_epoch_ms", result.toInstant().toEpochMilli());
+ return out.toString();
+ }
+
+ if ("diff".equals(op)) {
+ ZonedDateTime other = parseDateTime(args.optString("other_datetime", ""), zone);
+ if (other == null) return ToolHandler.errorJson("Missing or invalid other_datetime");
+ Duration d = Duration.between(base.toInstant(), other.toInstant());
+ out.put("other_datetime", other.toString());
+ out.put("difference_seconds", d.getSeconds());
+ out.put("difference_minutes", d.toMinutes());
+ out.put("difference_hours", d.toHours());
+ out.put("difference_days", d.toDays());
+ return out.toString();
+ }
+
+ if ("start_of".equals(op) || "end_of".equals(op)) {
+ String period = args.optString("period", "").trim().toLowerCase(Locale.ROOT);
+ if (period.isEmpty()) return ToolHandler.errorJson("Missing period");
+ ZonedDateTime result = "start_of".equals(op) ? startOf(base, period) : endOf(base, period);
+ out.put("period", period);
+ out.put("result_datetime", result.toString());
+ out.put("result_epoch_ms", result.toInstant().toEpochMilli());
+ return out.toString();
+ }
+ return ToolHandler.errorJson("Unsupported operation: " + op);
+ } catch (IllegalArgumentException | JSONException e) {
+ return ToolHandler.errorJson(e.getMessage());
+ }
+ }
+
+ private static ZoneId parseZone(String timezone) {
+ if (timezone == null || timezone.trim().isEmpty()) return ZoneId.systemDefault();
+ try {
+ return ZoneId.of(timezone.trim());
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ private static ZonedDateTime parseDateTime(String value, ZoneId zone) {
+ if (value == null || value.trim().isEmpty()) return null;
+ String v = value.trim();
+ try {
+ return ZonedDateTime.parse(v);
+ } catch (DateTimeParseException e) {}
+ try {
+ LocalDateTime ldt = LocalDateTime.parse(v, DateTimeFormatter.ISO_LOCAL_DATE_TIME);
+ return ldt.atZone(zone);
+ } catch (DateTimeParseException e) {}
+ try {
+ LocalDate ld = LocalDate.parse(v, DateTimeFormatter.ISO_LOCAL_DATE);
+ return ld.atStartOfDay(zone);
+ } catch (DateTimeParseException e) {}
+ return null;
+ }
+
+ private static String normalizeUnit(String unit) {
+ if (unit == null) return null;
+ String u = unit.trim().toLowerCase(Locale.ROOT);
+ if ("minutes".equals(u)) return "minute";
+ if ("hours".equals(u)) return "hour";
+ if ("days".equals(u)) return "day";
+ if ("weeks".equals(u)) return "week";
+ if ("months".equals(u)) return "month";
+ if ("years".equals(u)) return "year";
+ if ("minute".equals(u) || "hour".equals(u) || "day".equals(u) || "week".equals(u) || "month".equals(u) || "year".equals(u)) return u;
+ return null;
+ }
+
+ private static ZonedDateTime applyAdd(ZonedDateTime dt, long amount, String unit) {
+ if ("minute".equals(unit)) return dt.plus(amount, ChronoUnit.MINUTES);
+ if ("hour".equals(unit)) return dt.plus(amount, ChronoUnit.HOURS);
+ if ("day".equals(unit)) return dt.plus(amount, ChronoUnit.DAYS);
+ if ("week".equals(unit)) return dt.plus(amount, ChronoUnit.WEEKS);
+ if ("month".equals(unit)) return dt.plus(amount, ChronoUnit.MONTHS);
+ if ("year".equals(unit)) return dt.plus(amount, ChronoUnit.YEARS);
+ throw new IllegalArgumentException("Invalid unit: " + unit);
+ }
+
+ private static ZonedDateTime startOf(ZonedDateTime dt, String period) {
+ if ("day".equals(period)) {
+ return dt.toLocalDate().atStartOfDay(dt.getZone());
+ }
+ if ("week".equals(period)) {
+ LocalDate date = dt.toLocalDate();
+ while (date.getDayOfWeek() != DayOfWeek.MONDAY) date = date.minusDays(1);
+ return date.atStartOfDay(dt.getZone());
+ }
+ if ("month".equals(period)) {
+ LocalDate first = YearMonth.from(dt).atDay(1);
+ return first.atStartOfDay(dt.getZone());
+ }
+ if ("year".equals(period)) {
+ LocalDate first = LocalDate.of(dt.getYear(), 1, 1);
+ return first.atStartOfDay(dt.getZone());
+ }
+ throw new IllegalArgumentException("Invalid period: " + period);
+ }
+
+ private static ZonedDateTime endOf(ZonedDateTime dt, String period) {
+ if ("day".equals(period)) return startOf(dt, "day").plusDays(1).minusNanos(1);
+ if ("week".equals(period)) return startOf(dt, "week").plusWeeks(1).minusNanos(1);
+ if ("month".equals(period)) return startOf(dt, "month").plusMonths(1).minusNanos(1);
+ if ("year".equals(period)) return startOf(dt, "year").plusYears(1).minusNanos(1);
+ throw new IllegalArgumentException("Invalid period: " + period);
+ }
+}
diff --git a/source/net/yacy/ai/tools/DateTimeTool.java b/source/net/yacy/ai/tools/DateTimeTool.java
new file mode 100644
index 000000000..d9ba10d59
--- /dev/null
+++ b/source/net/yacy/ai/tools/DateTimeTool.java
@@ -0,0 +1,112 @@
+/**
+ * DateTimeTool
+ * Copyright 2026 by Michael Peter Christen
+ * First released 06.02.2026 at https://yacy.net
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program in the file lgpl21.txt
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.ai.tools;
+
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import net.yacy.ai.ToolHandler;
+
+public class DateTimeTool implements ToolHandler {
+
+ private static final String NAME = "datetime";
+ private static final DateTimeFormatter DATETIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+
+ @Override
+ public JSONObject definition() throws JSONException {
+ JSONObject tool = new JSONObject(true);
+ tool.put("type", "function");
+ JSONObject fn = new JSONObject(true);
+ fn.put("name", NAME);
+ fn.put("description", "Get the current date and time. Optionally specify an IANA timezone.");
+ JSONObject params = new JSONObject(true);
+ params.put("type", "object");
+ JSONObject props = new JSONObject(true);
+ JSONObject tz = new JSONObject(true);
+ tz.put("type", "string");
+ tz.put("description", "IANA timezone name like America/New_York. Defaults to local time zone.");
+ props.put("timezone", tz);
+ params.put("properties", props);
+ params.put("required", new JSONArray());
+ fn.put("parameters", params);
+ tool.put("function", fn);
+ return tool;
+ }
+
+
+ public int maxCallsPerTurn() {
+ return 1;
+ }
+
+
+ public String execute(String arguments) {
+ String timezone = null;
+ if (arguments != null && !arguments.isEmpty()) {
+ try {
+ JSONObject obj = new JSONObject(arguments);
+ timezone = obj.optString("timezone", null);
+ } catch (JSONException e) {
+ return ToolHandler.errorJson("Invalid arguments JSON");
+ }
+ }
+
+ ZoneId localZone = ZoneId.systemDefault();
+ ZoneId zone = localZone;
+ String note = null;
+ if (timezone != null && !timezone.isEmpty()) {
+ try {
+ zone = ZoneId.of(timezone);
+ } catch (Exception e) {
+ zone = localZone;
+ note = "Invalid timezone \"" + timezone + "\". Used local timezone \"" + localZone.getId() + "\".";
+ }
+ }
+
+ ZonedDateTime now = ZonedDateTime.now(zone);
+ try {
+ JSONObject result = new JSONObject(true);
+ result.put("datetime", now.format(DATETIME_FORMAT));
+ result.put("timezone", zone.getId());
+ result.put("time_of_day", classifyTimeOfDay(now.getHour()));
+ result.put("utc_offset_minutes", now.getOffset().getTotalSeconds() / 60);
+ result.put("utc_datetime", Instant.now().toString());
+ result.put("epoch_ms", System.currentTimeMillis());
+ if (note != null) result.put("note", note);
+ return result.toString();
+ } catch (JSONException e) {
+ return ToolHandler.errorJson("Failed to build datetime response");
+ }
+ }
+
+ private static String classifyTimeOfDay(int hour) {
+ if (hour < 0 || hour > 23) return "night";
+ if (hour < 6) return "night";
+ if (hour < 12) return "morning";
+ if (hour < 18) return "afternoon";
+ return "evening";
+ }
+}
diff --git a/source/net/yacy/ai/tools/HttpJsonTool.java b/source/net/yacy/ai/tools/HttpJsonTool.java
new file mode 100644
index 000000000..16251b3e5
--- /dev/null
+++ b/source/net/yacy/ai/tools/HttpJsonTool.java
@@ -0,0 +1,244 @@
+/**
+ * HttpJsonTool
+ * Copyright 2026 by Michael Peter Christen
+ * First released 06.02.2026 at https://yacy.net
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program in the file lgpl21.txt
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.ai.tools;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONTokener;
+
+import net.yacy.ai.ToolHandler;
+import net.yacy.cora.document.id.DigestURL;
+
+public class HttpJsonTool implements ToolHandler {
+
+ private static final String NAME = "http_json";
+ private static final Set<String> ALLOWED_METHODS = new HashSet<>(Arrays.asList("GET", "POST", "PUT", "PATCH", "DELETE"));
+ private static final Set<String> BLOCKED_HEADERS = new HashSet<>(Arrays.asList("host", "content-length", "connection", "transfer-encoding"));
+ private static final int DEFAULT_TIMEOUT_MS = 15000;
+ private static final int MAX_TIMEOUT_MS = 60000;
+ private static final int DEFAULT_MAX_BYTES = 1_000_000;
+ private static final int MAX_ALLOWED_BYTES = 2_000_000;
+
+ @Override
+ public JSONObject definition() throws JSONException {
+ JSONObject tool = new JSONObject(true);
+ tool.put("type", "function");
+ JSONObject fn = new JSONObject(true);
+ fn.put("name", NAME);
+ fn.put("description", "Fetch an HTTP endpoint and return parsed JSON response.");
+
+ JSONObject params = new JSONObject(true);
+ params.put("type", "object");
+ JSONObject props = new JSONObject(true);
+
+ JSONObject url = new JSONObject(true);
+ url.put("type", "string");
+ url.put("description", "Absolute http or https URL.");
+ props.put("url", url);
+
+ JSONObject method = new JSONObject(true);
+ method.put("type", "string");
+ method.put("description", "HTTP method. Allowed: GET, POST, PUT, PATCH, DELETE.");
+ props.put("method", method);
+
+ JSONObject headers = new JSONObject(true);
+ headers.put("type", "object");
+ headers.put("description", "Optional request headers (string values).");
+ props.put("headers", headers);
+
+ JSONObject body = new JSONObject(true);
+ body.put("description", "Optional JSON body as object/array/string.");
+ props.put("body", body);
+
+ JSONObject timeout = new JSONObject(true);
+ timeout.put("type", "integer");
+ timeout.put("description", "Timeout in milliseconds (max 60000).");
+ props.put("timeout_ms", timeout);
+
+ JSONObject maxBytes = new JSONObject(true);
+ maxBytes.put("type", "integer");
+ maxBytes.put("description", "Maximum response size in bytes (max 2000000).");
+ props.put("max_bytes", maxBytes);
+
+ params.put("properties", props);
+ params.put("required", new JSONArray().put("url"));
+ fn.put("parameters", params);
+ tool.put("function", fn);
+ return tool;
+ }
+
+
+ public int maxCallsPerTurn() {
+ return 1;
+ }
+
+
+ public String execute(String arguments) {
+ final JSONObject args;
+ try {
+ args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments);
+ } catch (JSONException e) {
+ return ToolHandler.errorJson("Invalid arguments JSON. don't try again");
+ }
+
+ final String urlRaw = args.optString("url", "").trim();
+ if (urlRaw.isEmpty()) return ToolHandler.errorJson("Missing url. don't try again");
+
+ final DigestURL digestUrl;
+ try {
+ digestUrl = new DigestURL(urlRaw);
+ } catch (Exception e) {
+ return ToolHandler.errorJson("Invalid URL. don't try again");
+ }
+ final String protocol = digestUrl.getProtocol();
+ if (!"http".equalsIgnoreCase(protocol) && !"https".equalsIgnoreCase(protocol)) {
+ return ToolHandler.errorJson("Only http/https URLs are allowed. don't try again");
+ }
+
+ final String method = args.optString("method", "GET").trim().toUpperCase();
+ if (!ALLOWED_METHODS.contains(method)) {
+ return ToolHandler.errorJson("Unsupported method " + method + ". don't try again");
+ }
+ final int timeoutMs = clampPositive(args.optInt("timeout_ms", DEFAULT_TIMEOUT_MS), DEFAULT_TIMEOUT_MS, MAX_TIMEOUT_MS);
+ final int maxBytes = clampPositive(args.optInt("max_bytes", DEFAULT_MAX_BYTES), DEFAULT_MAX_BYTES, MAX_ALLOWED_BYTES);
+ final Object body = args.opt("body");
+ final JSONObject headers = args.optJSONObject("headers");
+
+ HttpURLConnection conn = null;
+ try {
+ final URL url = new URI(digestUrl.toNormalform(true)).toURL();
+ conn = (HttpURLConnection) url.openConnection();
+ conn.setInstanceFollowRedirects(true);
+ conn.setRequestMethod(method);
+ conn.setConnectTimeout(timeoutMs);
+ conn.setReadTimeout(timeoutMs);
+ conn.setRequestProperty("Accept", "application/json");
+ applyHeaders(conn, headers);
+
+ if (body != null && !"GET".equals(method)) {
+ conn.setDoOutput(true);
+ if (conn.getRequestProperty("Content-Type") == null) {
+ conn.setRequestProperty("Content-Type", "application/json");
+ }
+ byte[] payload = encodeBody(body);
+ try (OutputStream os = conn.getOutputStream()) {
+ os.write(payload);
+ }
+ }
+
+ int status = conn.getResponseCode();
+ String contentType = conn.getHeaderField("Content-Type");
+ if (contentType == null) contentType = "";
+ final InputStream stream = status >= 400 ? conn.getErrorStream() : conn.getInputStream();
+ if (stream == null) {
+ return ToolHandler.errorJson("Empty response body. don't try again");
+ }
+ final String text;
+ try (InputStream in = stream) {
+ text = new String(readLimited(in, maxBytes), StandardCharsets.UTF_8);
+ }
+ if (text.trim().isEmpty()) {
+ return ToolHandler.errorJson("Empty response body. don't try again");
+ }
+
+ final Object parsed;
+ try {
+ parsed = new JSONTokener(text).nextValue();
+ } catch (JSONException e) {
+ return ToolHandler.errorJson("Response is not valid JSON. don't try again");
+ }
+ if (!(parsed instanceof JSONObject) && !(parsed instanceof JSONArray)) {
+ return ToolHandler.errorJson("Response is not JSON object/array. don't try again");
+ }
+
+ JSONObject result = new JSONObject(true);
+ result.put("url", digestUrl.toNormalform(true));
+ result.put("method", method);
+ result.put("status", status);
+ result.put("content_type", contentType);
+ result.put("json", parsed);
+ return result.toString();
+ } catch (IOException e) {
+ return ToolHandler.errorJson("Fetch error: " + e.getMessage() + ". don't try again");
+ } catch (URISyntaxException e) {
+ return ToolHandler.errorJson("Invalid URL syntax. don't try again");
+ } catch (JSONException e) {
+ return ToolHandler.errorJson("Failed to build tool response. don't try again");
+ } finally {
+ if (conn != null) conn.disconnect();
+ }
+ }
+
+ private static void applyHeaders(HttpURLConnection conn, JSONObject headers) {
+ if (conn == null || headers == null) return;
+ for (String key : headers.keySet()) {
+ if (key == null) continue;
+ final String normalized = key.trim();
+ if (normalized.isEmpty()) continue;
+ if (BLOCKED_HEADERS.contains(normalized.toLowerCase())) continue;
+ final String value = headers.optString(key, "");
+ conn.setRequestProperty(normalized, value);
+ }
+ }
+
+ private static byte[] encodeBody(Object body) {
+ if (body == null) return new byte[0];
+ if (body instanceof JSONObject) return ((JSONObject) body).toString().getBytes(StandardCharsets.UTF_8);
+ if (body instanceof JSONArray) return ((JSONArray) body).toString().getBytes(StandardCharsets.UTF_8);
+ if (body instanceof String) return ((String) body).getBytes(StandardCharsets.UTF_8);
+ return String.valueOf(body).getBytes(StandardCharsets.UTF_8);
+ }
+
+ private static byte[] readLimited(InputStream in, int maxBytes) throws IOException {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ byte[] buffer = new byte[4096];
+ int total = 0;
+ int read;
+ while ((read = in.read(buffer)) != -1) {
+ total += read;
+ if (total > maxBytes) {
+ throw new IOException("Response too large");
+ }
+ out.write(buffer, 0, read);
+ }
+ return out.toByteArray();
+ }
+
+ private static int clampPositive(int value, int defaultValue, int maxValue) {
+ if (value <= 0) return defaultValue;
+ if (value > maxValue) return maxValue;
+ return value;
+ }
+}
diff --git a/source/net/yacy/ai/tools/NumberParserTool.java b/source/net/yacy/ai/tools/NumberParserTool.java
new file mode 100644
index 000000000..9880de5e5
--- /dev/null
+++ b/source/net/yacy/ai/tools/NumberParserTool.java
@@ -0,0 +1,403 @@
+/**
+ * NumberParserTool
+ * Copyright 2026 by Michael Peter Christen
+ * First released 07.02.2026 at https://yacy.net
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program in the file lgpl21.txt
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.ai.tools;
+
+import java.math.BigDecimal;
+import java.text.Normalizer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import net.yacy.ai.ToolHandler;
+
+public class NumberParserTool implements ToolHandler {
+
+ private static final String NAME = "number_parser";
+
+ private enum Lang {
+ DE, EN, FR, ES, IT
+ }
+
+ private static final class Lexicon {
+ final Map<String, Integer> small = new HashMap<>();
+ final Map<String, Integer> scales = new HashMap<>();
+ final String decimalMarker;
+ Lexicon(String decimalMarker) {
+ this.decimalMarker = decimalMarker;
+ }
+ }
+
+ private static final Map<Lang, Lexicon> LEX = buildLexicons();
+
+ @Override
+ public JSONObject definition() throws JSONException {
+ JSONObject tool = new JSONObject(true);
+ tool.put("type", "function");
+ JSONObject fn = new JSONObject(true);
+ fn.put("name", NAME);
+ fn.put("description", "Parse written number words into numeric values. Supports de/en/fr/es/it.");
+
+ JSONObject params = new JSONObject(true);
+ params.put("type", "object");
+ JSONObject props = new JSONObject(true);
+
+ props.put("text", new JSONObject(true)
+ .put("type", "string")
+ .put("description", "Number words, e.g. einundzwanzig, five hundred seven, vingt-et-un."));
+
+ props.put("language", new JSONObject(true)
+ .put("type", "string")
+ .put("description", "Optional: de|en|fr|es|it|auto (default auto)."));
+
+ props.put("strict", new JSONObject(true)
+ .put("type", "boolean")
+ .put("description", "If true (default), all tokens must be recognized as number words."));
+
+ params.put("properties", props);
+ params.put("required", new JSONArray().put("text"));
+ fn.put("parameters", params);
+ tool.put("function", fn);
+ return tool;
+ }
+
+ @Override
+ public int maxCallsPerTurn() {
+ return 10;
+ }
+
+ @Override
+ public String execute(String arguments) {
+ final JSONObject args;
+ try {
+ args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments);
+ } catch (JSONException e) {
+ return ToolHandler.errorJson("Invalid arguments JSON");
+ }
+
+ final String text = args.optString("text", "").trim();
+ if (text.isEmpty()) return ToolHandler.errorJson("Missing text");
+
+ final String language = args.optString("language", "auto").trim().toLowerCase(Locale.ROOT);
+ final boolean strict = args.optBoolean("strict", true);
+
+ ParseResult best = null;
+ Lang bestLang = null;
+
+ if (!"auto".equals(language)) {
+ Lang lang = parseLang(language);
+ if (lang == null) return ToolHandler.errorJson("Invalid language. Use de|en|fr|es|it|auto");
+ ParseResult r = parseWithLang(text, lang, strict);
+ if (r == null) return ToolHandler.errorJson("Could not parse number words");
+ best = r;
+ bestLang = lang;
+ } else {
+ for (Lang lang : Lang.values()) {
+ ParseResult r = parseWithLang(text, lang, strict);
+ if (r == null) continue;
+ if (best == null || r.score > best.score) {
+ best = r;
+ bestLang = lang;
+ }
+ }
+ if (best == null) return ToolHandler.errorJson("Could not parse number words in supported languages");
+ }
+
+ try {
+ JSONObject out = new JSONObject(true);
+ out.put("tool", NAME);
+ out.put("text", text);
+ out.put("language", bestLang.name().toLowerCase(Locale.ROOT));
+ out.put("value", best.value.stripTrailingZeros().toPlainString());
+ return out.toString();
+ } catch (JSONException e) {
+ return ToolHandler.errorJson("Failed to build number parser response");
+ }
+ }
+
+ private static Lang parseLang(String code) {
+ if ("de".equals(code)) return Lang.DE;
+ if ("en".equals(code)) return Lang.EN;
+ if ("fr".equals(code)) return Lang.FR;
+ if ("es".equals(code)) return Lang.ES;
+ if ("it".equals(code)) return Lang.IT;
+ return null;
+ }
+
+ private static final class ParseResult {
+ final BigDecimal value;
+ final int score;
+ ParseResult(BigDecimal value, int score) {
+ this.value = value;
+ this.score = score;
+ }
+ }
+
+ private static ParseResult parseWithLang(String input, Lang lang, boolean strict) {
+ String normalized = normalize(input);
+ List<String> tokens = tokenizeByLanguage(normalized, lang);
+ if (tokens.isEmpty()) return null;
+
+ Lexicon lx = LEX.get(lang);
+ if (lx == null) return null;
+
+ int decimalIndex = tokens.indexOf(lx.decimalMarker);
+ BigDecimal intPart;
+ int consumedInt;
+ if (decimalIndex >= 0) {
+ ParseSpan left = parseIntegerTokens(tokens.subList(0, decimalIndex), lang, strict);
+ if (left == null) return null;
+ intPart = new BigDecimal(left.value);
+ consumedInt = left.consumed;
+
+ ParseFraction frac = parseFractionTokens(tokens.subList(decimalIndex + 1, tokens.size()), lang, strict);
+ if (frac == null) return null;
+ BigDecimal value = intPart.add(frac.value);
+ int score = consumedInt + frac.consumed + 1;
+ return new ParseResult(value, score);
+ }
+
+ ParseSpan span = parseIntegerTokens(tokens, lang, strict);
+ if (span == null) return null;
+ return new ParseResult(new BigDecimal(span.value), span.consumed);
+ }
+
+ private static final class ParseSpan {
+ final long value;
+ final int consumed;
+ ParseSpan(long value, int consumed) {
+ this.value = value;
+ this.consumed = consumed;
+ }
+ }
+
+ private static ParseSpan parseIntegerTokens(List<String> tokens, Lang lang, boolean strict) {
+ if (tokens.isEmpty()) return new ParseSpan(0L, 0);
+ Lexicon lx = LEX.get(lang);
+ long total = 0L;
+ long current = 0L;
+ int consumed = 0;
+
+ for (int i = 0; i < tokens.size(); i++) {
+ String t = tokens.get(i);
+ if (t == null || t.isEmpty() || "and".equals(t) || "et".equals(t) || "y".equals(t) || "und".equals(t) || "e".equals(t)) {
+ continue;
+ }
+
+ if (isDigits(t)) {
+ long v = Long.parseLong(t);
+ current += v;
+ consumed++;
+ continue;
+ }
+
+ // French special: quatre vingt => 80
+ if (lang == Lang.FR && "quatre".equals(t) && i + 1 < tokens.size() && "vingt".equals(tokens.get(i + 1))) {
+ current += 80;
+ consumed += 2;
+ i++;
+ continue;
+ }
+
+ Integer small = lx.small.get(t);
+ if (small != null) {
+ current += small.intValue();
+ consumed++;
+ continue;
+ }
+
+ if ("hundred".equals(t) || "hundert".equals(t) || "cent".equals(t) || "cien".equals(t) || "ciento".equals(t) || "cento".equals(t)) {
+ if (current == 0) current = 1;
+ current *= 100;
+ consumed++;
+ continue;
+ }
+
+ Integer scale = lx.scales.get(t);
+ if (scale != null) {
+ if (current == 0) current = 1;
+ total += current * scale.longValue();
+ current = 0;
+ consumed++;
+ continue;
+ }
+
+ if (strict) return null;
+ }
+ return new ParseSpan(total + current, consumed);
+ }
+
+ private static final class ParseFraction {
+ final BigDecimal value;
+ final int consumed;
+ ParseFraction(BigDecimal value, int consumed) {
+ this.value = value;
+ this.consumed = consumed;
+ }
+ }
+
+ private static ParseFraction parseFractionTokens(List<String> tokens, Lang lang, boolean strict) {
+ if (tokens.isEmpty()) return new ParseFraction(BigDecimal.ZERO, 0);
+ Lexicon lx = LEX.get(lang);
+ StringBuilder digits = new StringBuilder();
+ int consumed = 0;
+ for (String t : tokens) {
+ if (t == null || t.isEmpty()) continue;
+ if (isDigits(t)) {
+ digits.append(t);
+ consumed++;
+ continue;
+ }
+ Integer v = lx.small.get(t);
+ if (v != null && v.intValue() >= 0 && v.intValue() <= 9) {
+ digits.append(v.intValue());
+ consumed++;
+ continue;
+ }
+ if (strict) return null;
+ }
+ if (digits.length() == 0) return new ParseFraction(BigDecimal.ZERO, consumed);
+ BigDecimal frac = new BigDecimal("0." + digits.toString());
+ return new ParseFraction(frac, consumed);
+ }
+
+ private static boolean isDigits(String t) {
+ if (t == null || t.isEmpty()) return false;
+ for (int i = 0; i < t.length(); i++) if (!Character.isDigit(t.charAt(i))) return false;
+ return true;
+ }
+
+ private static String normalize(String s) {
+ String n = s == null ? "" : s.toLowerCase(Locale.ROOT);
+ n = n.replace('’', '\'');
+ n = n.replace('-', ' ');
+ n = n.replace("ß", "ss");
+ n = Normalizer.normalize(n, Normalizer.Form.NFD).replaceAll("\\p{M}+", "");
+ n = n.replaceAll("[^a-z0-9' ]", " ");
+ n = n.replaceAll("\\s+", " ").trim();
+ return n;
+ }
+
+ private static List<String> tokenizeByLanguage(String normalized, Lang lang) {
+ if (normalized.isEmpty()) return new ArrayList<>();
+ String s = normalized;
+
+ if (lang == Lang.DE) {
+ // Split common German compounding boundaries in one pass to avoid
+ // destructive re-replacements (e.g. "hundert" -> "... t").
+ s = s.replaceAll("(millionen|million)", " million ");
+ s = s.replaceAll("(milliarden|milliarde)", " milliarde ");
+ s = s.replaceAll("(tausend)", " tausend ");
+ s = s.replaceAll("(hundert|hunder)", " hundert ");
+ s = s.replace("tausend", " tausend ");
+ // Split forms like "hundertundsieben" -> "hundert und sieben".
+ s = s.replaceAll("\\bund(ein|eine|eins|zwei|drei|vier|funf|fuenf|sechs|sieben|acht|neun|zehn|elf|zwolf|zwoelf|dreizehn|vierzehn|funfzehn|fuenfzehn|sechzehn|siebzehn|achtzehn|neunzehn|zwanzig|dreissig|dreiig|vierzig|funfzig|fuenfzig|sechzig|siebzig|achtzig|neunzig)\\b", "und $1");
+ // Split only canonical DE one-and-tens compounds (e.g. einundzwanzig).
+ s = s.replaceAll("(ein|eine|zwei|drei|vier|funf|fuenf|sechs|sieben|acht|neun)und(zwanzig|dreissig|dreiig|vierzig|funfzig|fuenfzig|sechzig|siebzig|achtzig|neunzig)", "$1 und $2");
+ s = s.replace("eins", "ein");
+ }
+
+ if (lang == Lang.ES) {
+ s = s.replaceAll("veinti([a-z]+)", "veinte $1");
+ s = s.replaceAll("dieci([a-z]+)", "diez $1");
+ }
+
+ if (lang == Lang.IT) {
+ s = s.replaceAll("(venti|trenta|quaranta|cinquanta|sessanta|settanta|ottanta|novanta)([a-z]+)", "$1 $2");
+ }
+
+ if (lang == Lang.FR) {
+ s = s.replace("quatre vingt", "quatre vingt");
+ }
+
+ s = s.replaceAll("\\s+", " ").trim();
+ if (s.isEmpty()) return new ArrayList<>();
+ return new ArrayList<>(Arrays.asList(s.split(" ")));
+ }
+
+ private static Map<Lang, Lexicon> buildLexicons() {
+ Map<Lang, Lexicon> map = new HashMap<>();
+
+ Lexicon de = new Lexicon("komma");
+ putAll(de.small,
+ "null",0, "ein",1, "eine",1, "einen",1, "eins",1, "zwo",2, "zwei",2, "drei",3, "vier",4, "funf",5, "fuenf",5, "sechs",6,
+ "sieben",7, "acht",8, "neun",9, "zehn",10, "elf",11, "zwolf",12, "zwoelf",12, "dreizehn",13, "vierzehn",14,
+ "funfzehn",15, "fuenfzehn",15, "sechzehn",16, "siebzehn",17, "achtzehn",18, "neunzehn",19,
+ "zwanzig",20, "dreissig",30, "dreiig",30, "vierzig",40, "funfzig",50, "fuenfzig",50, "sechzig",60, "siebzig",70,
+ "achtzig",80, "neunzig",90);
+ putAll(de.scales, "tausend",1000, "million",1000000, "millionen",1000000, "milliarde",1000000000, "milliarden",1000000000);
+ map.put(Lang.DE, de);
+
+ Lexicon en = new Lexicon("point");
+ putAll(en.small,
+ "zero",0, "one",1, "two",2, "three",3, "four",4, "five",5, "six",6, "seven",7, "eight",8, "nine",9,
+ "ten",10, "eleven",11, "twelve",12, "thirteen",13, "fourteen",14, "fifteen",15, "sixteen",16,
+ "seventeen",17, "eighteen",18, "nineteen",19,
+ "twenty",20, "thirty",30, "forty",40, "fifty",50, "sixty",60, "seventy",70, "eighty",80, "ninety",90);
+ putAll(en.scales, "thousand",1000, "million",1000000, "billion",1000000000);
+ map.put(Lang.EN, en);
+
+ Lexicon fr = new Lexicon("virgule");
+ putAll(fr.small,
+ "zero",0, "un",1, "une",1, "deux",2, "trois",3, "quatre",4, "cinq",5, "six",6, "sept",7, "huit",8, "neuf",9,
+ "dix",10, "onze",11, "douze",12, "treize",13, "quatorze",14, "quinze",15, "seize",16,
+ "vingt",20, "trente",30, "quarante",40, "cinquante",50, "soixante",60, "soixantedix",70,
+ "quatrevingt",80, "quatrevingtdix",90);
+ putAll(fr.scales, "mille",1000, "million",1000000, "milliard",1000000000);
+ map.put(Lang.FR, fr);
+
+ Lexicon es = new Lexicon("coma");
+ putAll(es.small,
+ "cero",0, "uno",1, "una",1, "dos",2, "tres",3, "cuatro",4, "cinco",5, "seis",6, "siete",7, "ocho",8, "nueve",9,
+ "diez",10, "once",11, "doce",12, "trece",13, "catorce",14, "quince",15, "dieciseis",16,
+ "diecisiete",17, "dieciocho",18, "diecinueve",19,
+ "veinte",20, "treinta",30, "cuarenta",40, "cincuenta",50, "sesenta",60, "setenta",70, "ochenta",80, "noventa",90,
+ "cien",100, "ciento",100, "doscientos",200, "trescientos",300, "cuatrocientos",400, "quinientos",500,
+ "seiscientos",600, "setecientos",700, "ochocientos",800, "novecientos",900);
+ putAll(es.scales, "mil",1000, "millon",1000000, "millones",1000000, "milmillon",1000000000);
+ map.put(Lang.ES, es);
+
+ Lexicon it = new Lexicon("virgola");
+ putAll(it.small,
+ "zero",0, "uno",1, "una",1, "due",2, "tre",3, "quattro",4, "cinque",5, "sei",6, "sette",7, "otto",8, "nove",9,
+ "dieci",10, "undici",11, "dodici",12, "tredici",13, "quattordici",14, "quindici",15, "sedici",16,
+ "diciassette",17, "diciotto",18, "diciannove",19,
+ "venti",20, "trenta",30, "quaranta",40, "cinquanta",50, "sessanta",60, "settanta",70, "ottanta",80, "novanta",90,
+ "cento",100, "duecento",200, "trecento",300, "quattrocento",400, "cinquecento",500,
+ "seicento",600, "settecento",700, "ottocento",800, "novecento",900);
+ putAll(it.scales, "mille",1000, "mila",1000, "milione",1000000, "milioni",1000000, "miliardo",1000000000);
+ map.put(Lang.IT, it);
+
+ return map;
+ }
+
+ private static void putAll(Map<String, Integer> m, Object... kv) {
+ for (int i = 0; i + 1 < kv.length; i += 2) {
+ m.put(String.valueOf(kv[i]), Integer.valueOf(((Number) kv[i + 1]).intValue()));
+ }
+ }
+}
diff --git a/source/net/yacy/ai/tools/PromptToMermaidTool.java b/source/net/yacy/ai/tools/PromptToMermaidTool.java
new file mode 100644
index 000000000..dc685f0d9
--- /dev/null
+++ b/source/net/yacy/ai/tools/PromptToMermaidTool.java
@@ -0,0 +1,314 @@
+/**
+ * PromptToMermaidTool
+ * Copyright 2026 by Michael Peter Christen
+ * First released 06.02.2026 at https://yacy.net
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program in the file lgpl21.txt
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.ai.tools;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+import java.util.regex.Pattern;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+import org.json.JSONTokener;
+
+import net.yacy.ai.ToolHandler;
+import net.yacy.ai.LLM;
+
+public class PromptToMermaidTool implements ToolHandler {
+
+ private static final String NAME = "prompt_to_mermaid";
+ //private static final List<String> DIAGRAM_TYPES = Arrays.asList("flowchart", "sequence", "graph");
+
+ private static final String SYSTEM_PROMPT =
+ "You convert natural-language process descriptions into Mermaid diagrams with deterministic, transparent behavior. "
+ + "Prefer explicit structure only from user text. Do not invent hidden steps. "
+ + "Support flowchart, sequence diagram, and graph only. "
+ + "If prompt is ambiguous or too vague, return fallback with plain-language summary and no forced diagram. "
+ + "Return ONLY JSON matching the requested fields.";
+
+ private static final JSONObject RESPONSE_SCHEMA = buildResponseSchema();
+
+ private static final Pattern EMAIL_PATTERN = Pattern.compile("(?i)\\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}\\b");
+ private static final Pattern PHONE_PATTERN = Pattern.compile("\\b\\+?[0-9][0-9\\s().-]{6,}[0-9]\\b");
+
+ @Override
+ public JSONObject definition() throws JSONException {
+ JSONObject tool = new JSONObject(true);
+ tool.put("type", "function");
+ JSONObject fn = new JSONObject(true);
+ fn.put("name", NAME);
+ fn.put("description", "Convert a user prompt into a Mermaid diagram with fallback summary when prompt is ambiguous.");
+
+ JSONObject params = new JSONObject(true);
+ params.put("type", "object");
+ JSONObject props = new JSONObject(true);
+
+ JSONObject prompt = new JSONObject(true);
+ prompt.put("type", "string");
+ prompt.put("description", "Natural-language diagram request.");
+ props.put("prompt", prompt);
+
+ JSONObject preferredType = new JSONObject(true);
+ preferredType.put("type", "string");
+ preferredType.put("description", "Optional hint: flowchart, sequence, graph.");
+ props.put("preferred_diagram_type", preferredType);
+
+ params.put("properties", props);
+ params.put("required", new JSONArray().put("prompt"));
+ fn.put("parameters", params);
+ tool.put("function", fn);
+ return tool;
+ }
+
+
+ public int maxCallsPerTurn() {
+ return 1;
+ }
+
+
+ public String execute(String arguments) {
+ final JSONObject args;
+ try {
+ args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments);
+ } catch (JSONException e) {
+ return fallback("Invalid arguments JSON", "", null, Arrays.asList("Could not parse tool arguments"));
+ }
+
+ String prompt = args.optString("prompt", "").trim();
+ String preferredType = normalizeType(args.optString("preferred_diagram_type", null));
+ if (prompt.isEmpty()) {
+ return fallback("Missing prompt", "", preferredType, Arrays.asList("No prompt text provided"));
+ }
+
+ String redactedPrompt = redactSensitive(prompt);
+ if (isVague(redactedPrompt)) {
+ return fallback("Prompt is too vague for deterministic diagram generation", redactedPrompt, preferredType,
+ Arrays.asList("Add concrete steps, actors, or relationships", "Fallback used to avoid hallucinated structure"));
+ }
+
+ LLM.LLMModel llmModel = LLM.llmFromUsage(LLM.LLMUsage.chat);
+ if (llmModel == null || llmModel.llm == null || llmModel.model == null || llmModel.model.isEmpty()) {
+ return fallback("No chat model configured for Mermaid generation", redactedPrompt, preferredType,
+ Arrays.asList("Configure a chat model in AI production models"));
+ }
+
+ final String userPrompt = buildUserPrompt(redactedPrompt, preferredType);
+ try {
+ String raw = queryModel(llmModel, userPrompt, RESPONSE_SCHEMA);
+ JSONObject parsed = parseResponseObject(raw);
+ return normalizeResult(parsed, redactedPrompt, preferredType).toString();
+ } catch (IOException e) {
+ try {
+ String raw = queryModel(llmModel, userPrompt, null);
+ JSONObject parsed = parseResponseObject(raw);
+ return normalizeResult(parsed, redactedPrompt, preferredType).toString();
+ } catch (Exception retryError) {
+ return fallback("LLM generation failed", redactedPrompt, preferredType,
+ Arrays.asList("Model call failed: " + retryError.getMessage()));
+ }
+ } catch (Exception e) {
+ return fallback("Invalid model output", redactedPrompt, preferredType,
+ Arrays.asList("Failed to parse model output: " + e.getMessage()));
+ }
+ }
+
+ private static String queryModel(LLM.LLMModel llmModel, String userPrompt, JSONObject schema) throws IOException {
+ try {
+ LLM.Context context = new LLM.Context(SYSTEM_PROMPT);
+ context.addPrompt(userPrompt);
+ return llmModel.llm.chat(llmModel.model, context, schema, 1200);
+ } catch (JSONException e) {
+ throw new IOException(e.getMessage(), e);
+ }
+ }
+
+ private static JSONObject parseResponseObject(String raw) throws JSONException {
+ if (raw == null) throw new JSONException("Empty model response");
+ String trimmed = raw.trim();
+ if (trimmed.startsWith("```")) {
+ int firstNewline = trimmed.indexOf('\n');
+ if (firstNewline >= 0) {
+ trimmed = trimmed.substring(firstNewline + 1);
+ }
+ int lastFence = trimmed.lastIndexOf("```");
+ if (lastFence >= 0) {
+ trimmed = trimmed.substring(0, lastFence).trim();
+ }
+ }
+ return new JSONObject(new JSONTokener(trimmed));
+ }
+
+ private static JSONObject normalizeResult(JSONObject parsed, String prompt, String preferredType) throws JSONException {
+ String status = parsed.optString("status", "fallback").trim().toLowerCase(Locale.ROOT);
+ if (!"success".equals(status) && !"fallback".equals(status)) status = "fallback";
+
+ String diagramType = normalizeType(parsed.optString("diagram_type", null));
+ String mermaidCode = parsed.optString("mermaid_code", "").trim();
+ String reasoning = parsed.optString("reasoning", "").trim();
+ JSONArray warnings = parsed.optJSONArray("warnings");
+ if (warnings == null) warnings = new JSONArray();
+
+ if (reasoning.isEmpty()) {
+ reasoning = "Generated from prompt using local structured conversion rules and LLM formatting.";
+ }
+
+ if ("success".equals(status)) {
+ if (diagramType == null) {
+ status = "fallback";
+ warnings.put("diagram_type missing or invalid");
+ }
+ if (!isValidMermaid(mermaidCode, diagramType)) {
+ status = "fallback";
+ warnings.put("Invalid Mermaid syntax for declared diagram_type");
+ }
+ }
+
+ if ("fallback".equals(status)) {
+ String fallbackReason = reasoning.isEmpty() ? "Prompt could not be deterministically mapped to a clear diagram." : reasoning;
+ return buildResult("fallback", null, "", fallbackReason, warnings, prompt, preferredType);
+ }
+ return buildResult("success", diagramType, mermaidCode, reasoning, warnings, prompt, preferredType);
+ }
+
+ private static JSONObject buildResult(String status, String diagramType, String mermaidCode, String reasoning,
+ JSONArray warnings, String prompt, String preferredType) throws JSONException {
+ JSONObject out = new JSONObject(true);
+ out.put("status", status);
+ out.put("diagram_type", diagramType == null ? JSONObject.NULL : diagramType);
+ out.put("mermaid_code", mermaidCode == null ? "" : mermaidCode);
+ out.put("reasoning", reasoning == null ? "" : reasoning);
+ out.put("warnings", warnings == null ? new JSONArray() : warnings);
+ if (prompt != null && !prompt.isEmpty()) out.put("prompt_redacted", prompt);
+ if (preferredType != null) out.put("preferred_diagram_type", preferredType);
+ return out;
+ }
+
+ private static boolean isValidMermaid(String code, String diagramType) {
+ if (code == null || code.trim().isEmpty() || diagramType == null) return false;
+ String text = code.trim().toLowerCase(Locale.ROOT);
+ if ("flowchart".equals(diagramType)) return text.startsWith("flowchart");
+ if ("sequence".equals(diagramType)) return text.startsWith("sequencediagram");
+ if ("graph".equals(diagramType)) return text.startsWith("graph");
+ return false;
+ }
+
+ private static String buildUserPrompt(String prompt, String preferredType) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Convert the following user request into Mermaid.\n");
+ sb.append("Requirements:\n");
+ sb.append("- deterministic interpretation only\n");
+ sb.append("- supported diagram types: flowchart, sequence, graph\n");
+ sb.append("- if ambiguous, return fallback with summary reasoning\n");
+ sb.append("- include warnings for assumptions\n");
+ sb.append("- output JSON only with keys: status, diagram_type, mermaid_code, reasoning, warnings\n");
+ if (preferredType != null) {
+ sb.append("- user preferred diagram type: ").append(preferredType).append("\n");
+ }
+ sb.append("\nUser prompt:\n");
+ sb.append(prompt);
+ return sb.toString();
+ }
+
+ private static boolean isVague(String prompt) {
+ if (prompt == null) return true;
+ String trimmed = prompt.trim();
+ if (trimmed.length() < 24) return true;
+ String[] tokens = trimmed.split("\\s+");
+ if (tokens.length < 6) return true;
+ String lower = trimmed.toLowerCase(Locale.ROOT);
+ return !(lower.contains("then") || lower.contains("first") || lower.contains("after")
+ || lower.contains("if") || lower.contains("when") || lower.contains("->")
+ || lower.contains("sequence") || lower.contains("workflow") || lower.contains("relationship"));
+ }
+
+ private static String redactSensitive(String prompt) {
+ if (prompt == null) return "";
+ String s = EMAIL_PATTERN.matcher(prompt).replaceAll("[EMAIL]");
+ s = PHONE_PATTERN.matcher(s).replaceAll("[PHONE]");
+ return s;
+ }
+
+ private static String normalizeType(String value) {
+ if (value == null) return null;
+ String type = value.trim().toLowerCase(Locale.ROOT);
+ if (type.isEmpty()) return null;
+ if ("flowchart".equals(type) || "sequence".equals(type) || "graph".equals(type)) return type;
+ if ("sequence diagram".equals(type) || "sequencediagram".equals(type)) return "sequence";
+ if ("flow".equals(type)) return "flowchart";
+ return null;
+ }
+
+ private static String fallback(String reason, String prompt, String preferredType, List<String> warnings) {
+ try {
+ JSONArray w = new JSONArray();
+ if (warnings != null) {
+ for (String warning : warnings) {
+ if (warning != null && !warning.isEmpty()) w.put(warning);
+ }
+ }
+ return buildResult("fallback", null, "",
+ reason == null ? "Fallback used due to insufficient structure in prompt." : reason,
+ w, prompt, preferredType).toString();
+ } catch (JSONException e) {
+ return ToolHandler.errorJson("Failed to build fallback response");
+ }
+ }
+
+ private static JSONObject buildResponseSchema() {
+ try {
+ JSONObject schema = new JSONObject(true);
+ schema.put("title", "PromptToMermaidResult");
+ schema.put("type", "object");
+ JSONObject properties = new JSONObject(true);
+
+ JSONObject status = new JSONObject(true);
+ status.put("type", "string");
+ status.put("enum", new JSONArray().put("success").put("fallback"));
+ properties.put("status", status);
+
+ JSONObject diagramType = new JSONObject(true);
+ diagramType.put("type", "string");
+ diagramType.put("enum", new JSONArray().put("flowchart").put("sequence").put("graph"));
+ properties.put("diagram_type", diagramType);
+
+ JSONObject mermaid = new JSONObject(true);
+ mermaid.put("type", "string");
+ properties.put("mermaid_code", mermaid);
+
+ JSONObject reasoning = new JSONObject(true);
+ reasoning.put("type", "string");
+ properties.put("reasoning", reasoning);
+
+ JSONObject warnings = new JSONObject(true);
+ warnings.put("type", "array");
+ warnings.put("items", new JSONObject(true).put("type", "string"));
+ properties.put("warnings", warnings);
+
+ schema.put("properties", properties);
+ schema.put("required", new JSONArray().put("status").put("diagram_type").put("mermaid_code").put("reasoning").put("warnings"));
+ return schema;
+ } catch (JSONException e) {
+ return null;
+ }
+ }
+}
diff --git a/source/net/yacy/ai/tools/SelfReflectTool.java b/source/net/yacy/ai/tools/SelfReflectTool.java
new file mode 100644
index 000000000..d1f99d099
--- /dev/null
+++ b/source/net/yacy/ai/tools/SelfReflectTool.java
@@ -0,0 +1,115 @@
+/**
+ * SelfReflectTool
+ * Copyright 2026 by Michael Peter Christen
+ * First released 06.02.2026 at https://yacy.net
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program in the file lgpl21.txt
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.ai.tools;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import net.yacy.ai.ToolHandler;
+
+public class SelfReflectTool implements ToolHandler {
+
+ private static final String NAME = "self_reflect";
+
+ private static final String PURPOSE =
+ "YaCy AI aims to be the good one: privacy-aware, open, and useful AI that combines LLM responses with transparent tools and retrieval from YaCy search.";
+
+ private static final List<String> FACTS = Collections.unmodifiableList(Arrays.asList(
+ "You can choose default dialog augmentation mode: no search, local search, or global search.",
+ "You can click the search button for the current message to attach search results as context without uploading a file.",
+ "You can run with no search and attach your own files instead, so you control exactly what context is sent.",
+ "You can attach PNG/JPG images and text files (.txt, .md, .tex), preview them, download them, or open them in a full view.",
+ "You can drag and drop supported files directly into the composer.",
+ "You can see transparency in action: search results are turned into virtual attachments so you can inspect which documents contributed to the answer. Click on the attachment button in your prompt to see the search answers!",
+ "You can see each tool call with wrench icons (click on it!) in assistant messages and open per-call details (arguments and tool response).",
+ "You can use deterministic tools such as datetime, calculator, webfetch, http_json, and table_ops to improve answer quality.",
+ "You can copy assistant answers, trim user prompts, and delete a user/assistant turn pair for iterative editing.",
+ "You can clear chat history, download the chat as JSON, and upload it again to continue later.",
+ "You can show or hide the system prompt in the interface to better understand chat behavior.",
+ "You can read markdown-formatted answers, including code blocks with syntax highlighting and collapsible model-thought sections when present.",
+ "You can use YaCy AI components beyond chat via standards-oriented integration points such as RAG proxy and MCP tooling."
+ ));
+
+ private static final List<String> IDEAS = Collections.unmodifiableList(Arrays.asList(
+ "Transparency, not mystery: ranking, retrieval, and model use should be explainable and inspectable.",
+ "Autonomy, no dependency: avoid hidden data flows and centralized lock-in whenever possible.",
+ "Collaboration with user control: intelligence should grow from shared knowledge while users stay in charge.",
+ "Reproducibility: users should be able to choose models and verify outcomes.",
+ "Integration instead of isolation: AI should be part of the open web, not a wall around it.",
+ "Teach openly: communicate purpose and benefit so AI is understandable, not opaque or intimidating.",
+ "YaCy extends open-search principles into AI: openness, autonomy, and trust."
+ ));
+
+ private static final String TALKS_URL = "https://www.youtube.com/orbiterlab";
+
+ @Override
+ public JSONObject definition() throws JSONException {
+ JSONObject tool = new JSONObject(true);
+ tool.put("type", "function");
+ JSONObject fn = new JSONObject(true);
+ fn.put("name", NAME);
+ fn.put("description", "This is the help function. Explain what YaCy AI does and why it is built this way. Use when user asks about abilities, purpose, or philosophy.");
+ JSONObject params = new JSONObject(true);
+ params.put("type", "object");
+ params.put("properties", new JSONObject(true));
+ params.put("required", new JSONArray());
+ fn.put("parameters", params);
+ tool.put("function", fn);
+ return tool;
+ }
+
+
+ public int maxCallsPerTurn() {
+ return 1;
+ }
+
+
+ public String execute(String arguments) {
+ try {
+ JSONObject result = new JSONObject(true);
+ result.put("tool", NAME);
+ result.put("purpose", PURPOSE);
+ result.put("facts", new JSONArray(FACTS));
+ result.put("ideas", new JSONArray(IDEAS));
+ result.put("talks_url", TALKS_URL);
+ return result.toString();
+ } catch (JSONException e) {
+ return ToolHandler.errorJson("Failed to build self_reflect response");
+ }
+ }
+
+ public static List<String> facts() {
+ return FACTS;
+ }
+
+ public static List<String> ideas() {
+ return IDEAS;
+ }
+
+ public static String purpose() {
+ return PURPOSE;
+ }
+}
diff --git a/source/net/yacy/ai/tools/TableOpsTool.java b/source/net/yacy/ai/tools/TableOpsTool.java
new file mode 100644
index 000000000..fe1b71699
--- /dev/null
+++ b/source/net/yacy/ai/tools/TableOpsTool.java
@@ -0,0 +1,285 @@
+/**
+ * TableOpsTool
+ * Copyright 2026 by Michael Peter Christen
+ * First released 06.02.2026 at https://yacy.net
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program in the file lgpl21.txt
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.ai.tools;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import net.yacy.ai.ToolHandler;
+
+public class TableOpsTool implements ToolHandler {
+
+ private static final String NAME = "table_ops";
+
+ @Override
+ public JSONObject definition() throws JSONException {
+ JSONObject tool = new JSONObject(true);
+ tool.put("type", "function");
+ JSONObject fn = new JSONObject(true);
+ fn.put("name", NAME);
+ fn.put("description", "Apply deterministic table operations to a JSON array of row objects.");
+
+ JSONObject params = new JSONObject(true);
+ params.put("type", "object");
+ JSONObject props = new JSONObject(true);
+
+ JSONObject rows = new JSONObject(true);
+ rows.put("type", "array");
+ rows.put("description", "Input table rows as an array of objects.");
+ props.put("rows", rows);
+
+ JSONObject operations = new JSONObject(true);
+ operations.put("type", "array");
+ operations.put("description", "Pipeline of operations: filter, sort, limit, project, group_count.");
+ props.put("operations", operations);
+
+ params.put("properties", props);
+ params.put("required", new JSONArray().put("rows").put("operations"));
+ fn.put("parameters", params);
+ tool.put("function", fn);
+ return tool;
+ }
+
+
+ public int maxCallsPerTurn() {
+ return 3;
+ }
+
+
+ public String execute(String arguments) {
+ final JSONObject args;
+ try {
+ args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments);
+ } catch (JSONException e) {
+ return ToolHandler.errorJson("Invalid arguments JSON");
+ }
+
+ final JSONArray rowsArray = args.optJSONArray("rows");
+ final JSONArray ops = args.optJSONArray("operations");
+ if (rowsArray == null) return ToolHandler.errorJson("Missing rows");
+ if (ops == null) return ToolHandler.errorJson("Missing operations");
+
+ try {
+ List<JSONObject> rows = toObjectRows(rowsArray);
+ for (int i = 0; i < ops.length(); i++) {
+ JSONObject op = ops.optJSONObject(i);
+ if (op == null) return ToolHandler.errorJson("Operation at index " + i + " must be an object");
+ rows = applyOperation(rows, op);
+ }
+ JSONObject result = new JSONObject(true);
+ result.put("row_count", rows.size());
+ result.put("rows", new JSONArray(rows));
+ return result.toString();
+ } catch (IllegalArgumentException | JSONException e) {
+ return ToolHandler.errorJson(e.getMessage());
+ }
+ }
+
+ private static List<JSONObject> toObjectRows(JSONArray array) {
+ List<JSONObject> rows = new ArrayList<>();
+ for (int i = 0; i < array.length(); i++) {
+ JSONObject row = array.optJSONObject(i);
+ if (row == null) throw new IllegalArgumentException("Row at index " + i + " is not an object");
+ rows.add(row);
+ }
+ return rows;
+ }
+
+ private static List<JSONObject> applyOperation(List<JSONObject> rows, JSONObject op) {
+ String type = op.optString("type", "").trim().toLowerCase();
+ if (type.isEmpty()) throw new IllegalArgumentException("Operation missing type");
+ if ("filter".equals(type)) return applyFilter(rows, op);
+ if ("sort".equals(type)) return applySort(rows, op);
+ if ("limit".equals(type)) return applyLimit(rows, op);
+ if ("project".equals(type)) return applyProject(rows, op);
+ if ("group_count".equals(type)) return applyGroupCount(rows, op);
+ throw new IllegalArgumentException("Unsupported operation type: " + type);
+ }
+
+ private static List<JSONObject> applyFilter(List<JSONObject> rows, JSONObject op) {
+ String field = requiredField(op, "field");
+ String cmp = op.optString("op", "eq").trim().toLowerCase();
+ Object expected = op.opt("value");
+ List<JSONObject> result = new ArrayList<>();
+ for (JSONObject row : rows) {
+ Object actual = row.opt(field);
+ if (matches(actual, cmp, expected)) result.add(row);
+ }
+ return result;
+ }
+
+ private static List<JSONObject> applySort(List<JSONObject> rows, JSONObject op) {
+ String field = requiredField(op, "field");
+ String order = op.optString("order", "asc").trim().toLowerCase();
+ final int direction = "desc".equals(order) ? -1 : 1;
+
+ List<JSONObject> copy = new ArrayList<>(rows);
+ Collections.sort(copy, new Comparator<JSONObject>() {
+ @Override
+ public int compare(JSONObject a, JSONObject b) {
+ Object va = a.opt(field);
+ Object vb = b.opt(field);
+ return direction * compareValues(va, vb);
+ }
+ });
+ return copy;
+ }
+
+ private static List<JSONObject> applyLimit(List<JSONObject> rows, JSONObject op) {
+ int count = op.optInt("count", -1);
+ int offset = op.optInt("offset", 0);
+ if (count < 0) throw new IllegalArgumentException("limit.count must be >= 0");
+ if (offset < 0) throw new IllegalArgumentException("limit.offset must be >= 0");
+ if (offset >= rows.size()) return new ArrayList<>();
+ int end = Math.min(rows.size(), offset + count);
+ return new ArrayList<>(rows.subList(offset, end));
+ }
+
+ private static List<JSONObject> applyProject(List<JSONObject> rows, JSONObject op) {
+ JSONArray fields = op.optJSONArray("fields");
+ if (fields == null || fields.length() == 0) throw new IllegalArgumentException("project.fields must be a non-empty array");
+ List<String> keep = new ArrayList<>();
+ for (int i = 0; i < fields.length(); i++) {
+ String f = fields.optString(i, "").trim();
+ if (!f.isEmpty()) keep.add(f);
+ }
+ if (keep.isEmpty()) throw new IllegalArgumentException("project.fields must contain at least one field");
+
+ List<JSONObject> out = new ArrayList<>();
+ for (JSONObject row : rows) {
+ JSONObject projected = new JSONObject(true);
+ for (String key : keep) {
+ if (row.has(key)) {
+ try {
+ projected.put(key, row.get(key));
+ } catch (JSONException e) {
+ throw new IllegalArgumentException("Failed to project field: " + key);
+ }
+ }
+ }
+ out.add(projected);
+ }
+ return out;
+ }
+
+ private static List<JSONObject> applyGroupCount(List<JSONObject> rows, JSONObject op) {
+ String field = requiredField(op, "field");
+ String order = op.optString("order", "desc").trim().toLowerCase();
+ final int direction = "asc".equals(order) ? 1 : -1;
+
+ Map<String, Integer> counts = new LinkedHashMap<>();
+ for (JSONObject row : rows) {
+ Object v = row.opt(field);
+ String key = v == null || v == JSONObject.NULL ? "null" : String.valueOf(v);
+ Integer cur = counts.get(key);
+ counts.put(key, cur == null ? 1 : cur + 1);
+ }
+
+ List<JSONObject> grouped = new ArrayList<>();
+ for (Map.Entry<String, Integer> entry : counts.entrySet()) {
+ try {
+ JSONObject g = new JSONObject(true);
+ g.put("key", entry.getKey());
+ g.put("count", entry.getValue());
+ grouped.add(g);
+ } catch (JSONException e) {
+ throw new IllegalArgumentException("Failed to build group_count result");
+ }
+ }
+ Collections.sort(grouped, new Comparator<JSONObject>() {
+ @Override
+ public int compare(JSONObject a, JSONObject b) {
+ return direction * Integer.compare(a.optInt("count", 0), b.optInt("count", 0));
+ }
+ });
+ return grouped;
+ }
+
+ private static String requiredField(JSONObject obj, String field) {
+ String value = obj.optString(field, "").trim();
+ if (value.isEmpty()) throw new IllegalArgumentException("Missing " + field);
+ return value;
+ }
+
+ private static boolean matches(Object actual, String cmp, Object expected) {
+ if ("eq".equals(cmp)) return valuesEqual(actual, expected);
+ if ("ne".equals(cmp)) return !valuesEqual(actual, expected);
+ if ("gt".equals(cmp)) return compareValues(actual, expected) > 0;
+ if ("gte".equals(cmp)) return compareValues(actual, expected) >= 0;
+ if ("lt".equals(cmp)) return compareValues(actual, expected) < 0;
+ if ("lte".equals(cmp)) return compareValues(actual, expected) <= 0;
+ if ("contains".equals(cmp)) {
+ if (actual == null || actual == JSONObject.NULL || expected == null || expected == JSONObject.NULL) return false;
+ if (actual instanceof JSONArray) return arrayContains((JSONArray) actual, expected);
+ return String.valueOf(actual).toLowerCase().contains(String.valueOf(expected).toLowerCase());
+ }
+ if ("in".equals(cmp)) {
+ if (!(expected instanceof JSONArray)) return false;
+ return arrayContains((JSONArray) expected, actual);
+ }
+ throw new IllegalArgumentException("Unsupported filter op: " + cmp);
+ }
+
+ private static boolean valuesEqual(Object a, Object b) {
+ if (a == b) return true;
+ if (a == null || a == JSONObject.NULL) return b == null || b == JSONObject.NULL;
+ if (b == null || b == JSONObject.NULL) return false;
+ Double na = asDouble(a);
+ Double nb = asDouble(b);
+ if (na != null && nb != null) return Double.compare(na, nb) == 0;
+ return String.valueOf(a).equals(String.valueOf(b));
+ }
+
+ private static int compareValues(Object a, Object b) {
+ if (a == b) return 0;
+ if (a == null || a == JSONObject.NULL) return -1;
+ if (b == null || b == JSONObject.NULL) return 1;
+ Double na = asDouble(a);
+ Double nb = asDouble(b);
+ if (na != null && nb != null) return Double.compare(na, nb);
+ return String.valueOf(a).compareToIgnoreCase(String.valueOf(b));
+ }
+
+ private static Double asDouble(Object value) {
+ if (value == null || value == JSONObject.NULL) return null;
+ if (value instanceof Number) return Double.valueOf(((Number) value).doubleValue());
+ try {
+ return Double.valueOf(String.valueOf(value));
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ }
+
+ private static boolean arrayContains(JSONArray array, Object wanted) {
+ for (int i = 0; i < array.length(); i++) {
+ if (valuesEqual(array.opt(i), wanted)) return true;
+ }
+ return false;
+ }
+}
diff --git a/source/net/yacy/ai/tools/UnitConverterTool.java b/source/net/yacy/ai/tools/UnitConverterTool.java
new file mode 100644
index 000000000..dfa41d100
--- /dev/null
+++ b/source/net/yacy/ai/tools/UnitConverterTool.java
@@ -0,0 +1,200 @@
+/**
+ * UnitConverterTool
+ * Copyright 2026 by Michael Peter Christen
+ * First released 06.02.2026 at https://yacy.net
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program in the file lgpl21.txt
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.ai.tools;
+
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import net.yacy.ai.ToolHandler;
+
+public class UnitConverterTool implements ToolHandler {
+
+ private static final String NAME = "unit_converter";
+
+ private static final Map<String, Double> LENGTH_TO_M = new HashMap<>();
+ private static final Map<String, Double> MASS_TO_KG = new HashMap<>();
+ private static final Map<String, Double> VOLUME_TO_L = new HashMap<>();
+ private static final Map<String, Double> SPEED_TO_MPS = new HashMap<>();
+
+ static {
+ LENGTH_TO_M.put("mm", 0.001d);
+ LENGTH_TO_M.put("cm", 0.01d);
+ LENGTH_TO_M.put("m", 1d);
+ LENGTH_TO_M.put("km", 1000d);
+ LENGTH_TO_M.put("in", 0.0254d);
+ LENGTH_TO_M.put("ft", 0.3048d);
+ LENGTH_TO_M.put("yd", 0.9144d);
+ LENGTH_TO_M.put("mi", 1609.344d);
+
+ MASS_TO_KG.put("mg", 0.000001d);
+ MASS_TO_KG.put("g", 0.001d);
+ MASS_TO_KG.put("kg", 1d);
+ MASS_TO_KG.put("t", 1000d);
+ MASS_TO_KG.put("oz", 0.028349523125d);
+ MASS_TO_KG.put("lb", 0.45359237d);
+
+ VOLUME_TO_L.put("ml", 0.001d);
+ VOLUME_TO_L.put("l", 1d);
+ VOLUME_TO_L.put("m3", 1000d);
+ VOLUME_TO_L.put("tsp", 0.00492892159375d);
+ VOLUME_TO_L.put("tbsp", 0.01478676478125d);
+ VOLUME_TO_L.put("cup", 0.2365882365d);
+ VOLUME_TO_L.put("pt", 0.473176473d);
+ VOLUME_TO_L.put("qt", 0.946352946d);
+ VOLUME_TO_L.put("gal", 3.785411784d);
+
+ SPEED_TO_MPS.put("m/s", 1d);
+ SPEED_TO_MPS.put("km/h", 0.2777777777777778d);
+ SPEED_TO_MPS.put("mph", 0.44704d);
+ SPEED_TO_MPS.put("kn", 0.514444d);
+ }
+
+ @Override
+ public JSONObject definition() throws JSONException {
+ JSONObject tool = new JSONObject(true);
+ tool.put("type", "function");
+ JSONObject fn = new JSONObject(true);
+ fn.put("name", NAME);
+ fn.put("description", "Convert numeric values between common units (length, mass, temperature, volume, speed).");
+ JSONObject params = new JSONObject(true);
+ params.put("type", "object");
+ JSONObject props = new JSONObject(true);
+ props.put("value", new JSONObject(true).put("type", "number").put("description", "Numeric input value."));
+ props.put("from", new JSONObject(true).put("type", "string").put("description", "Source unit, e.g. km, lb, C."));
+ props.put("to", new JSONObject(true).put("type", "string").put("description", "Target unit, e.g. mi, kg, F."));
+ params.put("properties", props);
+ params.put("required", new JSONArray().put("value").put("from").put("to"));
+ fn.put("parameters", params);
+ tool.put("function", fn);
+ return tool;
+ }
+
+
+ public int maxCallsPerTurn() {
+ return 10;
+ }
+
+
+ public String execute(String arguments) {
+ final JSONObject args;
+ try {
+ args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments);
+ } catch (JSONException e) {
+ return ToolHandler.errorJson("Invalid arguments JSON");
+ }
+ if (!args.has("value")) return ToolHandler.errorJson("Missing value");
+ double value = args.optDouble("value", Double.NaN);
+ if (Double.isNaN(value) || Double.isInfinite(value)) return ToolHandler.errorJson("Invalid value");
+ String from = normalize(args.optString("from", ""));
+ String to = normalize(args.optString("to", ""));
+ if (from.isEmpty() || to.isEmpty()) return ToolHandler.errorJson("Missing from/to");
+
+ try {
+ ConversionResult result = convert(value, from, to);
+ JSONObject out = new JSONObject(true);
+ out.put("tool", NAME);
+ out.put("category", result.category);
+ out.put("value", value);
+ out.put("from", from);
+ out.put("to", to);
+ out.put("result", result.value);
+ return out.toString();
+ } catch (IllegalArgumentException | JSONException e) {
+ return ToolHandler.errorJson(e.getMessage());
+ }
+ }
+
+ private static ConversionResult convert(double value, String from, String to) {
+ if (isTemp(from) && isTemp(to)) {
+ double c = toCelsius(value, from);
+ return new ConversionResult("temperature", fromCelsius(c, to));
+ }
+ if (LENGTH_TO_M.containsKey(from) && LENGTH_TO_M.containsKey(to)) {
+ double m = value * LENGTH_TO_M.get(from);
+ return new ConversionResult("length", m / LENGTH_TO_M.get(to));
+ }
+ if (MASS_TO_KG.containsKey(from) && MASS_TO_KG.containsKey(to)) {
+ double kg = value * MASS_TO_KG.get(from);
+ return new ConversionResult("mass", kg / MASS_TO_KG.get(to));
+ }
+ if (VOLUME_TO_L.containsKey(from) && VOLUME_TO_L.containsKey(to)) {
+ double l = value * VOLUME_TO_L.get(from);
+ return new ConversionResult("volume", l / VOLUME_TO_L.get(to));
+ }
+ if (SPEED_TO_MPS.containsKey(from) && SPEED_TO_MPS.containsKey(to)) {
+ double mps = value * SPEED_TO_MPS.get(from);
+ return new ConversionResult("speed", mps / SPEED_TO_MPS.get(to));
+ }
+ throw new IllegalArgumentException("Unsupported conversion pair: " + from + " -> " + to);
+ }
+
+ private static boolean isTemp(String unit) {
+ return "c".equals(unit) || "f".equals(unit) || "k".equals(unit);
+ }
+
+ private static double toCelsius(double v, String from) {
+ if ("c".equals(from)) return v;
+ if ("f".equals(from)) return (v - 32d) * 5d / 9d;
+ if ("k".equals(from)) return v - 273.15d;
+ throw new IllegalArgumentException("Unsupported temperature unit: " + from);
+ }
+
+ private static double fromCelsius(double c, String to) {
+ if ("c".equals(to)) return c;
+ if ("f".equals(to)) return (c * 9d / 5d) + 32d;
+ if ("k".equals(to)) return c + 273.15d;
+ throw new IllegalArgumentException("Unsupported temperature unit: " + to);
+ }
+
+ private static String normalize(String s) {
+ if (s == null) return "";
+ String t = s.trim().toLowerCase(Locale.ROOT);
+ if ("°c".equals(t) || "celsius".equals(t)) return "c";
+ if ("°f".equals(t) || "fahrenheit".equals(t)) return "f";
+ if ("kelvin".equals(t)) return "k";
+ if ("meter".equals(t) || "metre".equals(t)) return "m";
+ if ("kilometer".equals(t) || "kilometre".equals(t)) return "km";
+ if ("mile".equals(t)) return "mi";
+ if ("foot".equals(t)) return "ft";
+ if ("inch".equals(t)) return "in";
+ if ("pound".equals(t)) return "lb";
+ if ("ounce".equals(t)) return "oz";
+ if ("liter".equals(t) || "litre".equals(t)) return "l";
+ if ("gallon".equals(t)) return "gal";
+ if ("kph".equals(t)) return "km/h";
+ if ("fps".equals(t)) return "ft/s";
+ return t;
+ }
+
+ private static final class ConversionResult {
+ final String category;
+ final double value;
+ ConversionResult(String category, double value) {
+ this.category = category;
+ this.value = value;
+ }
+ }
+}
diff --git a/source/net/yacy/ai/tools/WebFetchTool.java b/source/net/yacy/ai/tools/WebFetchTool.java
new file mode 100644
index 000000000..e09669807
--- /dev/null
+++ b/source/net/yacy/ai/tools/WebFetchTool.java
@@ -0,0 +1,168 @@
+/**
+ * WebFetchTool
+ * Copyright 2026 by Michael Peter Christen
+ * First released 06.02.2026 at https://yacy.net
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program in the file lgpl21.txt
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.ai.tools;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Date;
+import java.util.HashSet;
+
+import org.apache.http.Header;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import net.yacy.ai.ToolHandler;
+import net.yacy.cora.document.analysis.Classification;
+import net.yacy.cora.document.analysis.Classification.ContentDomain;
+import net.yacy.cora.document.id.DigestURL;
+import net.yacy.cora.document.id.MultiProtocolURL;
+import net.yacy.cora.protocol.ClientIdentification;
+import net.yacy.cora.protocol.HeaderFramework;
+import net.yacy.cora.protocol.http.HTTPClient;
+import net.yacy.document.Document;
+import net.yacy.document.Parser;
+import net.yacy.document.TextParser;
+import net.yacy.document.VocabularyScraper;
+import net.yacy.document.parser.html.TagValency;
+
+public class WebFetchTool implements ToolHandler {
+
+ private static final String NAME = "webfetch";
+ private static final int TOOL_WEBFETCH_MAX_BYTES = 2_000_000;
+ private static final int TOOL_WEBFETCH_MAX_CHARS = 12_000;
+
+ @Override
+ public JSONObject definition() throws JSONException {
+ JSONObject tool = new JSONObject(true);
+ tool.put("type", "function");
+ JSONObject fn = new JSONObject(true);
+ fn.put("name", NAME);
+ fn.put("description", "Fetch a URL and return text content. Parse HTML/documents into markdown-style text when possible.");
+ JSONObject params = new JSONObject(true);
+ params.put("type", "object");
+ JSONObject props = new JSONObject(true);
+ JSONObject url = new JSONObject(true);
+ url.put("type", "string");
+ url.put("description", "Absolute http or https URL to fetch.");
+ props.put("url", url);
+ params.put("properties", props);
+ params.put("required", new JSONArray().put("url"));
+ fn.put("parameters", params);
+ tool.put("function", fn);
+ return tool;
+ }
+
+
+ public int maxCallsPerTurn() {
+ return 3;
+ }
+
+
+ public String execute(String arguments) {
+ String urlRaw;
+ try {
+ JSONObject obj = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments);
+ urlRaw = obj.optString("url", "").trim();
+ } catch (JSONException e) {
+ return ToolHandler.errorJson("Invalid arguments JSON. don't try again");
+ }
+ if (urlRaw == null || urlRaw.isEmpty()) return ToolHandler.errorJson("Missing url. don't try again");
+
+ final DigestURL url;
+ try {
+ url = new DigestURL(urlRaw);
+ } catch (Exception e) {
+ return ToolHandler.errorJson("Invalid URL. don't try again");
+ }
+ final String protocol = url.getProtocol();
+ if (!"http".equalsIgnoreCase(protocol) && !"https".equalsIgnoreCase(protocol)) {
+ return ToolHandler.errorJson("Only http/https URLs are allowed. don't try again");
+ }
+
+ try (HTTPClient client = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, 30000)) {
+ byte[] content = client.GETbytes(url, null, null, TOOL_WEBFETCH_MAX_BYTES, false);
+ int status = client.getStatusCode();
+ if (status < 200 || status >= 300) return ToolHandler.errorJson("HTTP status " + status + ". don't try again");
+ if (content == null || content.length == 0) return ToolHandler.errorJson("Empty response body. don't try again");
+
+ String contentType = "application/octet-stream";
+ Header ct = client.getHttpResponse() == null ? null : client.getHttpResponse().getFirstHeader(HeaderFramework.CONTENT_TYPE);
+ if (ct != null && ct.getValue() != null && !ct.getValue().isEmpty()) contentType = ct.getValue();
+ int semicolon = contentType.indexOf(';');
+ if (semicolon > 0) contentType = contentType.substring(0, semicolon).trim().toLowerCase();
+
+ final String ext = MultiProtocolURL.getFileExtension(url.getFileName());
+ final ContentDomain mimeDomain = Classification.getContentDomainFromMime(contentType);
+ if (mimeDomain == ContentDomain.IMAGE || mimeDomain == ContentDomain.AUDIO || mimeDomain == ContentDomain.VIDEO
+ || Classification.isMediaExtension(ext)) {
+ return ToolHandler.errorJson("Media content is not supported. don't try again");
+ }
+
+ String text;
+ if (contentType.startsWith("text/plain") || contentType.startsWith("text/markdown") || "txt".equals(ext) || "md".equals(ext)) {
+ text = truncate(new String(content, StandardCharsets.UTF_8), TOOL_WEBFETCH_MAX_CHARS);
+ } else {
+ String supportError = TextParser.supports(url, contentType);
+ if (supportError == null) {
+ Document[] docs = TextParser.parseSource(url, contentType, "UTF-8", TagValency.EVAL,
+ new HashSet<String>(), new VocabularyScraper(), 0, 0, content, new Date());
+ Document merged = Document.mergeDocuments(url, contentType, docs);
+ text = truncate(documentToMarkdown(merged), TOOL_WEBFETCH_MAX_CHARS);
+ } else if (contentType.startsWith("text/")) {
+ text = truncate(new String(content, StandardCharsets.UTF_8), TOOL_WEBFETCH_MAX_CHARS);
+ } else {
+ return ToolHandler.errorJson("Unsupported content type " + contentType + ". don't try again");
+ }
+ }
+
+ JSONObject result = new JSONObject(true);
+ result.put("url", url.toNormalform(true));
+ result.put("content_type", contentType);
+ result.put("content", text == null ? "" : text);
+ return result.toString();
+ } catch (IOException e) {
+ return ToolHandler.errorJson("Fetch error: " + e.getMessage() + ". don't try again");
+ } catch (Parser.Failure e) {
+ return ToolHandler.errorJson("Parse error: " + e.getMessage() + ". don't try again");
+ } catch (JSONException e) {
+ return ToolHandler.errorJson("Failed to build tool response. don't try again");
+ }
+ }
+
+ private static String documentToMarkdown(Document doc) {
+ if (doc == null) return "";
+ StringBuilder sb = new StringBuilder();
+ String title = doc.dc_title();
+ if (title != null && !title.isEmpty()) {
+ sb.append("# ").append(title).append("\n\n");
+ }
+ String text = doc.getTextString();
+ if (text != null && !text.isEmpty()) sb.append(text);
+ return sb.toString();
+ }
+
+ private static String truncate(String text, int maxChars) {
+ if (text == null) return "";
+ if (maxChars <= 0 || text.length() <= maxChars) return text;
+ return text.substring(0, maxChars);
+ }
+}
diff --git a/source/net/yacy/ai/tools/WikipediaLinkCreatorTool.java b/source/net/yacy/ai/tools/WikipediaLinkCreatorTool.java
new file mode 100644
index 000000000..977d9574d
--- /dev/null
+++ b/source/net/yacy/ai/tools/WikipediaLinkCreatorTool.java
@@ -0,0 +1,169 @@
+/**
+ * WikipediaLinkCreatorTool
+ * Copyright 2026 by Michael Peter Christen
+ * First released 06.02.2026 at https://yacy.net
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program in the file lgpl21.txt
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.ai.tools;
+
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.regex.Pattern;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import net.yacy.ai.ToolHandler;
+
+public class WikipediaLinkCreatorTool implements ToolHandler {
+
+ private static final String NAME = "wikipedia_link_creator";
+ private static final Pattern LANG_PATTERN = Pattern.compile("^[a-z]{2,10}(-[a-z0-9]{2,8})?$");
+
+ @Override
+ public JSONObject definition() throws JSONException {
+ JSONObject tool = new JSONObject(true);
+ tool.put("type", "function");
+ JSONObject fn = new JSONObject(true);
+ fn.put("name", NAME);
+ fn.put("description", "In case you don't know the answer correctly or you are unsure. Build a Wikipedia article URL from topic keywords so it can be fetched with webfetch.");
+
+ JSONObject params = new JSONObject(true);
+ params.put("type", "object");
+ JSONObject props = new JSONObject(true);
+
+ JSONObject topic = new JSONObject(true);
+ topic.put("type", "string");
+ topic.put("description", "Target Wikipedia article title/topic (preferred input).");
+ props.put("topic", topic);
+
+ JSONObject keywords = new JSONObject(true);
+ keywords.put("type", "array");
+ keywords.put("description", "Optional keyword list from which a topic is composed.");
+ keywords.put("items", new JSONObject(true).put("type", "string"));
+ props.put("keywords", keywords);
+
+ JSONObject language = new JSONObject(true);
+ language.put("type", "string");
+ language.put("description", "Wikipedia language code (e.g. en, de, fr). Defaults to en.");
+ props.put("language", language);
+
+ params.put("properties", props);
+ params.put("required", new JSONArray());
+ fn.put("parameters", params);
+ tool.put("function", fn);
+ return tool;
+ }
+
+
+ public int maxCallsPerTurn() {
+ return 3;
+ }
+
+
+ public String execute(String arguments) {
+ final JSONObject args;
+ try {
+ args = (arguments == null || arguments.isEmpty()) ? new JSONObject(true) : new JSONObject(arguments);
+ } catch (JSONException e) {
+ return ToolHandler.errorJson("Invalid arguments JSON");
+ }
+
+ String topic = normalizeTopic(args.optString("topic", ""));
+ JSONArray keywords = args.optJSONArray("keywords");
+ String language = normalizeLanguage(args.optString("language", "en"));
+ List<String> warnings = new ArrayList<>();
+
+ if (topic.isEmpty() && keywords != null) {
+ topic = normalizeTopic(joinKeywords(keywords));
+ if (!topic.isEmpty()) {
+ warnings.add("Used keywords to compose article topic");
+ }
+ }
+ if (topic.isEmpty()) {
+ return ToolHandler.errorJson("Missing topic/keywords");
+ }
+ if (language == null) {
+ language = "en";
+ warnings.add("Invalid language code; defaulted to en");
+ }
+
+ String encodedTitle = encodeTitle(topic);
+ String url = "https://" + language + ".wikipedia.org/wiki/" + encodedTitle;
+
+ try {
+ JSONObject result = new JSONObject(true);
+ result.put("tool", NAME);
+ result.put("topic_input", topic);
+ result.put("article_title", topic);
+ result.put("language", language);
+ result.put("url", url);
+ result.put("next_tool_hint", "Use webfetch with this URL to retrieve the article content.");
+ result.put("warnings", new JSONArray(warnings));
+ return result.toString();
+ } catch (JSONException e) {
+ return ToolHandler.errorJson("Failed to build wikipedia link response");
+ }
+ }
+
+ private static String joinKeywords(JSONArray keywords) {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < keywords.length(); i++) {
+ String k = keywords.optString(i, "").trim();
+ if (k.isEmpty()) continue;
+ if (sb.length() > 0) sb.append(' ');
+ sb.append(k);
+ }
+ return sb.toString();
+ }
+
+ private static String normalizeTopic(String topic) {
+ if (topic == null) return "";
+ String t = topic.trim();
+ while (t.contains(" ")) t = t.replace(" ", " ");
+ if (t.startsWith("https://") || t.startsWith("http://")) {
+ int wikiIdx = t.indexOf("/wiki/");
+ if (wikiIdx >= 0) {
+ t = t.substring(wikiIdx + 6);
+ }
+ }
+ t = t.replace('_', ' ').trim();
+ if (t.indexOf('#') >= 0) t = t.substring(0, t.indexOf('#')).trim();
+ return t;
+ }
+
+ private static String normalizeLanguage(String language) {
+ if (language == null) return "en";
+ String l = language.trim().toLowerCase(Locale.ROOT);
+ if (l.isEmpty()) return "en";
+ if (!LANG_PATTERN.matcher(l).matches()) return null;
+ return l;
+ }
+
+ private static String encodeTitle(String title) {
+ try {
+ return URLEncoder.encode(title, StandardCharsets.UTF_8.name()).replace("+", "_");
+ } catch (UnsupportedEncodingException e) {
+ return title.replace(' ', '_');
+ }
+ }
+}
diff --git a/source/net/yacy/http/servlets/MCPSearchServlet.java b/source/net/yacy/http/servlets/MCPSearchServlet.java
index ffca72aac..983ad24f7 100644
--- a/source/net/yacy/http/servlets/MCPSearchServlet.java
+++ b/source/net/yacy/http/servlets/MCPSearchServlet.java
@@ -20,6 +20,7 @@
package net.yacy.http.servlets;
+import net.yacy.ai.RAGAugmentor;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.util.ConcurrentLog;
@@ -58,6 +59,17 @@ public class MCPSearchServlet extends HttpServlet {
private static final int DEFAULT_RESULT_COUNT = 10;
private static final int MAX_RESULT_COUNT = 100;
+ /**
+ * Handles JSON-RPC requests for the MCP surface.
+ * <p>
+ * Accepts single request objects and batch arrays, supports notifications, and
+ * always returns UTF-8 JSON responses.
+ *
+ * @param request servlet request
+ * @param response servlet response
+ * @throws ServletException servlet errors
+ * @throws IOException I/O errors
+ */
@Override
public void service(final ServletRequest request, final ServletResponse response) throws ServletException, IOException {
request.setCharacterEncoding("UTF-8");
@@ -72,6 +84,7 @@ public class MCPSearchServlet extends HttpServlet {
final Method reqMethod = Method.getMethod(hrequest.getMethod());
if (reqMethod == Method.OTHER) {
+ // Browsers perform CORS preflight with OPTIONS; answer quickly.
hresponse.setStatus(HttpServletResponse.SC_OK);
return;
}
@@ -85,10 +98,12 @@ public class MCPSearchServlet extends HttpServlet {
parsed = tokener.nextValue();
} catch (JSONException e) {
writeJsonResponse(hresponse, errorResponse(JSONObject.NULL, -32700, e.getMessage()));
+ return;
}
try {
if (parsed instanceof JSONObject) {
+ // Support shorthand endpoint style: infer method from URI path.
if (((JSONObject) parsed).optString("method", "").length() == 0) {
String uri = hrequest.getRequestURI();
((JSONObject) parsed).put("method", uri.substring(1));
@@ -100,6 +115,7 @@ public class MCPSearchServlet extends HttpServlet {
hresponse.setStatus(HttpServletResponse.SC_NO_CONTENT);
}
} else if (parsed instanceof JSONArray) {
+ // Batch request mode.
final JSONArray requestArray = (JSONArray) parsed;
final JSONArray responseArray = new JSONArray();
for (int i = 0; i < requestArray.length(); i++) {
@@ -124,6 +140,12 @@ public class MCPSearchServlet extends HttpServlet {
}
}
+ /**
+ * Routes one JSON-RPC request object to the corresponding MCP handler.
+ *
+ * @param requestObject parsed request
+ * @return response object or {@code null} for notifications
+ */
private JSONObject handleRequest(final JSONObject requestObject) {
final Object id = requestObject.opt("id");
final String jsonrpc = requestObject.optString("jsonrpc", JSONRPC_VERSION);
@@ -147,6 +169,7 @@ public class MCPSearchServlet extends HttpServlet {
}
if (id == JSONObject.NULL || id == null) {
+ // Notifications must not produce a response body.
// Notification: acknowledge silently
if ("ping".equals(method)) {
return null;
@@ -167,6 +190,13 @@ public class MCPSearchServlet extends HttpServlet {
}
}
+ /**
+ * Handles MCP initialize call and advertises protocol/capabilities.
+ *
+ * @param id JSON-RPC id
+ * @param params initialize params
+ * @return JSON-RPC success or error response
+ */
private JSONObject handleInitialize(final Object id, final JSONObject params) {
try {log.info("MCPSearchServlet: initialize " + (params == null ? "" : params.toString(0)));} catch (JSONException e) {}
final JSONObject result = new JSONObject(true);
@@ -190,6 +220,13 @@ public class MCPSearchServlet extends HttpServlet {
return successResponse(id, result);
}
+ /**
+ * Handles MCP tool listing and returns schema for the single {@code search}
+ * tool.
+ *
+ * @param id JSON-RPC id
+ * @return JSON-RPC success or error response
+ */
private JSONObject handleToolsList(final Object id) {
log.info("MCPSearchServlet: list " + (id == null ? "" : id.toString()));
try {
@@ -245,6 +282,13 @@ public class MCPSearchServlet extends HttpServlet {
}
}
+ /**
+ * Handles tool invocation for {@code search}.
+ *
+ * @param id JSON-RPC id
+ * @param params tool call params
+ * @return JSON-RPC success or error response
+ */
private JSONObject handleToolsCall(final Object id, final JSONObject params) {
try {log.info("MCPSearchServlet: call " + (id == null ? "" : id.toString()) + " params: " + (params == null ? "" : params.toString(0)));} catch (JSONException e) {}
if (params == null) {
@@ -265,6 +309,7 @@ public class MCPSearchServlet extends HttpServlet {
return errorResponse(id, -32602, "Tool argument 'query' must be a non-empty string");
}
+ // Sanitize and clamp result limit.
int limit = arguments.optInt("limit", DEFAULT_RESULT_COUNT);
if (limit <= 0) {
limit = DEFAULT_RESULT_COUNT;
@@ -272,8 +317,9 @@ public class MCPSearchServlet extends HttpServlet {
limit = Math.min(limit, MAX_RESULT_COUNT);
final boolean includeSnippet = arguments.optBoolean("include_snippet", true);
+ // Delegate the actual search to the RAG helper utility.
JSONArray results;
- results = RAGProxyServlet.searchResults(query, limit, includeSnippet);
+ results = RAGAugmentor.searchResults(query, limit, includeSnippet);
try {
final JSONObject payload = new JSONObject(true);
@@ -294,6 +340,13 @@ public class MCPSearchServlet extends HttpServlet {
}
}
+ /**
+ * Reads full HTTP request body into a string.
+ *
+ * @param request servlet request
+ * @return request body (possibly empty)
+ * @throws IOException when reader access fails
+ */
private static String readBody(final ServletRequest request) throws IOException {
final StringBuilder builder = new StringBuilder();
try (BufferedReader reader = request.getReader()) {
@@ -305,6 +358,13 @@ public class MCPSearchServlet extends HttpServlet {
return builder.toString();
}
+ /**
+ * Builds a JSON-RPC success object.
+ *
+ * @param id JSON-RPC id
+ * @param result result payload
+ * @return JSON-RPC success response
+ */
private static JSONObject successResponse(final Object id, final JSONObject result) {
final JSONObject response = new JSONObject(true);
try {
@@ -318,6 +378,14 @@ public class MCPSearchServlet extends HttpServlet {
return response;
}
+ /**
+ * Builds a JSON-RPC error object.
+ *
+ * @param id JSON-RPC id
+ * @param code JSON-RPC error code
+ * @param message human-readable error message
+ * @return JSON-RPC error response
+ */
private static JSONObject errorResponse(final Object id, final int code, final String message) {
final JSONObject response = new JSONObject(true);
try {
@@ -333,6 +401,13 @@ public class MCPSearchServlet extends HttpServlet {
return response;
}
+ /**
+ * Serializes and writes either a JSON object or JSON array response.
+ *
+ * @param response servlet response
+ * @param payload JSONObject or JSONArray
+ * @throws IOException when writing fails
+ */
private static void writeJsonResponse(final HttpServletResponse response, final Object payload) throws IOException {
final String serialized = payload instanceof JSONObject ? ((JSONObject) payload).toString()
: payload instanceof JSONArray ? ((JSONArray) payload).toString() : payload.toString();
diff --git a/source/net/yacy/http/servlets/RAGProxyServlet.java b/source/net/yacy/http/servlets/RAGProxyServlet.java
index 840e5a564..c552b7387 100644
--- a/source/net/yacy/http/servlets/RAGProxyServlet.java
+++ b/source/net/yacy/http/servlets/RAGProxyServlet.java
@@ -22,31 +22,14 @@ package net.yacy.http.servlets;
import java.io.BufferedReader;
import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.net.HttpURLConnection;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.AbstractMap;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Base64;
-import java.util.Collection;
-import java.util.Comparator;
import java.util.Deque;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashSet;
import java.util.List;
-import java.util.Map;
import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ConcurrentLinkedDeque;
-import java.util.stream.Collectors;
import javax.servlet.ServletException;
import javax.servlet.ServletOutputStream;
@@ -56,38 +39,17 @@ import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrDocumentList;
-import org.apache.solr.common.SolrException;
import org.apache.solr.servlet.cache.Method;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
-import org.json.JSONTokener;
import net.yacy.ai.LLM;
-import net.yacy.cora.document.analysis.Classification;
-import net.yacy.cora.document.id.DigestURL;
-import net.yacy.cora.document.id.MultiProtocolURL;
-import net.yacy.cora.federate.solr.SolrType;
-import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector;
-import net.yacy.cora.federate.yacy.CacheStrategy;
-import net.yacy.cora.lod.vocabulary.Tagging;
-import net.yacy.cora.protocol.ClientIdentification;
+import net.yacy.ai.RAGAugmentor;
+import net.yacy.ai.ToolCallProtocol;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.util.ConcurrentLog;
-import net.yacy.kelondro.data.meta.URIMetadataNode;
import net.yacy.search.Switchboard;
-import net.yacy.search.SwitchboardConstants;
-import net.yacy.search.query.QueryGoal;
-import net.yacy.search.query.QueryModifier;
-import net.yacy.search.query.QueryParams;
-import net.yacy.search.query.SearchEvent;
-import net.yacy.search.query.SearchEventCache;
-import net.yacy.search.ranking.RankingProfile;
-import net.yacy.search.schema.CollectionSchema;
-import net.yacy.search.snippet.TextSnippet;
/**
* This class implements a Retrieval Augmented Generation ("RAG") proxy which
@@ -109,7 +71,6 @@ public class RAGProxyServlet extends HttpServlet {
private static final long serialVersionUID = 3411544789759643137L;
public static final String LLM_SYSTEM_PROMPT_DEFAULT = "You are a smart and helpful chatbot. If possible, use friendly emojies.";
- private static final String LLM_SYSTEM_PREFIX_DEFAULT = "\n\nYou may receive additional expert knowledge in the user prompt after a 'Additional Information' headline to enhance your knowledge. Use it only if applicable.";
private static final String LLM_USER_PREFIX_DEFAULT = "\n\nAdditional Information:\n\nbelow you find a collection of texts that might be useful to generate a response. Do not discuss these documents, just use them to answer the question above.\n\n";
private static final String LLM_QUERY_GENERATOR_PREFIX_DEFAULT = "Make a list of search words with low document frequency for the following prompt; use a JSON Array: ";
@@ -193,7 +154,6 @@ public class RAGProxyServlet extends HttpServlet {
// get messages and prepare user message attachments
JSONArray messages = bodyObject.optJSONArray("messages");
- final String systemPrefix = sb.getConfig("ai.llm-system-prefix", LLM_SYSTEM_PREFIX_DEFAULT);
final String userPrefix = sb.getConfig("ai.llm-user-prefix", LLM_USER_PREFIX_DEFAULT);
// debug
@@ -206,26 +166,40 @@ public class RAGProxyServlet extends HttpServlet {
userObject.attachAttachment(userPrefix);
}
}
- UserObject userObject = new UserObject(messages.getJSONObject(messages.length() - 1));
- String user = userObject.getContentText(); // this is the latest prompt
- final String userPrompt = user;
- String ragMode = userObject.getSearchMode();
+ UserObject userObject = null;
+ String user = "";
+ String ragMode = "no";
+ String userPrompt = "";
+ int lastUserIndex = -1;
+ for (int i = messages.length() - 1; i >= 0; i--) {
+ JSONObject message = messages.getJSONObject(i);
+ if ("user".equals(message.optString("role", ""))) {
+ lastUserIndex = i;
+ break;
+ }
+ }
+ if (lastUserIndex >= 0) {
+ userObject = new UserObject(messages.getJSONObject(lastUserIndex));
+ user = userObject.getContentText(); // this is the latest user prompt
+ userPrompt = user;
+ ragMode = userObject.getSearchMode();
+ }
ConcurrentLog.info("RAGProxy", "ragMode=" + ragMode + " userChars=" + (user == null ? 0 : user.length()));
//List<DataURL> data_urls = userObject.getContentAttachments(); // this list is a copy of the content data_urls
// RAG
String searchResultQuery = "";
String searchResultMarkdown = "";
- if (!"no".equals(ragMode)) {
+ if (userObject != null && !"no".equals(ragMode)) {
// modify system and user prompt here in bodyObject to enable RAG
final String queryPrefix = sb.getConfig("ai.llm-query-generator-prefix", LLM_QUERY_GENERATOR_PREFIX_DEFAULT);
final long queryStart = System.currentTimeMillis();
- searchResultQuery = this.searchWordsForPrompt(llm4tldr.llm, llm4tldr.model, queryPrefix + user); // might return null in case any error occurred
+ searchResultQuery = RAGAugmentor.searchWordsForPrompt(llm4tldr.llm, llm4tldr.model, queryPrefix + user); // might return null in case any error occurred
if (searchResultQuery == null || searchResultQuery.length() == 0) searchResultQuery = user; // in case there is an error we simply search with the prompt
final long queryElapsed = System.currentTimeMillis() - queryStart;
- final Set<String> boostTerms = intersectTokens(userPrompt, searchResultQuery, 8);
+ final Set<String> boostTerms = RAGAugmentor.intersectTokens(userPrompt, searchResultQuery, 8);
final long searchStart = System.currentTimeMillis();
- searchResultMarkdown = searchResultsAsMarkdown(searchResultQuery, 10, "global".equals(ragMode), boostTerms);
+ searchResultMarkdown = RAGAugmentor.searchResultsAsMarkdown(searchResultQuery, 10, "global".equals(ragMode), boostTerms);
final long searchElapsed = System.currentTimeMillis() - searchStart;
ConcurrentLog.info(
"RAGProxy",
@@ -236,76 +210,22 @@ public class RAGProxyServlet extends HttpServlet {
userObject.setContentText(user);
}
- // write back modified bodyMap to body
- body = bodyObject.toString();
-
- // Open request to back-end service
- final URL url = new URI(llm4Chat.llm.hoststub + "/v1/chat/completions").toURL();
- final HttpURLConnection conn = (HttpURLConnection) url.openConnection();
- conn.setRequestMethod("POST");
- conn.setRequestProperty("Content-Type", "application/json");
- if (!llm4Chat.llm.api_key.isEmpty()) {
- conn.setRequestProperty("Authorization", "Bearer " + llm4Chat.llm.api_key);
+ JSONObject initialMetadata = null;
+ if (searchResultMarkdown.length() > 0) {
+ initialMetadata = new JSONObject(true);
+ initialMetadata.put("search-filename", "search_result_" + searchResultQuery.replace(' ', '_') + ".md");
+ initialMetadata.put("search-text-base64", new String(Base64.getEncoder().encode(searchResultMarkdown.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8));
}
- conn.setDoOutput(true);
-
- // write the body to back-end LLM
- try (OutputStream os = conn.getOutputStream()) {
- os.write(body.getBytes());
- os.flush();
- } // here we wait for the response from upstream
- // write back response of the back-end service to the client; use status of
- // backend-response
- final int status = conn.getResponseCode();
- // String rmessage = conn.getResponseMessage();
+ // ToolCallProtocol owns request preparation, initial stream handling and follow-up tool rounds.
+ final int status = ToolCallProtocol.proxyToolLifecycle(out, llm4Chat, bodyObject, messages, initialMetadata);
hresponse.setStatus(status);
-
- if (status == 200) {
- final BlockingQueue<String> inputQueue = new LinkedBlockingQueue<>();
- final String POISON = "POISON";
- Thread readerThread = new Thread(() -> {
- // read the response of the back-end line-by-line and push it to a stack concurrently
- try {
- final BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream()));
- String inputLine;
- while ((inputLine = in.readLine()) != null) {inputQueue.put(inputLine);}
- in.close();
- inputQueue.put(POISON);
- } catch (IOException | InterruptedException e) {
- } finally {
- try {inputQueue.put(POISON);} catch (InterruptedException e) {}
- }
- });
- readerThread.start();
-
- // read the stack line-by-line and write it to the client line-by-line
- try {
- String inputLine;
- int count = 0;
- while (!(inputLine = inputQueue.take()).equals(POISON)) {
- if (count == 0 && searchResultMarkdown.length() > 0) {
- // for the first line we modify the data line to integrate the search result as file
- int p = inputLine.indexOf('{');
- if (p > 0) {
- JSONObject j = new JSONObject(new JSONTokener(inputLine.substring(p)));
- j.put("search-filename", "search_result_"+ searchResultQuery.replace(' ', '_') + ".md");
- j.put("search-text-base64", new String(Base64.getEncoder().encode(searchResultMarkdown.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8));
- inputLine = inputLine.substring(0, p) + j.toString();
- }
- }
- out.println(inputLine); // i.e. data: {"id":"chatcmpl-69","object":"chat.completion.chunk","created":1715908287,"model":"llama3:8b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"ߘŠ"},"finish_reason":null}]}
- out.flush();
- count++;
- }
- } catch (InterruptedException e) {}
- }
out.close(); // close this here to end transmission
- } catch (JSONException | URISyntaxException e) {
+ } catch (JSONException e) {
throw new IOException(e.getMessage());
}
}
-
+
public final static class DataURL {
private String mimetype;
private byte[] data;
@@ -482,495 +402,6 @@ public class RAGProxyServlet extends HttpServlet {
}
}
}
-
- public static JSONArray searchResults(String query, int count, final boolean includeSnippet) {
- return searchResults(query, count, includeSnippet, new LinkedHashSet<>());
- }
-
- public static JSONArray searchResults(String query, int count, final boolean includeSnippet, final Set<String> boostTerms) {
- final JSONArray results = new JSONArray();
- if (query == null || query.length() == 0 || count == 0) return results;
- Switchboard sb = Switchboard.getSwitchboard();
- EmbeddedSolrConnector connector = sb.index.fulltext().getDefaultEmbeddedConnector();
- // construct query
- final SolrQuery params = new SolrQuery();
- params.setQuery(query);
- params.set("defType", "edismax");
- params.set("qf",
- CollectionSchema.title.getSolrFieldName() + "^3 " +
- CollectionSchema.text_t.getSolrFieldName() + "^1 " +
- CollectionSchema.sku.getSolrFieldName() + "^0.5 " +
- CollectionSchema.h1_txt.getSolrFieldName() + "^2");
- params.set("pf",
- CollectionSchema.title.getSolrFieldName() + "^5 " +
- CollectionSchema.text_t.getSolrFieldName() + "^2");
- //params.set("mm", "2<75%"); // using mm is too strict; in many cases we don't get any hits
- final List<String> bqParts = new ArrayList<>();
- if (boostTerms != null && !boostTerms.isEmpty()) {
- for (String term : boostTerms) {
- if (term == null || term.isEmpty()) continue;
- bqParts.add(CollectionSchema.title.getSolrFieldName() + ":" + term + "^0.5");
- bqParts.add(CollectionSchema.h1_txt.getSolrFieldName() + ":" + term + "^0.4");
- bqParts.add(CollectionSchema.text_t.getSolrFieldName() + ":" + term + "^0.2");
- }
- }
- bqParts.add("(" + CollectionSchema.url_file_ext_s.getSolrFieldName() + ":(zip rar 7z tar gz bz2 xz tgz))^0.1");
- params.set("bq", String.join(" ", bqParts));
- params.setRows(count);
- params.setStart(0);
- params.setFacet(false);
- params.clearSorts();
- params.setFields(
- CollectionSchema.sku.getSolrFieldName(), CollectionSchema.title.getSolrFieldName(), CollectionSchema.text_t.getSolrFieldName(),
- CollectionSchema.description_txt.getSolrFieldName(), CollectionSchema.keywords.getSolrFieldName(), CollectionSchema.synonyms_sxt.getSolrFieldName(),
- CollectionSchema.h1_txt.getSolrFieldName(), CollectionSchema.h2_txt.getSolrFieldName(), CollectionSchema.h3_txt.getSolrFieldName(),
- CollectionSchema.h4_txt.getSolrFieldName(), CollectionSchema.h5_txt.getSolrFieldName(), CollectionSchema.h6_txt.getSolrFieldName()
- );
- params.setIncludeScore(true);
- params.set("df", CollectionSchema.text_t.getSolrFieldName());
-
- // query the server
- try {
- final SolrDocumentList sdl = connector.getDocumentListByParams(params);
- Iterator<SolrDocument> i = sdl.iterator();
- while (i.hasNext()) {
- try {
- SolrDocument doc = i.next();
- final JSONObject result = new JSONObject(true);
- String url = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName());
- result.put("url", url == null ? "" : url.trim());
- String title = getOneString(doc, CollectionSchema.title);
- result.put("title", title == null ? "" : title.trim());
- if (includeSnippet) {
- String text = (String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName());
- result.put("text", limitSnippet(text == null ? "" : text.trim(), 2000));
- }
- results.put(result);
- } catch (JSONException e) {
- // skip this result
- }
- }
- return results;
- } catch (SolrException | IOException e) {
- return results;
- }
- }
-
- public static String searchResultsAsMarkdown(String query, int count, boolean global) {
- return searchResultsAsMarkdown(query, count, global, new LinkedHashSet<>());
- }
-
- public static String searchResultsAsMarkdown(String query, int count, boolean global, final Set<String> boostTerms) {
- final long searchStart = System.currentTimeMillis();
- JSONArray searchResults = global ? searchResultsGlobal(query, count, true) : searchResults(query, count, true, boostTerms);
- ConcurrentLog.info("RAGProxy", "searchResults=" + searchResults.length() + " global=" + global + " searchMs=" + (System.currentTimeMillis() - searchStart));
- StringBuilder sb = new StringBuilder();
-
- // collect snippets
- List<Snippet> results = new ArrayList<>();
- for (int i = 0; i < searchResults.length(); i++) {
- try {
- JSONObject r = searchResults.getJSONObject(i);
- String title = r.optString("title", "");
- String url = r.optString("url", "");
- String text = r.optString("text", "");
- if (title.isEmpty()) title = url;
- if (text.isEmpty()) text = title;
- if (title.length() > 0 && text.length() > 0) {
- Snippet snippet = new Snippet(query, text, url, title, 256); // we always compute a snippet because that gives us a hint if the query appears at all
- if (snippet.getText().length() > 0) results.add(snippet);
- }
- } catch (JSONException e) {}
- }
-
- // sort snippets again by score
- results.sort(Comparator.comparingDouble(Snippet::getScore));
-
- int limit = results.size() / 2;
- if (results.size() > 0 && limit == 0) limit = 1;
- for (int i = 0; i < limit; i++) {
- Snippet snippet = results.get(i);
- sb.append("## ").append(snippet.getTitle()).append("\n");
- sb.append(snippet.text).append("\n");
- if (snippet.getURL().length() > 0) sb.append("Source: ").append(snippet.getURL()).append("\n");
- sb.append("\n\n");
- }
-
- ConcurrentLog.info("RAGProxy", "markdownChars=" + sb.length() + " snippetCount=" + results.size());
- return sb.toString();
- }
-
- public static JSONArray searchResultsGlobal(String query, int count, final boolean includeSnippet) {
- final JSONArray results = new JSONArray();
- if (query == null || query.length() == 0 || count == 0) return results;
- final Switchboard sb = Switchboard.getSwitchboard();
- final RankingProfile ranking = sb.getRanking();
- final int timezoneOffset = 0;
- final QueryModifier modifier = new QueryModifier(timezoneOffset);
- String querystring = modifier.parse(query);
- if (querystring.length() == 0) {
- querystring = query == null ? "" : query.trim();
- }
- if (querystring.length() == 0) return results;
- final QueryGoal qg = new QueryGoal(querystring);
- final QueryParams theQuery = new QueryParams(
- qg,
- modifier,
- 0,
- "",
- Classification.ContentDomain.TEXT,
- "",
- timezoneOffset,
- new HashSet<Tagging.Metatag>(),
- CacheStrategy.IFFRESH,
- count,
- 0,
- ".*",
- null,
- null,
- QueryParams.Searchdom.GLOBAL,
- null,
- true,
- DigestURL.hosthashess(sb.getConfig("search.excludehosth", "")),
- MultiProtocolURL.TLD_any_zone_filter,
- null,
- false,
- sb.index,
- ranking,
- ClientIdentification.yacyIntranetCrawlerAgent.userAgent(),
- 0.0d,
- 0.0d,
- 0.0d,
- sb.getConfigSet("search.navigation"));
- final SearchEvent theSearch = SearchEventCache.getEvent(
- theQuery,
- sb.peers,
- sb.tables,
- (sb.isRobinsonMode()) ? sb.clusterhashes : null,
- false,
- sb.loader,
- (int) sb.getConfigLong(
- SwitchboardConstants.REMOTESEARCH_MAXCOUNT_USER,
- sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXCOUNT_DEFAULT, 10)),
- sb.getConfigLong(
- SwitchboardConstants.REMOTESEARCH_MAXTIME_USER,
- sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000)));
- final long timeout = sb.getConfigLong(
- SwitchboardConstants.REMOTESEARCH_MAXTIME_USER,
- sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000));
- waitForFeedingAndResort(theSearch, timeout);
- for (int i = 0; i < count; i++) {
- URIMetadataNode node = theSearch.oneResult(i, timeout);
- if (node == null) break;
- try {
- final JSONObject result = new JSONObject(true);
- result.put("url", node.urlstring());
- result.put("title", node.title());
- if (includeSnippet) {
- String text = node.snippet();
- if (text == null || text.isEmpty()) {
- TextSnippet snippet = node.textSnippet();
- if (snippet != null && snippet.exists() && !snippet.getErrorCode().fail()) {
- text = snippet.getLineRaw();
- }
- }
- if (text == null || text.isEmpty()) {
- text = firstFieldString(node.getFieldValue(CollectionSchema.description_txt.getSolrFieldName()));
- }
- if (text == null || text.isEmpty()) {
- text = firstFieldString(node.getFieldValue(CollectionSchema.text_t.getSolrFieldName()));
- }
- result.put("text", limitSnippet(text == null ? "" : text.trim(), 2000));
- }
- results.put(result);
- } catch (JSONException e) {
- // skip this result
- }
- }
- return results;
- }
-
- private static String limitSnippet(String text, int maxChars) {
- if (text == null) return "";
- if (maxChars <= 0 || text.length() <= maxChars) return text;
- return text.substring(0, maxChars);
- }
-
- private static String firstFieldString(Object value) {
- if (value == null) return "";
- if (value instanceof Collection) {
- for (Object item : (Collection<?>) value) {
- if (item != null) return item.toString();
- }
- return "";
- }
- return value.toString();
- }
-
- private static void waitForFeedingAndResort(SearchEvent search, long timeoutMs) {
- if (search == null || timeoutMs <= 0) return;
- final long end = System.currentTimeMillis() + timeoutMs;
- while (!search.isFeedingFinished() && System.currentTimeMillis() < end) {
- try {
- Thread.sleep(100);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- break;
- }
- }
- search.resortCachedResults();
- }
-
-
- public static class Snippet {
-
- private String text, url, title;
- private double score;
-
- /**
- * Find a snippet inside a given text that contains most of the searched words plus some context.
- * @param query a string with a search query; query words are separated by space
- * @param text the text where we want to find the snippets
- * @param maxChunkLength the maximum length of a single chunk; however the snippet is three times as this.
- * @return one string containing the snippet.
- */
- public Snippet(String query, String text, String url, String title, int maxChunkLength) {
- this.url = url;
- this.title = title;
- this.score = 0.0;
-
- if (text == null || text.isEmpty() || maxChunkLength <= 0 || query == null) {
- this.text = "";
- return;
- }
-
- // Step 1: Slice text and make copy with lowercase version to support tf*idf computation
- List<String> chunks = slicer(text, maxChunkLength);
- if (chunks.isEmpty()) {
- this.text = "";
- return;
- }
- List<String> chunksLowerCase = new ArrayList<>(chunks.size());
- for (String chunk: chunks) chunksLowerCase.add(chunk.toLowerCase());
-
- // Step 2: Preprocess query
- Set<String> queryWordSet = querySet(query);
- if (queryWordSet.isEmpty()) {
- this.text = "";
- return;
- }
-
- // Step 3: Compute IDF
- // IDF uses a logarithm because the information gain of rare words grows non-linearly;
- // the log dampens extreme ratios (N/df), stabilizes TF-IDF values, and matches the
- // information-theoretic definition of word informativeness.
- int totalChunks = chunksLowerCase.size();
- Map<String, Double> idf = new HashMap<>();
- for (String word: queryWordSet) {
- int docFreq = 0;
- for (String chunk: chunksLowerCase) {
- if (chunk.contains(word)) docFreq++;
- }
- idf.put(word, Math.log((double) totalChunks / (docFreq + 1)) + 1);
- }
-
- // Step 4: Score chunks
- Map<Integer, Double> chunkScores = new HashMap<>();
- for (int i = 0; i < chunksLowerCase.size(); i++) {
- String chunk = chunksLowerCase.get(i);
- double score = 0.0;
- Map<String, Integer> tf = new HashMap<>(); // counts occurrence in query for each word in chunk
-
- // Extract words and clean
- String[] wordsInChunk = chunk.split("\\s+");
- for (String w : wordsInChunk) {
- String cleanWord = w.replaceAll("[.,!?;:]", "");
- if (cleanWord.length() > 0 && queryWordSet.contains(cleanWord)) {
- tf.put(cleanWord, tf.getOrDefault(cleanWord, 0) + 1);
- }
- }
-
- // Sum TF-IDF
- for (String word: queryWordSet) {
- int tfValue = tf.getOrDefault(word, 0);
- double tfIdf = (double) tfValue * idf.getOrDefault(word, 1.0);
- score += tfIdf;
- }
- chunkScores.put(i, score);
- }
-
- // Step 5: Find best chunk
- int topChunkIndex = -1;
- for (Map.Entry<Integer, Double> entry: chunkScores.entrySet()) {
- if (entry.getValue() > this.score) {
- this.score = entry.getValue();
- topChunkIndex = entry.getKey();
- }
- }
-
- // if there is no best chunk, return an empty snippet
- if (topChunkIndex < 0) {
- this.text = "";
- this.score = 0.0;
- return;
- }
-
- // Step 6: Get 3-slice snippet
- List<String> snippetChunks = new ArrayList<>();
- if (topChunkIndex > 0) {
- snippetChunks.add(chunks.get(topChunkIndex - 1));
- }
- snippetChunks.add(chunks.get(topChunkIndex));
- if (topChunkIndex < chunks.size() - 1) {
- snippetChunks.add(chunks.get(topChunkIndex + 1));
- }
-
- // Step 7: Join
- this.text = String.join(" ", snippetChunks);
- }
-
- public double getScore() {
- return this.score;
- }
-
- public String getText() {
- return this.text;
- }
-
- public String getURL() {
- return this.url;
- }
-
- public String getTitle() {
- return this.title;
- }
- }
-
- /**
- * Creates slices of a given text. We want slices of average same size,
- * but we want to prevent that cuts are made within sentences.
- * @param text the given text
- * @param len the minimum length of the wanted slices; actual slices may be longer
- * @return a list of text slices
- */
- public static List<String> slicer(String text, int len) {
- List<String> result = new ArrayList<>();
- if (text == null || len <= 0) return result;
-
- int start = 0;
- while (start < text.length()) {
- int end = Math.min(start + len, text.length());
-
- // Move end position further out until a a sentence end is found:
- // look for sentence boundary: .!?, followed by whitespace char.
- while (end < text.length()) {
- char ch = text.charAt(end - 1);
- if ((ch == '.' || ch == '?' || ch == '!') && Character.isWhitespace(text.charAt(end))) {
- break;
- }
- end++;
- }
- result.add(text.substring(start, end));
- start = end;
- }
-
- return result;
- }
-
- private static String getOneString(SolrDocument doc, CollectionSchema field) {
- assert field.isMultiValued();
- assert field.getType() == SolrType.string || field.getType() == SolrType.text_general;
- Object r = doc.getFieldValue(field.getSolrFieldName());
- if (r == null) return "";
- if (r instanceof ArrayList) {
- return (String) ((ArrayList<?>) r).get(0);
- }
- return r.toString();
- }
-
- private String searchWordsForPrompt(LLM llm, String model, String prompt) {
- final String question = prompt == null ? "" : prompt;
- if (llm == null || model == null || model.isEmpty()) {
- return null;
- }
- try {
- LLM.Context context = new LLM.Context(LLM_SYSTEM_PREFIX_DEFAULT);
- context.addPrompt(question);
- Set<String> singlewords = new LinkedHashSet<>();
- String[] a = LLM.stringsFromChat(llm.chat(model, context, LLM.listSchema, 200));
- if (a == null || a.length == 0) return null;
- // unfortunately this might not be a single word per line but several words; we collect them all.
- for (String s: a) {
- if (s == null) continue;
- for (String t: s.split(" ")) {
- if (!t.isEmpty()) singlewords.add(t.toLowerCase());
- }
- }
- if (singlewords.isEmpty()) {
- return null;
- }
- StringBuilder query = new StringBuilder();
- for (String s: singlewords) query.append(s).append(' ');
- String querys = query.toString().trim();
- if (querys.length() == 0) return null;
- return querys;
- } catch (IOException | JSONException e) {
- e.printStackTrace();
- return null;
- }
- }
-
- private static Set<String> querySet(String query) {
- Set<String> queryWordSet = Arrays.stream(query.trim().toLowerCase().split("\\s+"))
- .map(String::toLowerCase)
- .filter(word -> !word.isEmpty())
- .collect(Collectors.toSet());
- return queryWordSet;
- }
-
- private static Set<String> intersectTokens(String originalPrompt, String computedQuery, int maxTerms) {
- Set<String> promptTerms = querySet(originalPrompt == null ? "" : originalPrompt);
- Set<String> queryTerms = querySet(computedQuery == null ? "" : computedQuery);
- Set<String> intersection = new LinkedHashSet<>();
- for (String term : promptTerms) {
- if (!queryTerms.contains(term)) continue;
- final String cleaned = cleanToken(term);
- if (cleaned.isEmpty()) continue;
- intersection.add(cleaned);
- if (maxTerms > 0 && intersection.size() >= maxTerms) break;
- }
- return intersection;
- }
-
- private static String cleanToken(String term) {
- if (term == null) return "";
- String cleaned = term.replaceAll("[^A-Za-z0-9]", "");
- if (cleaned.length() < 2) return "";
- return cleaned.toLowerCase();
- }
-
- private static JSONObject responseLine(String payload) {
- JSONObject j = new JSONObject(true);
- try {
- j.put("id", "log");
- j.put("object", "chat.completion.chunk");
- j.put("created", System.currentTimeMillis() / 1000);
- j.put("model", "log");
- j.put("system_fingerprint", "YaCy");
- JSONArray choices = new JSONArray();
- JSONObject choice = new JSONObject(true); // {"index":0,"delta":{"role":"assistant","content":"ߘŠ"
- choice.put("index", 0);
- JSONObject delta = new JSONObject(true);
- delta.put("role", "assistant");
- delta.put("content", payload);
- choice.put("delta", delta);
- choices.put(choice);
- j.put("choices", choices);
- // j.put("finish_reason", null); // this is problematic with the JSON library
- } catch (JSONException e) {
- }
- return j;
- }
public static void pruneOldEntries(long now) {
while (true) {