summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMichael Peter Christen <mc@yacy.net>2026-07-04 10:57:22 +0200
committerMichael Peter Christen <mc@yacy.net>2026-07-04 10:57:22 +0200
commita8b90eb3653cea8322a9e0101eda6c14c3f25cc8 (patch)
tree721d222657f6d463be40fd0ae30a847f821cb0a8
parent177cc87b8967fc3f25b2f22e88611b99e61c02aa (diff)
parente1c988b5c38972ae855efdabbc5e24586e938733 (diff)
Merge branch 'master' of https://github.com/yacy/yacy_search_server.git
-rw-r--r--defaults/yacy.init25
-rw-r--r--defaults/yacy.logging2
-rw-r--r--htroot/AILab.html20
-rw-r--r--htroot/LLMSelection_p.html16
-rw-r--r--htroot/LogReports_p.html141
-rw-r--r--htroot/ViewLog_p.html21
-rw-r--r--htroot/api/logreports.json17
-rw-r--r--htroot/api/logreports.rss24
-rw-r--r--htroot/env/templates/submenuAI.template1
-rw-r--r--htroot/env/templates/submenuComputation.template1
-rw-r--r--source/net/yacy/ai/LLM.java50
-rw-r--r--source/net/yacy/ai/LogReportService.java732
-rw-r--r--source/net/yacy/ai/RAGAugmentor.java72
-rw-r--r--source/net/yacy/ai/ToolCallProtocol.java57
-rw-r--r--source/net/yacy/cora/util/LogRedaction.java54
-rw-r--r--source/net/yacy/htroot/AILab.java2
-rw-r--r--source/net/yacy/htroot/LLMSelection_p.java7
-rw-r--r--source/net/yacy/htroot/LogReports_p.java94
-rw-r--r--source/net/yacy/htroot/ViewLog_p.java139
-rw-r--r--source/net/yacy/htroot/api/logreports.java64
-rw-r--r--source/net/yacy/http/servlets/RAGProxyServlet.java46
-rw-r--r--source/net/yacy/kelondro/logging/GuiHandler.java2
-rw-r--r--source/net/yacy/yacy.java63
23 files changed, 1523 insertions, 127 deletions
diff --git a/defaults/yacy.init b/defaults/yacy.init
index a08516364..9afd2c82c 100644
--- a/defaults/yacy.init
+++ b/defaults/yacy.init
@@ -1437,14 +1437,23 @@ decoration.grafics.linkstructure = true
decoration.hostanalysis = false
decoration.simpleheadernavbar = navbar-default
-# ai settings
-ai.production_models = []
-ai.system-prompt = You are a smart and helpful chatbot. If possible, use friendly emojies.
-ai.llm-system-prefix = \n\nYou may receive additional expert knowledge in the user prompt after a 'Additional Information' headline to enhance your knowledge. Use it only if applicable.
-ai.llm-user-prefix = \n\nAdditional Information:\n\nbelow you find a collection of texts that might be useful to generate a response. Do not discuss these documents, just use them to answer the question above.\n\n
-ai.llm-query-generator-prefix = Make a list of search words with low document frequency for the following prompt; use a JSON Array:
-ai.shield.allow-nonlocalhost = false
-ai.shield.show-chat-link = false
+# ai settings
+ai.production_models = []
+ai.system-prompt = You are a smart and helpful chatbot. If possible, use friendly emojies.
+ai.llm-system-prefix = \n\nYou may receive additional expert knowledge in the user prompt after a 'Additional Information' headline to enhance your knowledge. Use it only if applicable.
+ai.llm-user-prefix = \n\nAdditional Information:\n\nbelow you find a collection of texts that might be useful to generate a response. Do not discuss these documents, just use them to answer the question above.\n\n
+ai.llm-query-generator-prefix = Make a list of search words with low document frequency for the following prompt; use a JSON Array:
+ai.logreport.enabled = true
+ai.logreport.dir = DATA/REPORTS/log
+ai.logreport.initial_delay_minutes = 5
+ai.logreport.period_minutes = 60
+ai.logreport.max_bucket_lines = 100000
+ai.logreport.max_tokens = 4096
+ai.logreport.daily_compression.enabled = true
+ai.logreport.daily_compression.delete_hourly = false
+ai.logreport.feed.max_entries = 100
+ai.shield.allow-nonlocalhost = false
+ai.shield.show-chat-link = false
ai.shield.rate.per-minute = 4
ai.shield.rate.per-hour = 60
ai.shield.rate.per-day = 120
diff --git a/defaults/yacy.logging b/defaults/yacy.logging
index 58703ebca..bfed68cae 100644
--- a/defaults/yacy.logging
+++ b/defaults/yacy.logging
@@ -61,7 +61,7 @@ net.yacy.crawler.robots.RobotsTxt.level = SEVERE
# Properties for the GuiHandler
net.yacy.kelondro.logging.GuiHandler.level = ALL
net.yacy.kelondro.logging.GuiHandler.formatter=net.yacy.kelondro.logging.SimpleLogFormatter
-net.yacy.kelondro.logging.GuiHandler.size = 10000
+net.yacy.kelondro.logging.GuiHandler.size = 100000
# Properties for the ConsoleOutErrHandler
net.yacy.kelondro.logging.ConsoleOutErrHandler.level = ALL
diff --git a/htroot/AILab.html b/htroot/AILab.html
index 1b52b4817..78e01a404 100644
--- a/htroot/AILab.html
+++ b/htroot/AILab.html
@@ -331,6 +331,26 @@
</div>
</div>
+ <div class="bs-callout bs-callout-optional quest-callout status-pending" data-quest="logreports" data-status="#[ailab_logreports_status]#">
+ <div class="quest-meta">
+ <span class="label label-info">Optional</span>
+ <span class="meta-arrow" aria-hidden="true">→</span>
+ <span class="status-pill">Needs setup</span>
+ </div>
+ <h3>Monitor log reports</h3>
+ <p>Assign a log-report model, then review generated hourly and daily self-enhancement reports.</p>
+ <div class="quest-body">
+ <div class="quest-visual">
+ <img src="env/grafics/AILab_Matrix.png" alt="Log report monitor" width="128" height="128" />
+ </div>
+ <div class="quest-actions">
+ <a class="btn btn-info btn-sm" href="LogReports_p.html">Open log reports</a>
+ <a class="btn btn-info btn-sm" href="LLMSelection_p.html#availableModels">Assign log-report model</a><br />
+ <span class="quest-note">Report generation stays inactive until a production model is assigned to the log-report role.</span>
+ </div>
+ </div>
+ </div>
+
<div class="bs-callout bs-callout-optional quest-callout status-pending" data-quest="shield" data-status="#[ailab_shield_status]#">
<div class="quest-meta">
<span class="label label-info">Optional</span>
diff --git a/htroot/LLMSelection_p.html b/htroot/LLMSelection_p.html
index 6e3e54f22..628ab1211 100644
--- a/htroot/LLMSelection_p.html
+++ b/htroot/LLMSelection_p.html
@@ -53,12 +53,12 @@
required: ["mood"]
};
- const PRODUCTION_MODEL_TOTAL_COLUMNS = 17;
+ const PRODUCTION_MODEL_TOTAL_COLUMNS = 18;
const PRODUCTION_MODEL_MODEL_COLUMN_INDEX = 1;
const PRODUCTION_MODEL_USAGE_COLUMN_START = 5;
- const PRODUCTION_MODEL_USAGE_COLUMN_END = 11; // including
- const PRODUCTION_MODEL_FEATURE_COLUMN_START = 12;
- const PRODUCTION_MODEL_FEATURE_COLUMN_END = 15; // including
+ const PRODUCTION_MODEL_USAGE_COLUMN_END = 12; // including
+ const PRODUCTION_MODEL_FEATURE_COLUMN_START = 13;
+ const PRODUCTION_MODEL_FEATURE_COLUMN_END = 16; // including
const PRODUCTION_MODEL_ACTION_COLUMN_INDEX = PRODUCTION_MODEL_TOTAL_COLUMNS - 1;
const PRODUCTION_MODEL_THINKING_COLUMN_INDEX = PRODUCTION_MODEL_FEATURE_COLUMN_START;
const PRODUCTION_MODEL_TOOLING_COLUMN_INDEX = PRODUCTION_MODEL_FEATURE_COLUMN_START + 1;
@@ -66,7 +66,8 @@
const PRODUCTION_MODEL_FORMAT_COLUMN_INDEX = PRODUCTION_MODEL_FEATURE_COLUMN_START + 3;
const PRODUCTION_MODEL_ENABLED_USAGE_COLUMNS = new Set([
6, // chat
- 11 // tldr
+ 11, // tldr
+ 12 // log-report
]);
const PRODUCTION_MODEL_COLUMN_NAMES = [
"service",
@@ -82,6 +83,7 @@
"query",
"qapairs",
"tldr",
+ "logreport",
"thinking",
"tooling",
@@ -111,7 +113,7 @@
//["qwen3-vl:30b-a3b-instruct-q4_K_M", "17.33", "22GB", "Very fast, exceptional good 30B model, ranking above GPT-4-turbo, GPT-4.1-nano, GPT-o1, GPT-4o-mini", "Alibaba", "apache-2.0"],
["qwen3.5:9b-q4_K_M", "18.37", "14GB", "multimodal, outstanding for its size, long-context 256K Tokens, strong instruction following model", "Alibaba", "apache-2.0"],
["hf.co/mradermacher/Ling-mini-2.0-GGUF:Q4_K_M", "20.00", "20GB", "very fast 16B MoE model with 1.4B activated parameters per expert", "InclusionAI", "mit"],
- ["qwen3.5:35b-a3b-q4_K_M", "44.63", "25GB", "Very fast MoE, exceptional good 35B model with vision, ranking above GPT-4-turbo, GPT-4.1-nano, GPT-o1, GPT-4o-mini", "Alibaba", "apache-2.0"]
+ ["qwen3.6:27b-mtp-q4_K_M", "80", "17GB", "Exceptional good 27B model with vision, ranking above GPT-4-turbo, GPT-4.1-nano, GPT-o1, GPT-4o-mini", "Alibaba", "apache-2.0"]
];
const MODEL_TABLE_HEADERS = ["Model", "Ranking", "Size", "Description", "Provider", "License", "Actions"];
@@ -1871,6 +1873,7 @@
<td class="narrow">search-query<br/><span class="info"><img src="env/grafics/i16.gif" width="16" height="16" alt="info"/><span>This model produces search queries to YaCy search from prompts in RAG or chat</span></span></td>
<td class="narrow">qa-pairs<br/><span class="info"><img src="env/grafics/i16.gif" width="16" height="16" alt="info"/><span>This model can be used to produce query-answer pairs which enhance search from chat prompts</span></span></td>
<td class="narrow">tldr-shortener<br/><span class="info"><img src="env/grafics/i16.gif" width="16" height="16" alt="info"/><span>This model is used to make summaries from web content</span></span></td>
+ <td class="narrow">log-report<br/><span class="info"><img src="env/grafics/i16.gif" width="16" height="16" alt="info"/><span>This model evaluates YaCy runtime logs and creates self-enhancement reports</span></span></td>
<td class="narrow">thinking<br/><span class="info"><img src="env/grafics/i16.gif" width="16" height="16" alt="info"/><span>we detect thinking only to be able to suppress thinking. thinking is not used in YaCy</span></span></td>
<td class="narrow">tooling<br/><span class="info"><img src="env/grafics/i16.gif" width="16" height="16" alt="info"/><span>tooling is required for agentic abilities.</span></span></td>
@@ -1895,6 +1898,7 @@
<td><input type="checkbox" #(query)#::checked=true#(/query)# disabled="disabled"></td>
<td><input type="checkbox" #(qapairs)#::checked=true#(/qapairs)# disabled="disabled"></td>
<td><input type="checkbox" #(tldr)#::checked=true#(/tldr)#></td>
+ <td><input type="checkbox" #(logreport)#::checked=true#(/logreport)#></td>
<td>#[thinking]#</td>
<td>#[tooling]#</td>
diff --git a/htroot/LogReports_p.html b/htroot/LogReports_p.html
new file mode 100644
index 000000000..a99a20eb4
--- /dev/null
+++ b/htroot/LogReports_p.html
@@ -0,0 +1,141 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+ <head>
+ <title>YaCy '#[clientname]#': Log Reports</title>
+ #%env/templates/metas.template%#
+ <style type="text/css">
+ .logreport-feeds {
+ margin: 0 0 16px 0;
+ }
+ .logreport-card {
+ border: 1px solid #ddd;
+ margin: 0 0 16px 0;
+ background: #fff;
+ }
+ .logreport-header {
+ background: #f5f5f5;
+ border-bottom: 1px solid #ddd;
+ padding: 10px 12px;
+ }
+ .logreport-title {
+ font-weight: bold;
+ margin: 0 0 4px 0;
+ }
+ .logreport-meta {
+ color: #666;
+ font-size: 0.9em;
+ }
+ .logreport-body {
+ white-space: pre-wrap;
+ word-break: break-word;
+ max-height: 520px;
+ overflow: auto;
+ margin: 0;
+ padding: 12px;
+ background: #fff;
+ border: 0;
+ font-family: Menlo, Consolas, monospace;
+ font-size: 12px;
+ line-height: 1.45;
+ }
+ .logreport-empty {
+ border: 1px solid #ddd;
+ background: #f9f9f9;
+ padding: 14px;
+ }
+ .logreport-warning {
+ border: 1px solid #f0ad4e;
+ border-left: 6px solid #f0ad4e;
+ background: #fcf8e3;
+ color: #6b4f13;
+ padding: 12px;
+ margin: 10px 0 16px 0;
+ }
+ .logreport-actions {
+ margin: 0 0 16px 0;
+ }
+ .logreport-message {
+ border: 1px solid #5bc0de;
+ border-left: 6px solid #5bc0de;
+ background: #eef9fc;
+ color: #245269;
+ padding: 12px;
+ margin: 10px 0 16px 0;
+ }
+ </style>
+ </head>
+ <body id="LogReports">
+ #%env/templates/header.template%#
+ #%env/templates/submenuComputation.template%#
+
+ <h2>Log Reports</h2>
+
+ <form action="LogReports_p.html" method="get" accept-charset="UTF-8">
+ <fieldset>
+ <label for="count">Reports</label>
+ <input type="number" name="count" id="count" min="0" max="#[maxcount]#" value="#[count]#" />
+ <input type="submit" value="refresh" />
+ </fieldset>
+ </form>
+
+ <form action="LogReports_p.html" method="post" accept-charset="UTF-8" class="logreport-actions">
+ <input type="hidden" name="count" value="#[count]#" />
+ <button type="submit" name="runReportNow" value="1" class="btn btn-primary">run report now</button>
+ </form>
+
+ #(runReportResult)#
+ ::
+ <div class="logreport-message">Current-hour report written: #[runReportFile]# (#[runReportDurationSeconds]# seconds)</div>
+ ::
+ <div class="logreport-warning">No log lines were found for the current hour.</div>
+ ::
+ <div class="logreport-warning">
+ No production model is configured for the log-report role. Assign one in the
+ <a href="LLMSelection_p.html#availableModels">Production Models Matrix</a>.
+ </div>
+ ::
+ <div class="logreport-warning">Report generation failed after #[runReportDurationSeconds]# seconds: #[runReportFile]#</div>
+ #(/runReportResult)#
+
+ #(modelConfigured)#
+ <div class="logreport-warning">
+ No production model is configured for the log-report role. Log report generation stays inactive until a model is assigned in the
+ <a href="LLMSelection_p.html#availableModels">Production Models Matrix</a>.
+ </div>
+ ::
+ #(/modelConfigured)#
+
+ <p class="logreport-feeds">
+ Reports found: #[reportCount]#<br />
+ Directory: #[reportdir]#<br />
+ Feeds:
+ <a href="#[jsonFeed]#">JSON</a>
+ |
+ <a href="#[rssFeed]#">RSS</a>
+ </p>
+
+ #(reportdirMissing)#
+ ::
+ <div class="logreport-empty">The report directory does not exist yet. Reports will appear here after the scheduler has generated the first completed hourly report.</div>
+ #(/reportdirMissing)#
+
+ #{reports}#
+ <div class="logreport-card">
+ <div class="logreport-header">
+ <div class="logreport-title">#[title]#</div>
+ <div class="logreport-meta">
+ #[type]# report - #[filename]# - #[published]# - #[chars]# characters
+ </div>
+ </div>
+ <pre class="logreport-body">#[content]#</pre>
+ </div>
+ #{/reports}#
+
+ #(noReports)#
+ ::
+ <div class="logreport-empty">No generated log reports were found.</div>
+ #(/noReports)#
+
+ #%env/templates/footer.template%#
+ </body>
+</html>
diff --git a/htroot/ViewLog_p.html b/htroot/ViewLog_p.html
index 6ef10300d..6efb12d11 100644
--- a/htroot/ViewLog_p.html
+++ b/htroot/ViewLog_p.html
@@ -11,14 +11,19 @@
<form action="ViewLog_p.html" method="get" accept-charset="UTF-8">
<fieldset>
- <input type="text" name="lines" id="lines" size ="6" value="#[lines]#" maxlength="9" /><label for="lines"> Lines (max. #[maxlines]#)</label>
- <input type="checkbox" name="mode" id="mode" value="reversed" #(reverseChecked)#::checked="checked"#(/reverseChecked)# /><label for="mode">reversed order</label>
- <input type="text" name="filter" id="filter" size ="30" value="#[filter]#" maxlength="60" />
- <input type="submit" value="refresh" />
- </fieldset>
- </form>
-
- <div class="log">
+ <input type="text" name="lines" id="lines" size ="6" value="#[lines]#" maxlength="9" /><label for="lines"> Lines (max. #[maxlines]#)</label>
+ <input type="checkbox" name="mode" id="mode" value="reversed" #(reverseChecked)#::checked="checked"#(/reverseChecked)# /><label for="mode">reversed order</label>
+ <select name="filterMode" id="filterMode">
+ <option value="regex" #(filterModeRegex)#::selected="selected"#(/filterModeRegex)#>regex</option>
+ <option value="terms" #(filterModeTerms)#::selected="selected"#(/filterModeTerms)#>terms</option>
+ </select>
+ <input type="text" name="filter" id="filter" size ="42" value="#[filter]#" maxlength="160" />
+ <input type="submit" value="refresh" />
+ </fieldset>
+ </form>
+ #(filterError)#::<p class="error">Invalid regular expression filter.</p>#(/filterError)#
+
+ <div class="log">
#{log}#<p class="log#(level)#Debug::Info::System::Warning::Severe#(/level)#" style="font-family:Monospace;margin-top:0px;margin-bottom:0px;">#[line]#</p>#{/log}#
</div>
diff --git a/htroot/api/logreports.json b/htroot/api/logreports.json
new file mode 100644
index 000000000..4b495dada
--- /dev/null
+++ b/htroot/api/logreports.json
@@ -0,0 +1,17 @@
+{
+ #(authorized)#
+ "error": "authorization required",
+ ::
+ #(/authorized)#
+ "reports": [
+ #{reports}#
+ {
+ "filename": "#[filename]#",
+ "title": "#[title]#",
+ "published": "#[published]#",
+ "daily": #(daily)#false::true#(/daily)#,
+ "content": "#[content]#"
+ }#(comma)#::,#(/comma)#
+ #{/reports}#
+ ]
+}
diff --git a/htroot/api/logreports.rss b/htroot/api/logreports.rss
new file mode 100644
index 000000000..8a0d9f2ed
--- /dev/null
+++ b/htroot/api/logreports.rss
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0">
+<channel>
+<title>#[channel_title]#</title>
+<description>#[channel_description]#</description>
+<pubDate>#[channel_pubDate]#</pubDate>
+#(authorized)#
+<item>
+<title>Authorization required</title>
+<description>Administrative authentication is required to read YaCy log reports.</description>
+<guid isPermaLink="false">authorization-required</guid>
+</item>
+::
+#(/authorized)#
+#{reports}#
+<item>
+<title>#[title-rss]#</title>
+<description>#[description-rss]#</description>
+<pubDate>#[pubDate-rss]#</pubDate>
+<guid isPermaLink="false">#[guid-rss]#</guid>
+</item>
+#{/reports}#
+</channel>
+</rss>
diff --git a/htroot/env/templates/submenuAI.template b/htroot/env/templates/submenuAI.template
index 8265b9ff0..b8fff30a0 100644
--- a/htroot/env/templates/submenuAI.template
+++ b/htroot/env/templates/submenuAI.template
@@ -5,6 +5,7 @@
<li><a href="LLMSelection_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">LLM Selection</a></li>
<li><a href="RAGConfig_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">RAG Config</a></li>
<li><a href="ToolsConfig_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Tools Config</a></li>
+ <li><a href="LogReports_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Log Reports</a></li>
<li><a href="AIShield_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">AI Shield</a></li>
<li><a href="yacychat.html" class="MenuItemLink">Chat</a></li>
</ul>
diff --git a/htroot/env/templates/submenuComputation.template b/htroot/env/templates/submenuComputation.template
index 9b24c0226..99d1a75de 100644
--- a/htroot/env/templates/submenuComputation.template
+++ b/htroot/env/templates/submenuComputation.template
@@ -12,6 +12,7 @@
<h3>Processes</h3>
<ul class="SubMenu">
<li><a href="ViewLog_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Server Log</a></li>
+ <li><a href="LogReports_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Log Reports</a></li>
<li><a href="Threaddump_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Thread Dump</a></li>
<li><a href="PerformanceConcurrency_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Concurrent Indexing</a></li>
<li><a href="PerformanceMemory_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Memory Usage</a></li>
diff --git a/source/net/yacy/ai/LLM.java b/source/net/yacy/ai/LLM.java
index 7f74943d0..2363230ab 100644
--- a/source/net/yacy/ai/LLM.java
+++ b/source/net/yacy/ai/LLM.java
@@ -39,10 +39,13 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONTokener;
+import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.cora.util.LogRedaction;
import net.yacy.search.Switchboard;
public class LLM {
+ private static final ConcurrentLog log = new ConcurrentLog("LLM");
private static final String MODEL_CAPABILITIES_CONFIG = "ai.model_capabilities";
private static String[] STOPTOKENS = new String[]{"[/INST]", "<|im_end|>", "<|end_of_turn|>", "<|eot_id|>", "<|end_header_id|>", "<EOS_TOKEN>", "</s>", "<|end|>"};
@@ -65,7 +68,8 @@ public class LLM {
classification,
query,
qapairs,
- tldr
+ tldr,
+ logreport
}
public static class LLMModel {
@@ -99,8 +103,25 @@ public class LLM {
* @return
*/
public static LLMModel llmFromUsage(LLMUsage llmUsage) {
- Switchboard sb = Switchboard.getSwitchboard();
- String pms = sb.getConfig("ai.production_models", "[]");
+ return llmFromUsage(llmUsage, null, null);
+ }
+
+ public static LLMModel llmFromUsage(final LLMUsage llmUsage, final String runId, final String caller) {
+ return llmFromUsage(llmUsage, runId, caller, true);
+ }
+
+ public static LLMModel llmFromUsageQuiet(final LLMUsage llmUsage) {
+ return llmFromUsage(llmUsage, null, null, false);
+ }
+
+ private static LLMModel llmFromUsage(final LLMUsage llmUsage, final String runId, final String caller, final boolean logRouting) {
+ final long start = System.currentTimeMillis();
+ final Switchboard sb = Switchboard.getSwitchboard();
+ if (sb == null) {
+ if (logRouting) log.warn(routePrefix(runId, caller) + "event=model-routing phase=fail usage=" + llmUsage + " reason=switchboard-unavailable durationMs=" + elapsed(start));
+ return null;
+ }
+ final String pms = sb.getConfig("ai.production_models", "[]");
JSONObject model_capabilities = readModelCapabilities();
try {
JSONArray production_models = new JSONArray(new JSONTokener(pms));
@@ -126,15 +147,34 @@ public class LLM {
}
LLM llm = new LLM(hoststub, api_key, max_tokens, type);
LLMModel llmmodel = new LLMModel(llm, model, tooling, thinking);
+ if (logRouting) {
+ log.info(routePrefix(runId, caller) + "event=model-routing phase=select usage=" + llmUsage + " row=" + i + " service=" + type.name() + " model=" + LogRedaction.redact(model) + " backend=" + LogRedaction.redact(llm.hoststub) + " maxTokens=" + llm.max_tokens + " tooling=" + tooling + " thinking=" + thinking + " productionRows=" + production_models.length() + " durationMs=" + elapsed(start));
+ }
return llmmodel;
}
}
- } catch (JSONException | NumberFormatException e) {
- e.printStackTrace();
+ if (logRouting) {
+ log.info(routePrefix(runId, caller) + "event=model-routing phase=miss usage=" + llmUsage + " productionRows=" + production_models.length() + " durationMs=" + elapsed(start));
+ }
+ } catch (JSONException | IllegalArgumentException e) {
+ if (logRouting) {
+ log.warn(routePrefix(runId, caller) + "event=model-routing phase=fail usage=" + llmUsage + " errorClass=" + e.getClass().getName() + " reason=" + LogRedaction.redactMessage(e) + " durationMs=" + elapsed(start));
+ }
}
// so if we don't find a model for that specific usage, we purposely return null to show that there is a missing configuration
return null;
}
+
+ private static String routePrefix(final String runId, final String caller) {
+ final StringBuilder prefix = new StringBuilder();
+ if (runId != null && !runId.isEmpty()) prefix.append("runId=").append(runId).append(' ');
+ if (caller != null && !caller.isEmpty()) prefix.append("caller=").append(caller).append(' ');
+ return prefix.toString();
+ }
+
+ private static long elapsed(final long start) {
+ return System.currentTimeMillis() - start;
+ }
public String getHoststub() {
return this.hoststub;
diff --git a/source/net/yacy/ai/LogReportService.java b/source/net/yacy/ai/LogReportService.java
new file mode 100644
index 000000000..ddee5db9c
--- /dev/null
+++ b/source/net/yacy/ai/LogReportService.java
@@ -0,0 +1,732 @@
+/**
+ * LogReportService
+ * Copyright 2026 by contributors to the YaCy project
+ * First released 26.06.2026 at https://yacy.net
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program in the file lgpl21.txt
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.ai;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.AtomicMoveNotSupportedException;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.LocalTime;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.UUID;
+import java.util.logging.Handler;
+import java.util.logging.Logger;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import net.yacy.ai.LLM.LLMModel;
+import net.yacy.ai.LLM.LLMUsage;
+import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.cora.util.LogRedaction;
+import net.yacy.kelondro.logging.GuiHandler;
+import net.yacy.search.Switchboard;
+
+public class LogReportService {
+
+ public static final String CONFIG_REPORT_DIR = "ai.logreport.dir";
+ public static final String CONFIG_ENABLED = "ai.logreport.enabled";
+ public static final String CONFIG_MAX_BUCKET_LINES = "ai.logreport.max_bucket_lines";
+ public static final String CONFIG_INITIAL_DELAY_MINUTES = "ai.logreport.initial_delay_minutes";
+ public static final String CONFIG_PERIOD_MINUTES = "ai.logreport.period_minutes";
+ public static final String CONFIG_MAX_TOKENS = "ai.logreport.max_tokens";
+ public static final String CONFIG_DAILY_COMPRESSION_ENABLED = "ai.logreport.daily_compression.enabled";
+ public static final String CONFIG_DELETE_HOURLY_AFTER_DAILY = "ai.logreport.daily_compression.delete_hourly";
+ public static final String CONFIG_FEED_MAX_ENTRIES = "ai.logreport.feed.max_entries";
+ public static final String DEFAULT_REPORT_DIR = "DATA/REPORTS/log";
+ public static final int DEFAULT_MAX_BUCKET_LINES = 100000;
+ public static final int DEFAULT_FEED_MAX_ENTRIES = 100;
+ public static final long DEFAULT_INITIAL_DELAY_MINUTES = 5L;
+ public static final long DEFAULT_PERIOD_MINUTES = 60L;
+
+ private static final ConcurrentLog log = new ConcurrentLog("LOGREPORT");
+ private static final DateTimeFormatter LOG_TIMESTAMP_FORMATTER = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss");
+ private static final DateTimeFormatter HOURLY_REPORT_FORMATTER = DateTimeFormatter.ofPattern("'report-'yyyy-MM-dd-HH'.md'");
+ private static final DateTimeFormatter DAILY_REPORT_FORMATTER = DateTimeFormatter.ofPattern("'report-'yyyy-MM-dd'.md'");
+ private static final ZoneId REPORT_ZONE = ZoneId.systemDefault();
+ private static final String SYSTEM_PROMPT =
+ "You evaluate YaCy runtime logs for operator-facing self-enhancement reports. " +
+ "Do not suggest automatic code changes. Identify operational evidence and practical development opportunities.";
+ private static final int NOISE_EXAMPLE_LIMIT = 3;
+
+ private final Switchboard sb;
+
+ public static class ReportEntry implements Comparable<ReportEntry> {
+ public final String filename;
+ public final String title;
+ public final ZonedDateTime published;
+ public final String content;
+ public final boolean daily;
+
+ public ReportEntry(final String filename, final String title, final ZonedDateTime published, final String content, final boolean daily) {
+ this.filename = filename;
+ this.title = title;
+ this.published = published;
+ this.content = content;
+ this.daily = daily;
+ }
+
+ @Override
+ public int compareTo(final ReportEntry other) {
+ final int dateCompare = other.published.compareTo(this.published);
+ if (dateCompare != 0) return dateCompare;
+ return other.filename.compareTo(this.filename);
+ }
+ }
+
+ private static class NoiseBucket {
+ int count;
+ final List<String> examples = new ArrayList<>(NOISE_EXAMPLE_LIMIT);
+ }
+
+ private static class NoiseSummary {
+ final int totalLines;
+ final Map<String, NoiseBucket> buckets = new LinkedHashMap<>();
+
+ NoiseSummary(final int totalLines) {
+ this.totalLines = totalLines;
+ }
+
+ int classifiedLines() {
+ int count = 0;
+ for (final NoiseBucket bucket : this.buckets.values()) count += bucket.count;
+ return count;
+ }
+ }
+
+ public LogReportService(final Switchboard sb) {
+ this.sb = sb;
+ }
+
+ public static boolean hasConfiguredLogReportModel() {
+ return LLM.llmFromUsageQuiet(LLMUsage.logreport) != null;
+ }
+
+ public static ScheduledExecutorService startScheduler(final Switchboard sb) {
+ if (sb == null) return null;
+ if (!sb.getConfigBool(CONFIG_ENABLED, true)) {
+ log.info("Log report scheduler is disabled by " + CONFIG_ENABLED + ".");
+ return null;
+ }
+ if (!hasConfiguredLogReportModel()) {
+ log.info("Log report scheduler is inactive because no production model is configured for logreport usage.");
+ return null;
+ }
+
+ final LogReportService service = new LogReportService(sb);
+ final long initialDelay = Math.max(0L, sb.getConfigLong(CONFIG_INITIAL_DELAY_MINUTES, DEFAULT_INITIAL_DELAY_MINUTES));
+ final long period = Math.max(1L, sb.getConfigLong(CONFIG_PERIOD_MINUTES, DEFAULT_PERIOD_MINUTES));
+ final AtomicBoolean missingModelLogged = new AtomicBoolean(false);
+ final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> {
+ final Thread thread = new Thread(r, "LogReportService.scheduler");
+ thread.setDaemon(true);
+ return thread;
+ });
+ scheduler.scheduleAtFixedRate(() -> {
+ final String runId = newRunId();
+ final long tickStart = System.currentTimeMillis();
+ try {
+ if (!hasConfiguredLogReportModel()) {
+ if (missingModelLogged.compareAndSet(false, true)) {
+ log.info("runId=" + runId + " event=scheduler-tick phase=skip reason=no-logreport-model");
+ }
+ return;
+ }
+ missingModelLogged.set(false);
+ log.info("runId=" + runId + " event=scheduler-tick phase=start periodMinutes=" + period);
+ final List<File> hourlyReports = service.generateHourlyReports(runId, "scheduler");
+ final List<File> dailyReports = service.generateDailyReports(runId, "scheduler");
+ log.info("runId=" + runId + " event=scheduler-tick phase=end result=success hourlyWritten=" + hourlyReports.size() + " dailyWritten=" + dailyReports.size() + " durationMs=" + elapsed(tickStart));
+ } catch (final Throwable e) {
+ log.warn("runId=" + runId + " event=scheduler-tick phase=end result=failure errorClass=" + e.getClass().getName() + " reason=" + LogRedaction.redactMessage(e) + " durationMs=" + elapsed(tickStart));
+ }
+ }, initialDelay, period, TimeUnit.MINUTES);
+ log.info("event=scheduler-start result=success initialDelayMinutes=" + initialDelay + " periodMinutes=" + period);
+ return scheduler;
+ }
+
+ public static void stopScheduler(final ScheduledExecutorService scheduler) {
+ if (scheduler == null) return;
+ scheduler.shutdownNow();
+ log.info("Log report scheduler stopped.");
+ }
+
+ public File getReportDirectory() {
+ return this.sb.getDataPath(CONFIG_REPORT_DIR, DEFAULT_REPORT_DIR);
+ }
+
+ public List<ReportEntry> discoverReports(final int maxEntries) {
+ final File reportDirectory = getReportDirectory();
+ if (!reportDirectory.isDirectory()) {
+ log.info("No log reports discovered because report directory does not exist: " + reportDirectory.getAbsolutePath());
+ return Collections.emptyList();
+ }
+ final File[] files = reportDirectory.listFiles();
+ if (files == null || files.length == 0) {
+ log.info("No log reports discovered in " + reportDirectory.getAbsolutePath() + ".");
+ return Collections.emptyList();
+ }
+
+ final List<ReportEntry> reports = new ArrayList<>();
+ int candidates = 0;
+ for (final File file : files) {
+ if (!file.isFile()) continue;
+ candidates++;
+ final ReportEntry report = reportEntry(file);
+ if (report != null) reports.add(report);
+ }
+ Collections.sort(reports);
+ log.info("Discovered " + reports.size() + " log report(s) from " + candidates + " file(s) in " + reportDirectory.getAbsolutePath() + ".");
+ if (maxEntries >= 0 && reports.size() > maxEntries) {
+ return new ArrayList<>(reports.subList(0, maxEntries));
+ }
+ return reports;
+ }
+
+ public List<File> generateHourlyReports() {
+ return generateHourlyReports(newRunId(), "direct");
+ }
+
+ private List<File> generateHourlyReports(final String runId, final String trigger) {
+ final long start = System.currentTimeMillis();
+ log.info("runId=" + runId + " event=hourly-reports phase=start trigger=" + trigger);
+ final LLMModel model = LLM.llmFromUsage(LLMUsage.logreport, runId, "hourly-reports");
+ if (model == null) {
+ log.info("runId=" + runId + " event=hourly-reports phase=skip reason=no-logreport-model durationMs=" + elapsed(start));
+ return Collections.emptyList();
+ }
+
+ final List<String> logLines = readRuntimeLogLines();
+ final Map<LocalDateTime, List<String>> buckets = completedHourlyBuckets(logLines);
+ log.info("runId=" + runId + " event=hourly-reports phase=bucket-scan runtimeLines=" + logLines.size() + " buckets=" + buckets.size());
+ if (buckets.isEmpty()) {
+ log.info("runId=" + runId + " event=hourly-reports phase=skip reason=no-completed-buckets durationMs=" + elapsed(start));
+ return Collections.emptyList();
+ }
+
+ final File reportDirectory = getReportDirectory();
+ if (!reportDirectory.exists() && !reportDirectory.mkdirs()) {
+ log.warn("runId=" + runId + " event=hourly-reports phase=prepare-directory result=failure path=" + reportDirectory.getAbsolutePath());
+ return Collections.emptyList();
+ }
+
+ final List<File> writtenReports = new ArrayList<>();
+ int skippedExisting = 0;
+ int failedReports = 0;
+ for (final Map.Entry<LocalDateTime, List<String>> bucket : buckets.entrySet()) {
+ final File reportFile = new File(reportDirectory, hourlyReportFilename(bucket.getKey()));
+ if (reportFile.exists()) {
+ skippedExisting++;
+ log.info("runId=" + runId + " event=hourly-report phase=skip bucket=" + bucket.getKey() + " reason=report-exists file=" + reportFile.getAbsolutePath());
+ continue;
+ }
+ try {
+ final NoiseSummary noiseSummary = classifyNoise(bucket.getValue());
+ final String prompt = hourlyPrompt(bucket.getKey(), bucket.getValue(), noiseSummary);
+ log.info("runId=" + runId + " event=hourly-report phase=classify-noise bucket=" + bucket.getKey() + " inputLines=" + bucket.getValue().size() + " noiseLines=" + noiseSummary.classifiedLines() + " noiseCategories=" + noiseSummary.buckets.size());
+ final int configuredMaxTokens = this.sb.getConfigInt(CONFIG_MAX_TOKENS, model.llm.max_tokens);
+ final int maxTokens = Math.max(1, Math.min(model.llm.max_tokens, configuredMaxTokens));
+ log.info("runId=" + runId + " event=hourly-report phase=model-call bucket=" + bucket.getKey() + " model=" + LogRedaction.redact(model.model) + " backend=" + LogRedaction.redact(model.llm.hoststub) + " inputLines=" + bucket.getValue().size() + " promptChars=" + prompt.length() + " maxTokens=" + maxTokens);
+ final long modelStart = System.currentTimeMillis();
+ final String report = model.llm.chat(model.model, SYSTEM_PROMPT, prompt, maxTokens);
+ final long modelDuration = elapsed(modelStart);
+ log.info("runId=" + runId + " event=hourly-report phase=model-return bucket=" + bucket.getKey() + " durationMs=" + modelDuration + " outputChars=" + (report == null ? 0 : report.length()));
+ if (report == null || report.trim().isEmpty()) {
+ throw new IOException("model returned an empty report");
+ }
+ writeReport(reportFile, reportDocument(bucket.getKey(), bucket.getValue().size(), report));
+ writtenReports.add(reportFile);
+ log.info("runId=" + runId + " event=hourly-report phase=write result=success bucket=" + bucket.getKey() + " inputLines=" + bucket.getValue().size() + " outputChars=" + report.length() + " file=" + reportFile.getAbsolutePath());
+ } catch (final IOException e) {
+ failedReports++;
+ log.warn("runId=" + runId + " event=hourly-report phase=write result=failure bucket=" + bucket.getKey() + " file=" + LogRedaction.redact(reportFile.getAbsolutePath()) + " errorClass=" + e.getClass().getName() + " reason=" + LogRedaction.redactMessage(e));
+ }
+ }
+ log.info("runId=" + runId + " event=hourly-reports phase=end result=success buckets=" + buckets.size() + " written=" + writtenReports.size() + " skippedExisting=" + skippedExisting + " failed=" + failedReports + " durationMs=" + elapsed(start));
+ return writtenReports;
+ }
+
+ public File generateCurrentHourReportOverwrite() throws IOException {
+ final String runId = newRunId();
+ final long start = System.currentTimeMillis();
+ log.info("runId=" + runId + " event=current-hour-report phase=start mode=manual");
+ final LLMModel model = LLM.llmFromUsage(LLMUsage.logreport, runId, "current-hour-report");
+ if (model == null) {
+ log.info("runId=" + runId + " event=current-hour-report phase=skip reason=no-logreport-model durationMs=" + elapsed(start));
+ return null;
+ }
+
+ final LocalDateTime currentHour = LocalDateTime.now().truncatedTo(ChronoUnit.HOURS);
+ final List<String> logLines = readRuntimeLogLines();
+ final List<String> bucketLines = hourlyBucket(logLines, currentHour);
+ log.info("runId=" + runId + " event=current-hour-report phase=bucket-scan bucket=" + currentHour + " runtimeLines=" + logLines.size() + " inputLines=" + bucketLines.size());
+ if (bucketLines.isEmpty()) {
+ log.info("runId=" + runId + " event=current-hour-report phase=skip bucket=" + currentHour + " reason=no-matching-log-lines durationMs=" + elapsed(start));
+ return null;
+ }
+
+ final File reportDirectory = getReportDirectory();
+ if (!reportDirectory.exists() && !reportDirectory.mkdirs()) {
+ log.warn("runId=" + runId + " event=current-hour-report phase=prepare-directory result=failure path=" + reportDirectory.getAbsolutePath());
+ throw new IOException("cannot create log report directory " + reportDirectory.getAbsolutePath());
+ }
+
+ final File reportFile = new File(reportDirectory, hourlyReportFilename(currentHour));
+ final NoiseSummary noiseSummary = classifyNoise(bucketLines);
+ final String prompt = hourlyPrompt(currentHour, bucketLines, noiseSummary);
+ log.info("runId=" + runId + " event=current-hour-report phase=classify-noise bucket=" + currentHour + " inputLines=" + bucketLines.size() + " noiseLines=" + noiseSummary.classifiedLines() + " noiseCategories=" + noiseSummary.buckets.size());
+ final int configuredMaxTokens = this.sb.getConfigInt(CONFIG_MAX_TOKENS, model.llm.max_tokens);
+ final int maxTokens = Math.max(1, Math.min(model.llm.max_tokens, configuredMaxTokens));
+ log.info("runId=" + runId + " event=current-hour-report phase=model-call bucket=" + currentHour + " model=" + LogRedaction.redact(model.model) + " backend=" + LogRedaction.redact(model.llm.hoststub) + " inputLines=" + bucketLines.size() + " promptChars=" + prompt.length() + " maxTokens=" + maxTokens);
+ final long modelStart = System.currentTimeMillis();
+ final String report = model.llm.chat(model.model, SYSTEM_PROMPT, prompt, maxTokens);
+ log.info("runId=" + runId + " event=current-hour-report phase=model-return bucket=" + currentHour + " durationMs=" + elapsed(modelStart) + " outputChars=" + (report == null ? 0 : report.length()));
+ if (report == null || report.trim().isEmpty()) {
+ log.warn("runId=" + runId + " event=current-hour-report phase=model-return result=failure bucket=" + currentHour + " reason=empty-report");
+ throw new IOException("model returned an empty report");
+ }
+ if (reportFile.exists()) {
+ log.info("runId=" + runId + " event=current-hour-report phase=write action=overwrite file=" + reportFile.getAbsolutePath());
+ }
+ writeReport(reportFile, reportDocument(currentHour, bucketLines.size(), report), true);
+ log.info("runId=" + runId + " event=current-hour-report phase=end result=success bucket=" + currentHour + " inputLines=" + bucketLines.size() + " outputChars=" + report.length() + " file=" + reportFile.getAbsolutePath() + " durationMs=" + elapsed(start));
+ return reportFile;
+ }
+
+ public List<File> generateDailyReports() {
+ return generateDailyReports(newRunId(), "direct");
+ }
+
+ private List<File> generateDailyReports(final String runId, final String trigger) {
+ final long start = System.currentTimeMillis();
+ log.info("runId=" + runId + " event=daily-reports phase=start trigger=" + trigger);
+ if (!this.sb.getConfigBool(CONFIG_DAILY_COMPRESSION_ENABLED, true)) {
+ log.info("runId=" + runId + " event=daily-reports phase=skip reason=daily-compression-disabled durationMs=" + elapsed(start));
+ return Collections.emptyList();
+ }
+ final LLMModel model = LLM.llmFromUsage(LLMUsage.logreport, runId, "daily-reports");
+ if (model == null) {
+ log.info("runId=" + runId + " event=daily-reports phase=skip reason=no-logreport-model durationMs=" + elapsed(start));
+ return Collections.emptyList();
+ }
+
+ final File reportDirectory = getReportDirectory();
+ if (!reportDirectory.isDirectory()) {
+ log.info("runId=" + runId + " event=daily-reports phase=skip reason=report-directory-missing path=" + reportDirectory.getAbsolutePath() + " durationMs=" + elapsed(start));
+ return Collections.emptyList();
+ }
+
+ final Map<LocalDate, List<File>> completeDays = completeHourlyReportSets(reportDirectory);
+ log.info("runId=" + runId + " event=daily-reports phase=scan completeDays=" + completeDays.size() + " path=" + reportDirectory.getAbsolutePath());
+ if (completeDays.isEmpty()) {
+ log.info("runId=" + runId + " event=daily-reports phase=skip reason=no-complete-day durationMs=" + elapsed(start));
+ return Collections.emptyList();
+ }
+
+ final List<File> writtenReports = new ArrayList<>();
+ int skippedExisting = 0;
+ int failedReports = 0;
+ for (final Map.Entry<LocalDate, List<File>> day : completeDays.entrySet()) {
+ final File dailyReportFile = new File(reportDirectory, dailyReportFilename(day.getKey()));
+ if (dailyReportFile.exists()) {
+ skippedExisting++;
+ log.info("runId=" + runId + " event=daily-report phase=skip day=" + day.getKey() + " reason=report-exists file=" + dailyReportFile.getAbsolutePath());
+ continue;
+ }
+ try {
+ final String prompt = dailyPrompt(day.getKey(), day.getValue());
+ final int configuredMaxTokens = this.sb.getConfigInt(CONFIG_MAX_TOKENS, model.llm.max_tokens);
+ final int maxTokens = Math.max(1, Math.min(model.llm.max_tokens, configuredMaxTokens));
+ log.info("runId=" + runId + " event=daily-report phase=model-call day=" + day.getKey() + " model=" + LogRedaction.redact(model.model) + " backend=" + LogRedaction.redact(model.llm.hoststub) + " sourceReports=" + day.getValue().size() + " promptChars=" + prompt.length() + " maxTokens=" + maxTokens);
+ final long modelStart = System.currentTimeMillis();
+ final String report = model.llm.chat(model.model, SYSTEM_PROMPT, prompt, maxTokens);
+ log.info("runId=" + runId + " event=daily-report phase=model-return day=" + day.getKey() + " durationMs=" + elapsed(modelStart) + " outputChars=" + (report == null ? 0 : report.length()));
+ if (report == null || report.trim().isEmpty()) {
+ throw new IOException("model returned an empty daily report");
+ }
+ writeReport(dailyReportFile, dailyReportDocument(day.getKey(), day.getValue().size(), report));
+ writtenReports.add(dailyReportFile);
+ log.info("runId=" + runId + " event=daily-report phase=write result=success day=" + day.getKey() + " sourceReports=" + day.getValue().size() + " outputChars=" + report.length() + " file=" + dailyReportFile.getAbsolutePath());
+ if (this.sb.getConfigBool(CONFIG_DELETE_HOURLY_AFTER_DAILY, false)) {
+ deleteHourlyReports(day.getValue());
+ log.info("runId=" + runId + " event=daily-report phase=delete-hourly day=" + day.getKey() + " sourceReports=" + day.getValue().size());
+ }
+ } catch (final IOException e) {
+ failedReports++;
+ log.warn("runId=" + runId + " event=daily-report phase=write result=failure day=" + day.getKey() + " file=" + LogRedaction.redact(dailyReportFile.getAbsolutePath()) + " errorClass=" + e.getClass().getName() + " reason=" + LogRedaction.redactMessage(e));
+ }
+ }
+ log.info("runId=" + runId + " event=daily-reports phase=end result=success completeDays=" + completeDays.size() + " written=" + writtenReports.size() + " skippedExisting=" + skippedExisting + " failed=" + failedReports + " durationMs=" + elapsed(start));
+ return writtenReports;
+ }
+
+ public Map<LocalDateTime, List<String>> completedHourlyBuckets(final List<String> logLines) {
+ final int maxBucketLines = this.sb.getConfigInt(CONFIG_MAX_BUCKET_LINES, DEFAULT_MAX_BUCKET_LINES);
+ final LocalDateTime currentHour = LocalDateTime.now().truncatedTo(ChronoUnit.HOURS);
+ final Map<LocalDateTime, List<String>> buckets = new TreeMap<>();
+
+ for (final String line : logLines) {
+ final LocalDateTime timestamp = parseLogTimestamp(line);
+ if (timestamp == null) continue;
+ final LocalDateTime bucket = timestamp.truncatedTo(ChronoUnit.HOURS);
+ if (!bucket.isBefore(currentHour)) continue;
+ List<String> bucketLines = buckets.get(bucket);
+ if (bucketLines == null) {
+ bucketLines = new ArrayList<>();
+ buckets.put(bucket, bucketLines);
+ }
+ bucketLines.add(line);
+ while (bucketLines.size() > maxBucketLines) {
+ bucketLines.remove(0);
+ }
+ }
+ return buckets;
+ }
+
+ private List<String> hourlyBucket(final List<String> logLines, final LocalDateTime selectedHour) {
+ final int maxBucketLines = this.sb.getConfigInt(CONFIG_MAX_BUCKET_LINES, DEFAULT_MAX_BUCKET_LINES);
+ final List<String> bucketLines = new ArrayList<>();
+ for (final String line : logLines) {
+ final LocalDateTime timestamp = parseLogTimestamp(line);
+ if (timestamp == null || !selectedHour.equals(timestamp.truncatedTo(ChronoUnit.HOURS))) continue;
+ bucketLines.add(line);
+ while (bucketLines.size() > maxBucketLines) {
+ bucketLines.remove(0);
+ }
+ }
+ return bucketLines;
+ }
+
+ public List<String> readRuntimeLogLines() {
+ final GuiHandler handler = findGuiHandler();
+ if (handler == null) return Collections.emptyList();
+ final String[] lines = handler.getLogLines(false, handler.getSize());
+ final List<String> result = new ArrayList<>(lines.length);
+ for (final String line : lines) {
+ if (line == null) continue;
+ final String trimmed = line.trim();
+ if (!trimmed.isEmpty()) result.add(trimmed);
+ }
+ return result;
+ }
+
+ public static String hourlyReportFilename(final LocalDateTime bucket) {
+ return HOURLY_REPORT_FORMATTER.format(bucket);
+ }
+
+ public static String dailyReportFilename(final LocalDate day) {
+ return DAILY_REPORT_FORMATTER.format(day);
+ }
+
+ static LocalDateTime parseLogTimestamp(final String line) {
+ if (line == null || line.length() < 21) return null;
+ try {
+ return LocalDateTime.parse(line.substring(2, 21), LOG_TIMESTAMP_FORMATTER);
+ } catch (final RuntimeException e) {
+ return null;
+ }
+ }
+
+ private static GuiHandler findGuiHandler() {
+ final Logger rootLogger = Logger.getLogger("");
+ final Handler[] handlers = rootLogger.getHandlers();
+ for (final Handler handler : handlers) {
+ if (handler instanceof GuiHandler) return (GuiHandler) handler;
+ }
+ return null;
+ }
+
+ private static String newRunId() {
+ return UUID.randomUUID().toString();
+ }
+
+ private static long elapsed(final long start) {
+ return System.currentTimeMillis() - start;
+ }
+
+ private static String hourlyPrompt(final LocalDateTime bucket, final List<String> lines) {
+ return hourlyPrompt(bucket, lines, classifyNoise(lines));
+ }
+
+ private static String hourlyPrompt(final LocalDateTime bucket, final List<String> lines, final NoiseSummary noiseSummary) {
+ final StringBuilder prompt = new StringBuilder(1024 + lines.size() * 120);
+ prompt.append("Create a YaCy self-enhancement log report for hour ")
+ .append(bucket)
+ .append(".\n\n")
+ .append("Noise classification:\n")
+ .append(noiseSummaryText(noiseSummary))
+ .append("\nGuidance: repeated peer availability, remote search miss, and logreport self-observation noise should be summarized as operational background unless it correlates with user-visible failure, long latency, or a repeated code exception.\n\n")
+ .append("Use these fixed sections:\n")
+ .append("1. Summary / key takeaway\n")
+ .append("2. Usage types\n")
+ .append("3. Challenges\n")
+ .append("4. Errors and risks\n")
+ .append("5. Possible improvements\n")
+ .append("6. Concrete actions\n\n")
+ .append("Each concrete action must identify the log trigger and expected effect. ")
+ .append("Do not include secrets, credentials, or raw user content beyond what already appears in the log metadata.\n\n")
+ .append("Log lines:\n");
+ for (final String line : lines) {
+ prompt.append(line).append('\n');
+ }
+ return prompt.toString();
+ }
+
+ private static NoiseSummary classifyNoise(final List<String> lines) {
+ final NoiseSummary summary = new NoiseSummary(lines == null ? 0 : lines.size());
+ if (lines == null || lines.isEmpty()) return summary;
+ for (final String line : lines) {
+ final String category = noiseCategory(line);
+ if (category == null) continue;
+ NoiseBucket bucket = summary.buckets.get(category);
+ if (bucket == null) {
+ bucket = new NoiseBucket();
+ summary.buckets.put(category, bucket);
+ }
+ bucket.count++;
+ if (bucket.examples.size() < NOISE_EXAMPLE_LIMIT) {
+ bucket.examples.add(LogRedaction.redact(line));
+ }
+ }
+ return summary;
+ }
+
+ private static String noiseCategory(final String line) {
+ if (line == null || line.isEmpty()) return null;
+ final String normalized = line.toLowerCase();
+ if (normalized.contains("network") && normalized.contains("publish: disconnected")) {
+ return "peer-publish-disconnected";
+ }
+ if (normalized.contains("remote search - no answer from remote peer")) {
+ return "remote-search-no-answer";
+ }
+ if (normalized.contains("remote search - interrupted search to remote peer")) {
+ return "remote-search-interrupted";
+ }
+ if (normalized.contains("transfer to peer") && normalized.contains("failed")) {
+ return "peer-transfer-failed";
+ }
+ if (normalized.contains("found not enough") && normalized.contains("peers for distribution")) {
+ return "dht-not-enough-peers";
+ }
+ if (normalized.contains("logreport") && (normalized.contains("discovered ") || normalized.contains("no log reports discovered"))) {
+ return "logreport-self-observation";
+ }
+ if (normalized.contains("event=model-routing phase=select") || normalized.contains("event=model-routing phase=miss")) {
+ return "llm-routing-observation";
+ }
+ return null;
+ }
+
+ private static String noiseSummaryText(final NoiseSummary summary) {
+ if (summary == null || summary.buckets.isEmpty()) {
+ return "- Classified noise lines: 0 / " + (summary == null ? 0 : summary.totalLines) + "\n";
+ }
+ final StringBuilder text = new StringBuilder(512);
+ text.append("- Classified noise lines: ")
+ .append(summary.classifiedLines())
+ .append(" / ")
+ .append(summary.totalLines)
+ .append('\n');
+ for (final Map.Entry<String, NoiseBucket> entry : summary.buckets.entrySet()) {
+ text.append("- ")
+ .append(entry.getKey())
+ .append(": ")
+ .append(entry.getValue().count)
+ .append('\n');
+ for (final String example : entry.getValue().examples) {
+ text.append(" example: ").append(example).append('\n');
+ }
+ }
+ return text.toString();
+ }
+
+ private static Map<LocalDate, List<File>> completeHourlyReportSets(final File reportDirectory) {
+ final File[] files = reportDirectory.listFiles();
+ if (files == null || files.length == 0) return Collections.emptyMap();
+
+ final Map<LocalDate, List<File>> candidates = new TreeMap<>();
+ for (final File file : files) {
+ if (!file.isFile()) continue;
+ final LocalDateTime hour = parseHourlyReportFilename(file.getName());
+ if (hour == null) continue;
+ final LocalDate day = hour.toLocalDate();
+ List<File> dayFiles = candidates.get(day);
+ if (dayFiles == null) {
+ dayFiles = new ArrayList<>();
+ candidates.put(day, dayFiles);
+ }
+ dayFiles.add(file);
+ }
+
+ final Map<LocalDate, List<File>> complete = new TreeMap<>();
+ for (final Map.Entry<LocalDate, List<File>> entry : candidates.entrySet()) {
+ if (entry.getValue().size() != 24) continue;
+ final List<File> ordered = hourlyFilesForDay(reportDirectory, entry.getKey());
+ if (ordered.size() == 24) complete.put(entry.getKey(), ordered);
+ }
+ return complete;
+ }
+
+ private static LocalDateTime parseHourlyReportFilename(final String filename) {
+ if (filename == null || !filename.matches("report-\\d{4}-\\d{2}-\\d{2}-\\d{2}\\.md")) return null;
+ try {
+ return LocalDateTime.parse(filename, HOURLY_REPORT_FORMATTER);
+ } catch (final RuntimeException e) {
+ log.warn("Could not parse hourly log report filename " + filename + ": " + e.getMessage());
+ return null;
+ }
+ }
+
+ private static LocalDate parseDailyReportFilename(final String filename) {
+ if (filename == null || !filename.matches("report-\\d{4}-\\d{2}-\\d{2}\\.md")) return null;
+ try {
+ return LocalDate.parse(filename, DAILY_REPORT_FORMATTER);
+ } catch (final RuntimeException e) {
+ log.warn("Could not parse daily log report filename " + filename + ": " + e.getMessage());
+ return null;
+ }
+ }
+
+ private static ReportEntry reportEntry(final File file) {
+ final String filename = file.getName();
+ try {
+ final LocalDate daily = parseDailyReportFilename(filename);
+ if (daily != null) {
+ final ZonedDateTime published = daily.atTime(LocalTime.MAX).atZone(REPORT_ZONE);
+ return new ReportEntry(filename, "YaCy daily log report " + daily, published, readFile(file), true);
+ }
+ final LocalDateTime hourly = parseHourlyReportFilename(filename);
+ if (hourly != null) {
+ final ZonedDateTime published = hourly.atZone(REPORT_ZONE);
+ return new ReportEntry(filename, "YaCy hourly log report " + hourly, published, readFile(file), false);
+ }
+ } catch (final IOException e) {
+ log.warn("Could not read log report " + file.getAbsolutePath() + ": " + e.getMessage());
+ }
+ log.info("Ignoring file in log report directory because it does not match a report filename: " + file.getAbsolutePath());
+ return null;
+ }
+
+ private static String readFile(final File file) throws IOException {
+ return new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8);
+ }
+
+ private static List<File> hourlyFilesForDay(final File reportDirectory, final LocalDate day) {
+ final List<File> files = new ArrayList<>(24);
+ for (int hour = 0; hour < 24; hour++) {
+ final File file = new File(reportDirectory, hourlyReportFilename(day.atTime(hour, 0)));
+ if (!file.isFile()) return Collections.emptyList();
+ files.add(file);
+ }
+ return files;
+ }
+
+ private static String dailyPrompt(final LocalDate day, final List<File> hourlyReports) throws IOException {
+ final StringBuilder prompt = new StringBuilder(4096);
+ prompt.append("Create one consolidated YaCy self-enhancement report for ")
+ .append(day)
+ .append(" from the following hourly log reports.\n\n")
+ .append("Use these fixed sections:\n")
+ .append("1. Summary / key takeaway\n")
+ .append("2. Usage types\n")
+ .append("3. Challenges\n")
+ .append("4. Errors and risks\n")
+ .append("5. Possible improvements\n")
+ .append("6. Concrete actions\n\n")
+ .append("Find common patterns across the day. Each concrete action must identify the repeated log trigger and expected effect.\n\n");
+ for (final File hourlyReport : hourlyReports) {
+ prompt.append("\n\n## ").append(hourlyReport.getName()).append("\n\n")
+ .append(new String(Files.readAllBytes(hourlyReport.toPath()), StandardCharsets.UTF_8));
+ }
+ return prompt.toString();
+ }
+
+ private static String reportDocument(final LocalDateTime bucket, final int lineCount, final String report) {
+ final StringBuilder document = new StringBuilder(report.length() + 256);
+ document.append("# YaCy Log Report\n\n")
+ .append("- Bucket: ").append(bucket).append('\n')
+ .append("- Lines: ").append(lineCount).append('\n')
+ .append("- Generated: ").append(LocalDateTime.now()).append("\n\n")
+ .append(report.trim())
+ .append('\n');
+ return document.toString();
+ }
+
+ private static String dailyReportDocument(final LocalDate day, final int hourlyReportCount, final String report) {
+ final StringBuilder document = new StringBuilder(report.length() + 256);
+ document.append("# YaCy Daily Log Report\n\n")
+ .append("- Day: ").append(day).append('\n')
+ .append("- Hourly reports: ").append(hourlyReportCount).append('\n')
+ .append("- Generated: ").append(LocalDateTime.now()).append("\n\n")
+ .append(report.trim())
+ .append('\n');
+ return document.toString();
+ }
+
+ private static void deleteHourlyReports(final List<File> hourlyReports) {
+ for (final File hourlyReport : hourlyReports) {
+ if (!hourlyReport.delete()) {
+ log.warn("Could not delete hourly log report after daily compression: " + hourlyReport.getAbsolutePath());
+ }
+ }
+ }
+
+ private static void writeReport(final File reportFile, final String report) throws IOException {
+ writeReport(reportFile, report, false);
+ }
+
+ private static void writeReport(final File reportFile, final String report, final boolean replaceExisting) throws IOException {
+ final File parent = reportFile.getParentFile();
+ if (parent != null && !parent.exists() && !parent.mkdirs()) {
+ throw new IOException("cannot create directory " + parent.getAbsolutePath());
+ }
+ final File tmpFile = File.createTempFile(reportFile.getName() + ".", ".tmp", parent);
+ Files.write(tmpFile.toPath(), report.getBytes(StandardCharsets.UTF_8));
+ try {
+ if (replaceExisting) {
+ Files.move(tmpFile.toPath(), reportFile.toPath(), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+ } else {
+ Files.move(tmpFile.toPath(), reportFile.toPath(), StandardCopyOption.ATOMIC_MOVE);
+ }
+ } catch (final AtomicMoveNotSupportedException e) {
+ if (replaceExisting) {
+ Files.move(tmpFile.toPath(), reportFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ } else {
+ Files.move(tmpFile.toPath(), reportFile.toPath());
+ }
+ }
+ }
+}
diff --git a/source/net/yacy/ai/RAGAugmentor.java b/source/net/yacy/ai/RAGAugmentor.java
index a2dde813c..d2f38520d 100644
--- a/source/net/yacy/ai/RAGAugmentor.java
+++ b/source/net/yacy/ai/RAGAugmentor.java
@@ -41,6 +41,7 @@ import net.yacy.cora.federate.yacy.CacheStrategy;
import net.yacy.cora.lod.vocabulary.Tagging;
import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.cora.util.LogRedaction;
import net.yacy.kelondro.data.meta.URIMetadataNode;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
@@ -99,8 +100,12 @@ public final class RAGAugmentor {
* @return JSON array with {@code url,title[,text]} entries
*/
public static JSONArray searchResults(String query, int count, final boolean includeSnippet) {
+ return searchResults(query, count, includeSnippet, null);
+ }
+
+ public static JSONArray searchResults(String query, int count, final boolean includeSnippet, final String runId) {
final QueryParams theQuery = buildTextQueryParams(query, count, QueryParams.Searchdom.LOCAL);
- return searchResults(theQuery, count, includeSnippet);
+ return searchResults(theQuery, count, includeSnippet, runId);
}
/**
@@ -112,9 +117,13 @@ public final class RAGAugmentor {
* @return markdown context block
*/
public static String searchResultsAsMarkdown(String query, int count, boolean global) {
+ return searchResultsAsMarkdown(query, count, global, null);
+ }
+
+ public static String searchResultsAsMarkdown(String query, int count, boolean global, final String runId) {
final long searchStart = System.currentTimeMillis();
- JSONArray searchResults = global ? searchResultsGlobal(query, count, true) : searchResults(query, count, true);
- ConcurrentLog.info("RAGProxy", "searchResults=" + searchResults.length() + " global=" + global + " searchMs=" + (System.currentTimeMillis() - searchStart));
+ JSONArray searchResults = global ? searchResultsGlobal(query, count, true, runId) : searchResults(query, count, true, runId);
+ ConcurrentLog.info("RAGProxy", prefix(runId) + "event=rag-search phase=results resultCount=" + searchResults.length() + " global=" + global + " searchMs=" + (System.currentTimeMillis() - searchStart));
StringBuilder sb = new StringBuilder();
for (int i = 0; i < searchResults.length(); i++) {
@@ -135,7 +144,7 @@ public final class RAGAugmentor {
}
final String markdown = truncateSearchDocument(sb.toString());
- ConcurrentLog.info("RAGProxy", "markdownChars=" + markdown.length() + " resultCount=" + searchResults.length());
+ ConcurrentLog.info("RAGProxy", prefix(runId) + "event=rag-search phase=markdown markdownChars=" + markdown.length() + " resultCount=" + searchResults.length());
return markdown;
}
@@ -184,8 +193,12 @@ public final class RAGAugmentor {
* @return JSON array with normalized result objects
*/
public static JSONArray searchResultsGlobal(String query, int count, final boolean includeSnippet) {
+ return searchResultsGlobal(query, count, includeSnippet, null);
+ }
+
+ public static JSONArray searchResultsGlobal(String query, int count, final boolean includeSnippet, final String runId) {
final QueryParams theQuery = buildTextQueryParams(query, count, QueryParams.Searchdom.GLOBAL);
- return searchResults(theQuery, count, includeSnippet);
+ return searchResults(theQuery, count, includeSnippet, runId);
}
/**
@@ -199,9 +212,23 @@ public final class RAGAugmentor {
* @return JSON array with normalized result objects
*/
private static JSONArray searchResults(final QueryParams theQuery, final int count, final boolean includeSnippet) {
+ return searchResults(theQuery, count, includeSnippet, null);
+ }
+
+ private static JSONArray searchResults(final QueryParams theQuery, final int count, final boolean includeSnippet, final String runId) {
+ final long start = System.currentTimeMillis();
final JSONArray results = new JSONArray();
- if (theQuery == null || count == 0) return results;
+ if (theQuery == null || count == 0) {
+ ConcurrentLog.info("RAGProxy", prefix(runId) + "event=rag-search phase=skip reason=empty-query count=" + count);
+ return results;
+ }
final Switchboard sb = Switchboard.getSwitchboard();
+ if (sb == null) {
+ ConcurrentLog.warn("RAGProxy", prefix(runId) + "event=rag-search phase=fail reason=switchboard-unavailable");
+ return results;
+ }
+ final boolean globalSearch = !theQuery.isLocal();
+ ConcurrentLog.info("RAGProxy", prefix(runId) + "event=rag-search phase=start global=" + globalSearch + " count=" + count + " includeSnippet=" + includeSnippet);
final SearchEvent theSearch = SearchEventCache.getEvent(
theQuery,
sb.peers,
@@ -218,7 +245,6 @@ public final class RAGAugmentor {
final long timeout = sb.getConfigLong(
SwitchboardConstants.REMOTESEARCH_MAXTIME_USER,
sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000));
- final boolean globalSearch = !theQuery.isLocal();
if (globalSearch) {
theSearch.resortCachedResults();
} else {
@@ -255,8 +281,10 @@ public final class RAGAugmentor {
results.put(result);
resultIndex++;
} catch (JSONException e) {
+ ConcurrentLog.warn("RAGProxy", prefix(runId) + "event=rag-search phase=result result=failure errorClass=" + e.getClass().getName() + " reason=" + LogRedaction.redactMessage(e));
}
}
+ ConcurrentLog.info("RAGProxy", prefix(runId) + "event=rag-search phase=end result=success global=" + globalSearch + " requested=" + count + " returned=" + results.length() + " includeSnippet=" + includeSnippet + " durationMs=" + (System.currentTimeMillis() - start));
return results;
}
@@ -332,17 +360,29 @@ public final class RAGAugmentor {
* @return space-separated lowercase term list or {@code null} on failure
*/
public static String searchWordsForPrompt(final LLM llm, final String model, final String prompt, final String systemPrompt) {
+ return searchWordsForPrompt(llm, model, prompt, systemPrompt, null);
+ }
+
+ public static String searchWordsForPrompt(final LLM llm, final String model, final String prompt, final String systemPrompt, final String runId) {
+ final long start = System.currentTimeMillis();
final String question = prompt == null ? "" : prompt.trim();
final String instruction = systemPrompt == null || systemPrompt.trim().isEmpty()
? "Compress the user prompt into a short search description. Return only a JSON array of concise, discriminative search terms in lowercase."
: systemPrompt.trim();
- if (llm == null || model == null || model.isEmpty()) return null;
+ if (llm == null || model == null || model.isEmpty()) {
+ ConcurrentLog.warn("RAGProxy", prefix(runId) + "event=rag-query-generation phase=skip reason=no-model promptChars=" + question.length());
+ return null;
+ }
try {
+ ConcurrentLog.info("RAGProxy", prefix(runId) + "event=rag-query-generation phase=model-call model=" + LogRedaction.redact(model) + " backend=" + LogRedaction.redact(llm.hoststub) + " promptChars=" + question.length() + " promptWords=" + wordCount(question));
LLM.Context context = new LLM.Context(instruction);
context.addPrompt(question);
Set<String> singlewords = new LinkedHashSet<>();
String[] a = LLM.stringsFromChat(llm.chat(model, context, LLM.listSchema, 200));
- if (a == null || a.length == 0) return null;
+ if (a == null || a.length == 0) {
+ ConcurrentLog.warn("RAGProxy", prefix(runId) + "event=rag-query-generation phase=model-return result=empty durationMs=" + (System.currentTimeMillis() - start));
+ return null;
+ }
for (String s: a) {
if (s == null) continue;
// Flatten model output into unique lowercased tokens.
@@ -353,13 +393,25 @@ public final class RAGAugmentor {
for (String s: singlewords) query.append(s).append(' ');
String querys = query.toString().trim();
if (querys.length() == 0) return null;
+ ConcurrentLog.info("RAGProxy", prefix(runId) + "event=rag-query-generation phase=end result=success terms=" + singlewords.size() + " queryChars=" + querys.length() + " durationMs=" + (System.currentTimeMillis() - start));
return querys;
} catch (IOException | JSONException e) {
- e.printStackTrace();
+ ConcurrentLog.warn("RAGProxy", prefix(runId) + "event=rag-query-generation phase=end result=failure errorClass=" + e.getClass().getName() + " reason=" + LogRedaction.redactMessage(e) + " durationMs=" + (System.currentTimeMillis() - start));
return null;
}
}
+ private static String prefix(final String runId) {
+ return runId == null || runId.isEmpty() ? "" : "runId=" + runId + " ";
+ }
+
+ private static int wordCount(final String text) {
+ if (text == null) return 0;
+ final String trimmed = text.trim();
+ if (trimmed.isEmpty()) return 0;
+ return trimmed.split("\\s+").length;
+ }
+
/**
* Splits text into sentence-aware chunks around a target max length.
*
diff --git a/source/net/yacy/ai/ToolCallProtocol.java b/source/net/yacy/ai/ToolCallProtocol.java
index 2eab71687..d3e33dbaf 100644
--- a/source/net/yacy/ai/ToolCallProtocol.java
+++ b/source/net/yacy/ai/ToolCallProtocol.java
@@ -43,6 +43,8 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONTokener;
+import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.cora.util.LogRedaction;
/**
* Implements the protocol glue for streamed chat completions that include tool
@@ -63,6 +65,8 @@ import org.json.JSONTokener;
*/
public final class ToolCallProtocol {
+ private static final ConcurrentLog log = new ConcurrentLog("TOOLCALL");
+
/**
* Hard stop to prevent infinite tool loops (assistant requests tool calls
* repeatedly without converging to a final answer).
@@ -117,16 +121,24 @@ public final class ToolCallProtocol {
* @throws IOException on network/stream/protocol errors
*/
public static int proxyToolLifecycle(ServletOutputStream out, LLM.LLMModel llm4Chat, JSONObject originalBody, JSONArray messages, JSONObject initialMetadata) throws IOException {
+ return proxyToolLifecycle(out, llm4Chat, originalBody, messages, initialMetadata, null);
+ }
+
+ public static int proxyToolLifecycle(ServletOutputStream out, LLM.LLMModel llm4Chat, JSONObject originalBody, JSONArray messages, JSONObject initialMetadata, final String runId) throws IOException {
+ final long start = System.currentTimeMillis();
final JSONObject preparedBody = prepareToolRequestBody(originalBody, false, llm4Chat != null && llm4Chat.tooling);
if (llm4Chat != null && llm4Chat.thinking) {
LLM.applyNoThinkingParameters(preparedBody);
}
+ log.info(prefix(runId) + "event=tool-lifecycle phase=start tooling=" + (llm4Chat != null && llm4Chat.tooling) + " thinking=" + (llm4Chat != null && llm4Chat.thinking) + " messages=" + (messages == null ? 0 : messages.length()) + " metadata=" + (initialMetadata == null ? 0 : initialMetadata.length()));
final HttpURLConnection conn = openChatCompletionConnection(llm4Chat, preparedBody);
final int status = conn.getResponseCode();
//final String message = conn.getResponseMessage();
if (status == 200) {
- handleInitialStreamAndContinue(out, conn, llm4Chat, preparedBody, messages, initialMetadata);
+ handleInitialStreamAndContinue(out, conn, llm4Chat, preparedBody, messages, initialMetadata, runId);
+ log.info(prefix(runId) + "event=tool-lifecycle phase=end result=success status=" + status + " durationMs=" + (System.currentTimeMillis() - start));
} else {
+ log.warn(prefix(runId) + "event=tool-lifecycle phase=end result=upstream-error status=" + status + " reason=" + LogRedaction.redact(conn.getResponseMessage()) + " durationMs=" + (System.currentTimeMillis() - start));
Logger.getLogger("ToolCallProtocoll").severe(status + " " + conn.getResponseMessage());
}
return status;
@@ -196,6 +208,11 @@ public final class ToolCallProtocol {
* @throws IOException on I/O errors
*/
private static void handleInitialStreamAndContinue(ServletOutputStream out, HttpURLConnection conn, LLM.LLMModel llm4Chat, JSONObject originalBody, JSONArray messages, JSONObject initialMetadata) throws IOException {
+ handleInitialStreamAndContinue(out, conn, llm4Chat, originalBody, messages, initialMetadata, null);
+ }
+
+ private static void handleInitialStreamAndContinue(ServletOutputStream out, HttpURLConnection conn, LLM.LLMModel llm4Chat, JSONObject originalBody, JSONArray messages, JSONObject initialMetadata, final String runId) throws IOException {
+ final long start = System.currentTimeMillis();
final StringBuilder assistantContent = new StringBuilder();
final Map<Integer, ToolCall> toolCalls = new HashMap<>();
final boolean[] sawToolCalls = new boolean[]{false};
@@ -232,8 +249,9 @@ public final class ToolCallProtocol {
conn.disconnect();
}
+ log.info(prefix(runId) + "event=tool-stream phase=initial-end sawToolCalls=" + sawToolCalls[0] + " toolCalls=" + toolCalls.size() + " assistantChars=" + assistantContent.length() + " durationMs=" + (System.currentTimeMillis() - start));
if (sawToolCalls[0]) {
- handleToolCallsAndContinue(out, llm4Chat, originalBody, messages, assistantContent.toString(), toolCalls);
+ handleToolCallsAndContinue(out, llm4Chat, originalBody, messages, assistantContent.toString(), toolCalls, runId);
}
}
@@ -259,6 +277,11 @@ public final class ToolCallProtocol {
* @throws IOException when network I/O or JSON processing fails
*/
public static void handleToolCallsAndContinue(ServletOutputStream out, LLM.LLMModel llm4Chat, JSONObject originalBody, JSONArray messages, String assistantContent, Map<Integer, ToolCall> toolCalls) throws IOException {
+ handleToolCallsAndContinue(out, llm4Chat, originalBody, messages, assistantContent, toolCalls, null);
+ }
+
+ public static void handleToolCallsAndContinue(ServletOutputStream out, LLM.LLMModel llm4Chat, JSONObject originalBody, JSONArray messages, String assistantContent, Map<Integer, ToolCall> toolCalls, final String runId) throws IOException {
+ final long start = System.currentTimeMillis();
try {
// Work on a copy so caller-owned message arrays are not modified unexpectedly.
JSONArray newMessages = new JSONArray();
@@ -272,9 +295,10 @@ public final class ToolCallProtocol {
for (int round = 0; round < MAX_TOOL_ROUNDS; round++) {
// Convert captured tool calls into assistant/tool messages and execute tools.
- ToolRoundData roundData = appendAssistantAndToolMessages(newMessages, roundAssistantContent, roundToolCalls, toolCallCounters);
+ ToolRoundData roundData = appendAssistantAndToolMessages(newMessages, roundAssistantContent, roundToolCalls, toolCallCounters, runId, round);
if (roundData == null || roundData.toolResults.length() == 0) {
// No executable tool calls; terminate stream cleanly.
+ log.info(prefix(runId) + "event=tool-round phase=end round=" + round + " result=no-executable-tool-calls requested=" + (roundToolCalls == null ? 0 : roundToolCalls.size()));
out.println("data: [DONE]");
out.flush();
return;
@@ -286,9 +310,11 @@ public final class ToolCallProtocol {
LLM.applyNoThinkingParameters(followup);
}
followup.put("messages", newMessages);
+ log.info(prefix(runId) + "event=tool-round phase=followup-call round=" + round + " toolResults=" + roundData.toolResults.length() + " messages=" + newMessages.length());
final HttpURLConnection followConn = openChatCompletionConnection(llm4Chat, followup);
if (followConn.getResponseCode() != 200) {
// Upstream error: close downstream stream instead of sending broken chunks.
+ log.warn(prefix(runId) + "event=tool-round phase=followup-return round=" + round + " result=upstream-error status=" + followConn.getResponseCode() + " reason=" + LogRedaction.redact(followConn.getResponseMessage()));
out.println("data: [DONE]");
out.flush();
followConn.disconnect();
@@ -337,16 +363,22 @@ public final class ToolCallProtocol {
// Ignore phantom rounds where stream signaled tool-calls but no concrete calls were parsed.
if (nextToolCalls.isEmpty()) sawToolCalls[0] = false;
// Final answer reached; caller already received forwarded stream lines.
- if (!sawToolCalls[0]) return;
+ if (!sawToolCalls[0]) {
+ log.info(prefix(runId) + "event=tool-round phase=end round=" + round + " result=final-answer toolResults=" + roundData.toolResults.length() + " assistantChars=" + nextAssistantContent.length() + " durationMs=" + (System.currentTimeMillis() - start));
+ return;
+ }
// Continue with next round tool request that was found in follow-up stream.
+ log.info(prefix(runId) + "event=tool-round phase=continue round=" + round + " nextToolCalls=" + nextToolCalls.size() + " assistantChars=" + nextAssistantContent.length());
roundAssistantContent = nextAssistantContent.toString();
roundToolCalls = nextToolCalls;
}
// Safety fallback when round cap is hit.
+ log.warn(prefix(runId) + "event=tool-lifecycle phase=end result=max-rounds maxRounds=" + MAX_TOOL_ROUNDS + " durationMs=" + (System.currentTimeMillis() - start));
out.println("data: [DONE]");
out.flush();
} catch (JSONException e) {
+ log.warn(prefix(runId) + "event=tool-lifecycle phase=end result=failure errorClass=" + e.getClass().getName() + " reason=" + LogRedaction.redactMessage(e) + " durationMs=" + (System.currentTimeMillis() - start));
throw new IOException("JSON processing error in tool handling", e);
}
}
@@ -442,6 +474,10 @@ public final class ToolCallProtocol {
* results, or {@code null} when JSON assembly fails
*/
private static ToolRoundData appendAssistantAndToolMessages(JSONArray messages, String assistantContent, Map<Integer, ToolCall> toolCalls, Map<String, Integer> toolCallCounters) {
+ return appendAssistantAndToolMessages(messages, assistantContent, toolCalls, toolCallCounters, null, -1);
+ }
+
+ private static ToolRoundData appendAssistantAndToolMessages(JSONArray messages, String assistantContent, Map<Integer, ToolCall> toolCalls, Map<String, Integer> toolCallCounters, final String runId, final int round) {
try {
// Keep original model call order stable by sorting call indices.
List<Integer> indices = new ArrayList<>(toolCalls.keySet());
@@ -460,7 +496,10 @@ public final class ToolCallProtocol {
final String toolName = call.name == null ? "" : call.name.trim();
final int maxCalls = net.yacy.ai.ToolProvider.maxCallsPerTurn(toolName);
final int usedCalls = toolCallCounters.getOrDefault(toolName, Integer.valueOf(0)).intValue();
- if (usedCalls >= maxCalls) continue;
+ if (usedCalls >= maxCalls) {
+ log.warn(prefix(runId) + "event=tool-execution phase=skip round=" + round + " tool=" + LogRedaction.redact(toolName) + " reason=max-calls used=" + usedCalls + " max=" + maxCalls);
+ continue;
+ }
// Assistant message representation of requested tool call.
JSONObject toolCallJson = new JSONObject(true);
@@ -473,7 +512,10 @@ public final class ToolCallProtocol {
toolCallsArray.put(toolCallJson);
// Execute tool locally and create "tool" role response message.
+ final long toolStart = System.currentTimeMillis();
+ log.info(prefix(runId) + "event=tool-execution phase=start round=" + round + " tool=" + LogRedaction.redact(call.name) + " argumentChars=" + (call.arguments == null ? 0 : call.arguments.length()));
String result = net.yacy.ai.ToolProvider.executeTool(call.name, call.arguments);
+ log.info(prefix(runId) + "event=tool-execution phase=end round=" + round + " tool=" + LogRedaction.redact(call.name) + " resultChars=" + (result == null ? 0 : result.length()) + " durationMs=" + (System.currentTimeMillis() - toolStart));
JSONObject toolMessage = new JSONObject(true);
toolMessage.put("role", "tool");
toolMessage.put("tool_call_id", call.id);
@@ -506,10 +548,15 @@ public final class ToolCallProtocol {
return new ToolRoundData(toolCallsArray, toolResults);
} catch (JSONException e) {
+ log.warn(prefix(runId) + "event=tool-execution phase=end round=" + round + " result=failure errorClass=" + e.getClass().getName() + " reason=" + LogRedaction.redactMessage(e));
return null;
}
}
+ private static String prefix(final String runId) {
+ return runId == null || runId.isEmpty() ? "" : "runId=" + runId + " ";
+ }
+
/**
* Injects tool metadata into one outbound SSE line if the line contains JSON
* data.
diff --git a/source/net/yacy/cora/util/LogRedaction.java b/source/net/yacy/cora/util/LogRedaction.java
new file mode 100644
index 000000000..9ccc28ccd
--- /dev/null
+++ b/source/net/yacy/cora/util/LogRedaction.java
@@ -0,0 +1,54 @@
+/**
+ * LogRedaction
+ * Copyright 2026 by contributors to the YaCy project
+ * First released 27.06.2026 at https://yacy.net
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program in the file lgpl21.txt
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.cora.util;
+
+import java.util.regex.Pattern;
+
+/**
+ * Redacts common credential shapes before values are written to operator logs.
+ */
+public final class LogRedaction {
+
+ private static final String REDACTED = "[REDACTED]";
+ private static final Pattern AUTHORIZATION_HEADER = Pattern.compile("(?i)(authorization\\s*[:=]\\s*)(bearer\\s+|basic\\s+)?([^\\s,;]+)");
+ private static final Pattern COOKIE_HEADER = Pattern.compile("(?i)(cookie\\s*[:=]\\s*)([^\\r\\n]+)");
+ private static final Pattern SENSITIVE_ASSIGNMENT = Pattern.compile("(?i)\\b(api[_-]?key|access[_-]?token|auth[_-]?token|token|secret|password|passwd|pwd|login)\\b\\s*[:=]\\s*([^\\s,;&]+)");
+ private static final Pattern SENSITIVE_QUERY = Pattern.compile("(?i)([?&](?:api[_-]?key|access[_-]?token|auth[_-]?token|token|secret|password|passwd|pwd|login)=)([^&#\\s]+)");
+ private static final Pattern URL_USERINFO = Pattern.compile("(?i)(https?://)([^/@\\s:]+):([^/@\\s]+)@");
+
+ private LogRedaction() {
+ }
+
+ public static String redact(final String value) {
+ if (value == null || value.isEmpty()) return value;
+ String redacted = URL_USERINFO.matcher(value).replaceAll("$1" + REDACTED + ":" + REDACTED + "@");
+ redacted = AUTHORIZATION_HEADER.matcher(redacted).replaceAll("$1$2" + REDACTED);
+ redacted = COOKIE_HEADER.matcher(redacted).replaceAll("$1" + REDACTED);
+ redacted = SENSITIVE_QUERY.matcher(redacted).replaceAll("$1" + REDACTED);
+ redacted = SENSITIVE_ASSIGNMENT.matcher(redacted).replaceAll("$1=" + REDACTED);
+ return redacted;
+ }
+
+ public static String redactMessage(final Throwable error) {
+ if (error == null) return "";
+ return redact(error.getMessage());
+ }
+}
diff --git a/source/net/yacy/htroot/AILab.java b/source/net/yacy/htroot/AILab.java
index 3ba072dad..b6aefa3d7 100644
--- a/source/net/yacy/htroot/AILab.java
+++ b/source/net/yacy/htroot/AILab.java
@@ -12,6 +12,7 @@ import org.json.JSONException;
import org.json.JSONObject;
import org.json.JSONTokener;
+import net.yacy.ai.LogReportService;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.search.Switchboard;
import net.yacy.server.serverObjects;
@@ -87,6 +88,7 @@ public class AILab {
prop.put("ailab_rag_status", hasEngine && hasModel && (hasRagRole || ragVisited) ? "ready" : "pending");
prop.put("ailab_tools_status", hasToolsConfig ? "ready" : "pending");
prop.put("ailab_shield_status", hasShield ? "ready" : "pending");
+ prop.put("ailab_logreports_status", LogReportService.hasConfiguredLogReportModel() ? "ready" : "pending");
return prop;
}
diff --git a/source/net/yacy/htroot/LLMSelection_p.java b/source/net/yacy/htroot/LLMSelection_p.java
index 03229a9ce..960a3e985 100644
--- a/source/net/yacy/htroot/LLMSelection_p.java
+++ b/source/net/yacy/htroot/LLMSelection_p.java
@@ -76,6 +76,11 @@ public class LLMSelection_p {
return service + "|" + hoststub + "|" + model;
}
+ private static boolean optBooleanRole(final JSONObject row, final String canonicalKey, final String displayKey) {
+ if (row == null) return false;
+ return row.optBoolean(canonicalKey, row.optBoolean(displayKey, false));
+ }
+
private static JSONObject normalizeProductionModelRow(final JSONObject row) throws JSONException {
final JSONObject normalized = new JSONObject(true);
normalized.put("service", row.optString("service", "OLLAMA"));
@@ -91,6 +96,7 @@ public class LLMSelection_p {
normalized.put("query", false);
normalized.put("qapairs", false);
normalized.put("tldr", row.optBoolean("tldr", false));
+ normalized.put("logreport", optBooleanRole(row, "logreport", "log-report"));
normalized.put("thinking", row.optBoolean("thinking", false));
normalized.put("tooling", row.optBoolean("tooling", false));
@@ -186,6 +192,7 @@ public class LLMSelection_p {
prop.put("productionmodels_" + i + "_query", row.optBoolean("query", false));
prop.put("productionmodels_" + i + "_qapairs", row.optBoolean("qapairs", false));
prop.put("productionmodels_" + i + "_tldr", row.optBoolean("tldr", false));
+ prop.put("productionmodels_" + i + "_logreport", row.optBoolean("logreport", false));
final String key = capabilityKey(row);
JSONObject capabilityEntry = key.isEmpty() ? null : capabilities.optJSONObject(key);
diff --git a/source/net/yacy/htroot/LogReports_p.java b/source/net/yacy/htroot/LogReports_p.java
new file mode 100644
index 000000000..2a62e20d1
--- /dev/null
+++ b/source/net/yacy/htroot/LogReports_p.java
@@ -0,0 +1,94 @@
+/**
+ * LogReports_p
+ * Copyright 2026 by contributors to the YaCy project
+ * First released 27.06.2026 at https://yacy.net
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+package net.yacy.htroot;
+
+import java.io.File;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+
+import net.yacy.ai.LogReportService;
+import net.yacy.ai.LogReportService.ReportEntry;
+import net.yacy.cora.protocol.RequestHeader;
+import net.yacy.search.Switchboard;
+import net.yacy.server.serverObjects;
+import net.yacy.server.serverSwitch;
+
+public class LogReports_p {
+
+ public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
+ final Switchboard sb = (Switchboard) env;
+ final serverObjects prop = new serverObjects();
+ final LogReportService service = new LogReportService(sb);
+ final File reportDirectory = service.getReportDirectory();
+
+ final int configuredMax = sb.getConfigInt(LogReportService.CONFIG_FEED_MAX_ENTRIES, LogReportService.DEFAULT_FEED_MAX_ENTRIES);
+ final int requestedMax = post == null ? Math.min(20, configuredMax) : post.getInt("count", Math.min(20, configuredMax));
+ final int maxEntries = Math.max(0, Math.min(requestedMax, configuredMax));
+
+ sb.setConfig("ui.LogReports_p.visited", "true");
+
+ prop.put("runReportResult", "0");
+ prop.putHTML("runReportFile", "");
+ prop.putHTML("runReportResult_runReportFile", "");
+ prop.putNum("runReportResult_runReportDurationSeconds", 0);
+ if (post != null && post.containsKey("runReportNow")) {
+ final long start = System.currentTimeMillis();
+ try {
+ final File reportFile = service.generateCurrentHourReportOverwrite();
+ final long durationSeconds = Math.max(0L, (System.currentTimeMillis() - start) / 1000L);
+ prop.putNum("runReportResult_runReportDurationSeconds", durationSeconds);
+ if (reportFile == null) {
+ prop.put("runReportResult", LogReportService.hasConfiguredLogReportModel() ? "2" : "3");
+ } else {
+ prop.put("runReportResult", "1");
+ prop.putHTML("runReportFile", reportFile.getName());
+ prop.putHTML("runReportResult_runReportFile", reportFile.getName());
+ }
+ } catch (final Exception e) {
+ final long durationSeconds = Math.max(0L, (System.currentTimeMillis() - start) / 1000L);
+ prop.putNum("runReportResult_runReportDurationSeconds", durationSeconds);
+ prop.put("runReportResult", "4");
+ final String message = e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage();
+ prop.putHTML("runReportFile", message);
+ prop.putHTML("runReportResult_runReportFile", message);
+ }
+ }
+
+ final List<ReportEntry> reports = service.discoverReports(maxEntries);
+
+ prop.put("modelConfigured", LogReportService.hasConfiguredLogReportModel() ? "1" : "0");
+ prop.putNum("count", maxEntries);
+ prop.putNum("maxcount", configuredMax);
+ prop.putHTML("reportdir", reportDirectory.getAbsolutePath());
+ prop.put("reportdirExists", reportDirectory.isDirectory() ? "1" : "0");
+ prop.put("reportdirMissing", reportDirectory.isDirectory() ? "0" : "1");
+ prop.put("jsonFeed", "api/logreports.json?count=" + maxEntries);
+ prop.put("rssFeed", "api/logreports.rss?count=" + maxEntries);
+
+ for (int i = 0; i < reports.size(); i++) {
+ final ReportEntry report = reports.get(i);
+ prop.putHTML("reports_" + i + "_filename", report.filename);
+ prop.putHTML("reports_" + i + "_title", report.title);
+ prop.putHTML("reports_" + i + "_published", report.published.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME));
+ prop.putHTML("reports_" + i + "_type", report.daily ? "daily" : "hourly");
+ prop.put("reports_" + i + "_daily", report.daily ? "1" : "0");
+ prop.putNum("reports_" + i + "_chars", report.content.length());
+ prop.putHTML("reports_" + i + "_content", report.content);
+ }
+ prop.put("reports", reports.size());
+ prop.putNum("reportCount", reports.size());
+ prop.put("hasReports", reports.isEmpty() ? "0" : "1");
+ prop.put("noReports", reports.isEmpty() ? "1" : "0");
+
+ return prop;
+ }
+}
diff --git a/source/net/yacy/htroot/ViewLog_p.java b/source/net/yacy/htroot/ViewLog_p.java
index 3df1a45d0..06e51255a 100644
--- a/source/net/yacy/htroot/ViewLog_p.java
+++ b/source/net/yacy/htroot/ViewLog_p.java
@@ -33,17 +33,18 @@
package net.yacy.htroot;
import java.util.logging.Handler;
-import java.util.logging.Logger;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.regex.PatternSyntaxException;
-
-import net.yacy.cora.protocol.RequestHeader;
-import net.yacy.cora.util.ConcurrentLog;
-import net.yacy.kelondro.logging.GuiHandler;
-import net.yacy.kelondro.logging.LogalizerHandler;
-import net.yacy.server.serverObjects;
-import net.yacy.server.serverSwitch;
+import java.util.logging.Logger;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+import net.yacy.cora.protocol.RequestHeader;
+import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.cora.util.LogRedaction;
+import net.yacy.kelondro.logging.GuiHandler;
+import net.yacy.kelondro.logging.LogalizerHandler;
+import net.yacy.server.serverObjects;
+import net.yacy.server.serverSwitch;
public class ViewLog_p {
@@ -58,20 +59,26 @@ public class ViewLog_p {
* for the user to input regexes like ".*FOO.*" in the HTML
* interface.
*/
- String filter = ".*.*";
-
- if (post != null){
- reversed = (post.containsKey("mode") && "reversed".equals(post.get("mode")));
- json = post.containsKey("json");
+ String filter = ".*.*";
+ String filterMode = "regex";
+
+ if (post != null){
+ reversed = (post.containsKey("mode") && "reversed".equals(post.get("mode")));
+ json = post.containsKey("json");
if(post.containsKey("lines")){
lines = post.getInt("lines", lines);
}
- if(post.containsKey("filter")){
- filter = post.get("filter");
- }
- }
+ if(post.containsKey("filter")){
+ filter = post.get("filter");
+ }
+
+ if(post.containsKey("filterMode")){
+ filterMode = post.get("filterMode");
+ if (!"terms".equals(filterMode)) filterMode = "regex";
+ }
+ }
final Logger logger = Logger.getLogger("");
final Handler[] handlers = logger.getHandlers();
@@ -88,29 +95,44 @@ public class ViewLog_p {
prop.put("submenu", displaySubmenu ? "1" : "0");
prop.put("reverseChecked", reversed ? "1" : "0");
- prop.put("lines", lines);
- prop.put("maxlines", maxlines);
- prop.putHTML("filter", filter);
-
- // trying to compile the regular expression filter expression
- Matcher filterMatcher = null;
- try {
- final Pattern filterPattern = Pattern.compile(filter,Pattern.MULTILINE);
- filterMatcher = filterPattern.matcher("");
- } catch (final PatternSyntaxException e) {
- ConcurrentLog.logException(e);
- }
-
- int level = 0;
- int lc = 0;
- for (final String logLine : log) {
- if (logLine == null) break;
- final String nextLogLine = logLine.trim();
-
- if (filterMatcher != null) {
- filterMatcher.reset(nextLogLine);
- if (!filterMatcher.find()) continue;
- }
+ prop.put("lines", lines);
+ prop.put("maxlines", maxlines);
+ prop.putHTML("filter", filter);
+ prop.put("filterMode", filterMode);
+ prop.put("filterModeRegex", "regex".equals(filterMode) ? "1" : "0");
+ prop.put("filterModeTerms", "terms".equals(filterMode) ? "1" : "0");
+
+ Matcher filterMatcher = null;
+ String[] filterTerms = new String[0];
+ boolean validFilter = true;
+ if ("terms".equals(filterMode)) {
+ filterTerms = parseFilterTerms(filter);
+ } else {
+ // trying to compile the regular expression filter expression
+ try {
+ final Pattern filterPattern = Pattern.compile(filter,Pattern.MULTILINE);
+ filterMatcher = filterPattern.matcher("");
+ } catch (final PatternSyntaxException e) {
+ validFilter = false;
+ ConcurrentLog.warn("ViewLog", "Invalid log regex filter: " + LogRedaction.redactMessage(e));
+ }
+ }
+ prop.put("filterError", validFilter ? "0" : "1");
+
+ int level = 0;
+ int lc = 0;
+ for (final String logLine : log) {
+ if (logLine == null) break;
+ final String nextLogLine = logLine.trim();
+
+ if ("terms".equals(filterMode)) {
+ if (!matchesAllTerms(nextLogLine, filterTerms)) continue;
+ } else if (validFilter && filterMatcher != null) {
+ filterMatcher.reset(nextLogLine);
+ if (!filterMatcher.find()) continue;
+ } else if (!validFilter) {
+ continue;
+ }
if (nextLogLine.startsWith("E ")) {
level = 4;
@@ -136,7 +158,30 @@ public class ViewLog_p {
}
prop.put("log", lc);
- // return rewrite properties
- return prop;
- }
-}
+ // return rewrite properties
+ return prop;
+ }
+
+ private static String[] parseFilterTerms(final String filter) {
+ if (filter == null || filter.trim().isEmpty()) return new String[0];
+ final String[] rawTerms = filter.split(",");
+ int count = 0;
+ for (int i = 0; i < rawTerms.length; i++) {
+ final String term = rawTerms[i].trim().toLowerCase();
+ if (term.isEmpty()) continue;
+ rawTerms[count++] = term;
+ }
+ final String[] terms = new String[count];
+ System.arraycopy(rawTerms, 0, terms, 0, count);
+ return terms;
+ }
+
+ private static boolean matchesAllTerms(final String line, final String[] terms) {
+ if (terms == null || terms.length == 0) return true;
+ final String normalizedLine = line == null ? "" : line.toLowerCase();
+ for (final String term : terms) {
+ if (!normalizedLine.contains(term)) return false;
+ }
+ return true;
+ }
+}
diff --git a/source/net/yacy/htroot/api/logreports.java b/source/net/yacy/htroot/api/logreports.java
new file mode 100644
index 000000000..c13e293fd
--- /dev/null
+++ b/source/net/yacy/htroot/api/logreports.java
@@ -0,0 +1,64 @@
+/**
+ * logreports
+ * Copyright 2026 by contributors to the YaCy project
+ * First released 26.06.2026 at https://yacy.net
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+package net.yacy.htroot.api;
+
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+
+import net.yacy.ai.LogReportService;
+import net.yacy.ai.LogReportService.ReportEntry;
+import net.yacy.cora.protocol.RequestHeader;
+import net.yacy.search.Switchboard;
+import net.yacy.server.serverObjects;
+import net.yacy.server.serverSwitch;
+
+public class logreports {
+
+ public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
+ final serverObjects prop = new serverObjects();
+ prop.put("authorized", "0");
+ prop.put("reports", "0");
+ prop.putXML("channel_title", "YaCy Log Reports");
+ prop.putXML("channel_description", "Generated YaCy self-enhancement log reports");
+ prop.put("channel_pubDate", "");
+
+ if (header == null || env == null) return prop;
+ final Switchboard sb = (Switchboard) env;
+ if (!sb.verifyAuthentication(header)) return prop;
+ prop.put("authorized", "1");
+
+ final int configuredMax = sb.getConfigInt(LogReportService.CONFIG_FEED_MAX_ENTRIES, LogReportService.DEFAULT_FEED_MAX_ENTRIES);
+ final int requestedMax = post == null ? configuredMax : post.getInt("count", configuredMax);
+ final int maxEntries = Math.max(0, Math.min(requestedMax, configuredMax));
+ final List<ReportEntry> reports = new LogReportService(sb).discoverReports(maxEntries);
+
+ for (int i = 0; i < reports.size(); i++) {
+ final ReportEntry report = reports.get(i);
+ prop.putJSON("reports_" + i + "_filename", report.filename);
+ prop.putJSON("reports_" + i + "_title", report.title);
+ prop.put("reports_" + i + "_published", report.published.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME));
+ prop.putJSON("reports_" + i + "_content", report.content);
+ prop.put("reports_" + i + "_daily", report.daily ? "1" : "0");
+ prop.put("reports_" + i + "_comma", i + 1 < reports.size() ? "1" : "0");
+
+ prop.putXML("reports_" + i + "_title-rss", report.title);
+ prop.putXML("reports_" + i + "_description-rss", report.content);
+ prop.put("reports_" + i + "_pubDate-rss", report.published.format(DateTimeFormatter.RFC_1123_DATE_TIME));
+ prop.putXML("reports_" + i + "_guid-rss", report.filename);
+ }
+ prop.put("reports", reports.size());
+ if (!reports.isEmpty()) {
+ prop.put("channel_pubDate", reports.get(0).published.format(DateTimeFormatter.RFC_1123_DATE_TIME));
+ }
+ return prop;
+ }
+}
diff --git a/source/net/yacy/http/servlets/RAGProxyServlet.java b/source/net/yacy/http/servlets/RAGProxyServlet.java
index 5875dc4fd..c41789e9c 100644
--- a/source/net/yacy/http/servlets/RAGProxyServlet.java
+++ b/source/net/yacy/http/servlets/RAGProxyServlet.java
@@ -28,6 +28,7 @@ import java.util.ArrayList;
import java.util.Base64;
import java.util.Deque;
import java.util.List;
+import java.util.UUID;
import java.util.concurrent.ConcurrentLinkedDeque;
import javax.servlet.ServletException;
@@ -48,6 +49,7 @@ import net.yacy.ai.RAGAugmentor;
import net.yacy.ai.ToolCallProtocol;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.cora.util.LogRedaction;
import net.yacy.search.Switchboard;
/**
@@ -83,6 +85,8 @@ public class RAGProxyServlet extends HttpServlet {
@Override
public void service(ServletRequest request, ServletResponse response) throws IOException, ServletException {
+ final String runId = UUID.randomUUID().toString();
+ final long requestStart = System.currentTimeMillis();
response.setContentType("application/json;charset=utf-8");
HttpServletResponse hresponse = (HttpServletResponse) response;
@@ -96,15 +100,18 @@ public class RAGProxyServlet extends HttpServlet {
final Switchboard sb = Switchboard.getSwitchboard();
final String clientIP = hrequest.getRemoteAddr();
final boolean localhostAccess = Domains.isLocalhost(clientIP);
+ ConcurrentLog.info("RAGProxy", "runId=" + runId + " event=rag-request phase=start method=" + hrequest.getMethod() + " localhost=" + localhostAccess);
if (!localhostAccess) {
// obey the allow-nonlocalhost shield setting
final boolean allowNonLocal = sb.getConfigBool("ai.shield.allow-nonlocalhost", false);
if (!allowNonLocal) {
+ ConcurrentLog.warn("RAGProxy", "runId=" + runId + " event=rag-request phase=reject reason=nonlocalhost-blocked durationMs=" + elapsed(requestStart));
hresponse.sendError(HttpServletResponse.SC_FORBIDDEN);
return;
}
}
if (isRateLimited(sb, clientIP, localhostAccess)) {
+ ConcurrentLog.warn("RAGProxy", "runId=" + runId + " event=rag-request phase=reject reason=rate-limited localhost=" + localhostAccess + " durationMs=" + elapsed(requestStart));
hresponse.sendError(429, "Too Many Requests"); // standard status for rate limits
return;
}
@@ -114,11 +121,13 @@ public class RAGProxyServlet extends HttpServlet {
if (reqMethod == Method.OTHER) {
// required to handle CORS
hresponse.setStatus(HttpServletResponse.SC_OK);
+ ConcurrentLog.info("RAGProxy", "runId=" + runId + " event=rag-request phase=end result=options durationMs=" + elapsed(requestStart));
return;
}
// We expect a POST request
if (reqMethod != Method.POST) {
+ ConcurrentLog.warn("RAGProxy", "runId=" + runId + " event=rag-request phase=reject reason=method-not-allowed method=" + hrequest.getMethod() + " durationMs=" + elapsed(requestStart));
hresponse.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED);
return;
}
@@ -135,6 +144,7 @@ public class RAGProxyServlet extends HttpServlet {
bodyBuilder.append(line);
}
String body = bodyBuilder.toString();
+ ConcurrentLog.info("RAGProxy", "runId=" + runId + " event=rag-request phase=body-read bodyChars=" + body.length());
JSONObject bodyObject;
try {
// get system message and user prompt
@@ -148,8 +158,13 @@ public class RAGProxyServlet extends HttpServlet {
// resolve true model name from configuration
LLM.LLMUsage usage = LLM.LLMUsage.chat;
try {usage = LLM.LLMUsage.valueOf(model);} catch (IllegalArgumentException e) {}
- LLM.LLMModel llm4Chat = LLM.llmFromUsage(usage);
- LLM.LLMModel llm4tldr = LLM.llmFromUsage(LLM.LLMUsage.tldr);
+ LLM.LLMModel llm4Chat = LLM.llmFromUsage(usage, runId, "rag-chat");
+ LLM.LLMModel llm4tldr = LLM.llmFromUsage(LLM.LLMUsage.tldr, runId, "rag-query-generator");
+ if (llm4Chat == null) {
+ ConcurrentLog.warn("RAGProxy", "runId=" + runId + " event=rag-request phase=reject reason=no-chat-model usage=" + usage + " durationMs=" + elapsed(requestStart));
+ hresponse.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE, "No chat model configured");
+ return;
+ }
bodyObject.put("model", llm4Chat.model); // replace the model with the decoded model name
// get messages and prepare user message attachments
@@ -182,7 +197,7 @@ public class RAGProxyServlet extends HttpServlet {
user = userObject.getContentText(); // this is the latest user prompt
ragMode = userObject.getSearchMode();
}
- ConcurrentLog.info("RAGProxy", "ragMode=" + ragMode + " userChars=" + (user == null ? 0 : user.length()));
+ ConcurrentLog.info("RAGProxy", "runId=" + runId + " event=rag-request phase=messages messages=" + messages.length() + " lastUserIndex=" + lastUserIndex + " ragMode=" + ragMode + " userChars=" + (user == null ? 0 : user.length()));
//List<DataURL> data_urls = userObject.getContentAttachments(); // this list is a copy of the content data_urls
// RAG
@@ -194,17 +209,26 @@ public class RAGProxyServlet extends HttpServlet {
final long queryStart = System.currentTimeMillis();
if (countWords(user) <= DIRECT_SEARCH_WORD_LIMIT) {
searchResultQuery = user;
+ ConcurrentLog.info("RAGProxy", "runId=" + runId + " event=rag-query phase=select source=direct userWords=" + countWords(user));
} else {
- searchResultQuery = RAGAugmentor.searchWordsForPrompt(llm4tldr.llm, llm4tldr.model, user, queryPrefix); // might return null in case any error occurred
- if (searchResultQuery == null || searchResultQuery.length() == 0) searchResultQuery = user; // in case there is an error we simply search with the prompt
+ if (llm4tldr == null) {
+ searchResultQuery = user;
+ ConcurrentLog.warn("RAGProxy", "runId=" + runId + " event=rag-query phase=fallback reason=no-tldr-model userWords=" + countWords(user));
+ } else {
+ searchResultQuery = RAGAugmentor.searchWordsForPrompt(llm4tldr.llm, llm4tldr.model, user, queryPrefix, runId); // might return null in case any error occurred
+ if (searchResultQuery == null || searchResultQuery.length() == 0) {
+ searchResultQuery = user; // in case there is an error we simply search with the prompt
+ ConcurrentLog.warn("RAGProxy", "runId=" + runId + " event=rag-query phase=fallback reason=query-generation-empty userWords=" + countWords(user));
+ }
+ }
}
final long queryElapsed = System.currentTimeMillis() - queryStart;
final long searchStart = System.currentTimeMillis();
- searchResultMarkdown = RAGAugmentor.searchResultsAsMarkdown(searchResultQuery, 10, "global".equals(ragMode));
+ searchResultMarkdown = RAGAugmentor.searchResultsAsMarkdown(searchResultQuery, 10, "global".equals(ragMode), runId);
final long searchElapsed = System.currentTimeMillis() - searchStart;
ConcurrentLog.info(
"RAGProxy",
- "searchQuery=\"" + searchResultQuery + "\" queryMs=" + queryElapsed + " searchMs=" + searchElapsed +
+ "runId=" + runId + " event=rag-retrieval phase=end ragMode=" + ragMode + " queryChars=" + searchResultQuery.length() + " queryWords=" + countWords(searchResultQuery) + " queryMs=" + queryElapsed + " searchMs=" + searchElapsed +
" markdownChars=" + searchResultMarkdown.length());
user += userPrefix;
user += searchResultMarkdown;
@@ -217,14 +241,20 @@ public class RAGProxyServlet extends HttpServlet {
}
// ToolCallProtocol owns request preparation, initial stream handling and follow-up tool rounds.
- final int status = ToolCallProtocol.proxyToolLifecycle(out, llm4Chat, bodyObject, messages, initialMetadata);
+ final int status = ToolCallProtocol.proxyToolLifecycle(out, llm4Chat, bodyObject, messages, initialMetadata, runId);
hresponse.setStatus(status);
out.close(); // close this here to end transmission
+ ConcurrentLog.info("RAGProxy", "runId=" + runId + " event=rag-request phase=end result=success status=" + status + " durationMs=" + elapsed(requestStart));
} catch (JSONException e) {
+ ConcurrentLog.warn("RAGProxy", "runId=" + runId + " event=rag-request phase=end result=failure errorClass=" + e.getClass().getName() + " reason=" + LogRedaction.redactMessage(e) + " durationMs=" + elapsed(requestStart));
throw new IOException(e.getMessage());
}
}
+ private static long elapsed(final long start) {
+ return System.currentTimeMillis() - start;
+ }
+
private static int countWords(final String text) {
if (text == null) return 0;
final String trimmed = text.trim();
diff --git a/source/net/yacy/kelondro/logging/GuiHandler.java b/source/net/yacy/kelondro/logging/GuiHandler.java
index c1366e31d..e9c1684b0 100644
--- a/source/net/yacy/kelondro/logging/GuiHandler.java
+++ b/source/net/yacy/kelondro/logging/GuiHandler.java
@@ -43,7 +43,7 @@ import net.yacy.kelondro.util.MemoryControl;
public class GuiHandler extends Handler {
- private final static int DEFAULT_SIZE = 10000; // don't make this too big, it eats up a lot of memory!
+ private final static int DEFAULT_SIZE = 100000; // don't make this too big, it eats up a lot of memory!
private static int size = DEFAULT_SIZE;
private static String buffer[];
private static int start, count;
diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java
index 1425b7976..1a7166eb4 100644
--- a/source/net/yacy/yacy.java
+++ b/source/net/yacy/yacy.java
@@ -42,9 +42,10 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
+import java.util.Properties;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
import org.apache.http.Header;
import org.apache.http.HttpStatus;
@@ -60,9 +61,10 @@ import net.yacy.cora.protocol.ClientIdentification;
import net.yacy.cora.protocol.ConnectionInfo;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.TimeoutRequest;
-import net.yacy.cora.protocol.http.HTTPClient;
-import net.yacy.cora.util.ConcurrentLog;
-import net.yacy.data.TransactionManager;
+import net.yacy.cora.protocol.http.HTTPClient;
+import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.ai.LogReportService;
+import net.yacy.data.TransactionManager;
import net.yacy.data.Translator;
import net.yacy.gui.YaCyApp;
import net.yacy.gui.framework.Browser;
@@ -290,13 +292,14 @@ public final class yacy {
final int deleteOldDownloadsAfterDays = (int) sb.getConfigLong("update.deleteOld", 30);
yacyRelease.deleteOldDownloads(sb.releasePath, deleteOldDownloadsAfterDays );
- // start main threads
- final int port = sb.getLocalPort();
- final String host = sb.getLocalHost();
- try {
- // start http server
- YaCyHttpServer httpServer;
- httpServer = new YaCyHttpServer(port, host);
+ // start main threads
+ final int port = sb.getLocalPort();
+ final String host = sb.getLocalHost();
+ ScheduledExecutorService logReportScheduler = null;
+ try {
+ // start http server
+ YaCyHttpServer httpServer;
+ httpServer = new YaCyHttpServer(port, host);
httpServer.startupServer();
sb.setHttpServer(httpServer);
// TODO: this has no effect on Jetty (but needed to reflect configured value and limit is still used)
@@ -371,11 +374,13 @@ public final class yacy {
}
}
// initialize number formatter with this locale
- if (!lang.equals("browser")) // "default" is handled by .setLocale()
- Formatter.setLocale(lang);
-
- // registering shutdown hook
- ConcurrentLog.config("STARTUP", "Registering Shutdown Hook");
+ if (!lang.equals("browser")) // "default" is handled by .setLocale()
+ Formatter.setLocale(lang);
+
+ logReportScheduler = LogReportService.startScheduler(sb);
+
+ // registering shutdown hook
+ ConcurrentLog.config("STARTUP", "Registering Shutdown Hook");
final Runtime run = Runtime.getRuntime();
run.addShutdownHook(new shutdownHookThread(sb, shutdownSemaphore));
@@ -394,16 +399,18 @@ public final class yacy {
} catch (final Exception e) {
ConcurrentLog.severe("MAIN CONTROL LOOP", "PANIC: " + e.getMessage(),e);
}
- // shut down
- ConcurrentLog.config("SHUTDOWN", "caught termination signal");
- httpServer.stop();
-
- ConcurrentLog.config("SHUTDOWN", "server has terminated");
- sb.close();
- } catch (final Exception e) {
- ConcurrentLog.severe("STARTUP", "Unexpected Error: " + e.getClass().getName(),e);
- //System.exit(1);
- }
+ // shut down
+ ConcurrentLog.config("SHUTDOWN", "caught termination signal");
+ LogReportService.stopScheduler(logReportScheduler);
+ httpServer.stop();
+
+ ConcurrentLog.config("SHUTDOWN", "server has terminated");
+ sb.close();
+ } catch (final Exception e) {
+ LogReportService.stopScheduler(logReportScheduler);
+ ConcurrentLog.severe("STARTUP", "Unexpected Error: " + e.getClass().getName(),e);
+ //System.exit(1);
+ }
if (lock != null && lock.isValid()) lock.release();
if (channel != null && channel.isOpen()) channel.close();
} catch (final Exception ee) {