added log report generator for recursive self-improvement

8 files changed, 227 insertions, 14 deletions

diff --git a/htroot/AILab.html b/htroot/AILab.html
index 1b52b4817..78e01a404 100644
--- a/htroot/AILab.html
+++ b/htroot/AILab.html
@@ -331,6 +331,26 @@
           </div>
         </div>
 
+        <div class="bs-callout bs-callout-optional quest-callout status-pending" data-quest="logreports" data-status="#[ailab_logreports_status]#">
+          <div class="quest-meta">
+            <span class="label label-info">Optional</span>
+            <span class="meta-arrow" aria-hidden="true">→</span>
+            <span class="status-pill">Needs setup</span>
+          </div>
+          <h3>Monitor log reports</h3>
+          <p>Assign a log-report model, then review generated hourly and daily self-enhancement reports.</p>
+          <div class="quest-body">
+            <div class="quest-visual">
+              <img src="env/grafics/AILab_Matrix.png" alt="Log report monitor" width="128" height="128" />
+            </div>
+            <div class="quest-actions">
+              <a class="btn btn-info btn-sm" href="LogReports_p.html">Open log reports</a>
+              <a class="btn btn-info btn-sm" href="LLMSelection_p.html#availableModels">Assign log-report model</a><br />
+              <span class="quest-note">Report generation stays inactive until a production model is assigned to the log-report role.</span>
+            </div>
+          </div>
+        </div>
+
         <div class="bs-callout bs-callout-optional quest-callout status-pending" data-quest="shield" data-status="#[ailab_shield_status]#">
           <div class="quest-meta">
             <span class="label label-info">Optional</span>
diff --git a/htroot/LLMSelection_p.html b/htroot/LLMSelection_p.html
index 6e3e54f22..628ab1211 100644
--- a/htroot/LLMSelection_p.html
+++ b/htroot/LLMSelection_p.html
@@ -53,12 +53,12 @@
         required: ["mood"]
       };
 
-      const PRODUCTION_MODEL_TOTAL_COLUMNS = 17;
+      const PRODUCTION_MODEL_TOTAL_COLUMNS = 18;
       const PRODUCTION_MODEL_MODEL_COLUMN_INDEX = 1;
       const PRODUCTION_MODEL_USAGE_COLUMN_START = 5;
-      const PRODUCTION_MODEL_USAGE_COLUMN_END = 11; // including
-      const PRODUCTION_MODEL_FEATURE_COLUMN_START = 12;
-      const PRODUCTION_MODEL_FEATURE_COLUMN_END = 15; // including
+      const PRODUCTION_MODEL_USAGE_COLUMN_END = 12; // including
+      const PRODUCTION_MODEL_FEATURE_COLUMN_START = 13;
+      const PRODUCTION_MODEL_FEATURE_COLUMN_END = 16; // including
       const PRODUCTION_MODEL_ACTION_COLUMN_INDEX = PRODUCTION_MODEL_TOTAL_COLUMNS - 1;
       const PRODUCTION_MODEL_THINKING_COLUMN_INDEX = PRODUCTION_MODEL_FEATURE_COLUMN_START;
       const PRODUCTION_MODEL_TOOLING_COLUMN_INDEX = PRODUCTION_MODEL_FEATURE_COLUMN_START + 1;
@@ -66,7 +66,8 @@
       const PRODUCTION_MODEL_FORMAT_COLUMN_INDEX = PRODUCTION_MODEL_FEATURE_COLUMN_START + 3;
       const PRODUCTION_MODEL_ENABLED_USAGE_COLUMNS = new Set([
         6,  // chat
-        11  // tldr
+        11, // tldr
+        12  // log-report
       ]);
       const PRODUCTION_MODEL_COLUMN_NAMES = [
         "service",
@@ -82,6 +83,7 @@
         "query",
         "qapairs",
         "tldr",
+        "logreport",
 
         "thinking",
         "tooling",
@@ -111,7 +113,7 @@
         //["qwen3-vl:30b-a3b-instruct-q4_K_M",                  "17.33", "22GB", "Very fast, exceptional good 30B model, ranking above GPT-4-turbo, GPT-4.1-nano, GPT-o1, GPT-4o-mini", "Alibaba", "apache-2.0"],
 		["qwen3.5:9b-q4_K_M",                                 "18.37", "14GB", "multimodal, outstanding for its size, long-context 256K Tokens, strong instruction following model", "Alibaba", "apache-2.0"],
 		["hf.co/mradermacher/Ling-mini-2.0-GGUF:Q4_K_M",      "20.00", "20GB", "very fast 16B MoE model with 1.4B activated parameters per expert", "InclusionAI", "mit"],
-		["qwen3.5:35b-a3b-q4_K_M",                            "44.63", "25GB", "Very fast MoE, exceptional good 35B model with vision, ranking above GPT-4-turbo, GPT-4.1-nano, GPT-o1, GPT-4o-mini", "Alibaba", "apache-2.0"]
+		["qwen3.6:27b-mtp-q4_K_M",                            "80", "17GB", "Exceptional good 27B model with vision, ranking above GPT-4-turbo, GPT-4.1-nano, GPT-o1, GPT-4o-mini", "Alibaba", "apache-2.0"]
       ];
 	  
       const MODEL_TABLE_HEADERS = ["Model", "Ranking", "Size", "Description", "Provider", "License", "Actions"];
@@ -1871,6 +1873,7 @@
                     <td class="narrow">search-query<br/><span class="info"><img src="env/grafics/i16.gif" width="16" height="16" alt="info"/><span>This model produces search queries to YaCy search from prompts in RAG or chat</span></span></td>
                     <td class="narrow">qa-pairs<br/><span class="info"><img src="env/grafics/i16.gif" width="16" height="16" alt="info"/><span>This model can be used to produce query-answer pairs which enhance search from chat prompts</span></span></td>
                     <td class="narrow">tldr-shortener<br/><span class="info"><img src="env/grafics/i16.gif" width="16" height="16" alt="info"/><span>This model is used to make summaries from web content</span></span></td>
+                    <td class="narrow">log-report<br/><span class="info"><img src="env/grafics/i16.gif" width="16" height="16" alt="info"/><span>This model evaluates YaCy runtime logs and creates self-enhancement reports</span></span></td>
                     
                     <td class="narrow">thinking<br/><span class="info"><img src="env/grafics/i16.gif" width="16" height="16" alt="info"/><span>we detect thinking only to be able to suppress thinking. thinking is not used in YaCy</span></span></td>
                     <td class="narrow">tooling<br/><span class="info"><img src="env/grafics/i16.gif" width="16" height="16" alt="info"/><span>tooling is required for agentic abilities.</span></span></td>
@@ -1895,6 +1898,7 @@
                 <td><input type="checkbox" #(query)#::checked=true#(/query)# disabled="disabled"></td>
                 <td><input type="checkbox" #(qapairs)#::checked=true#(/qapairs)# disabled="disabled"></td>
                 <td><input type="checkbox" #(tldr)#::checked=true#(/tldr)#></td>
+                <td><input type="checkbox" #(logreport)#::checked=true#(/logreport)#></td>
                 
                 <td>#[thinking]#</td>
                 <td>#[tooling]#</td>
diff --git a/htroot/LogReports_p.html b/htroot/LogReports_p.html
new file mode 100644
index 000000000..a99a20eb4
--- /dev/null
+++ b/htroot/LogReports_p.html
@@ -0,0 +1,141 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+  <head>
+    <title>YaCy '#[clientname]#': Log Reports</title>
+    #%env/templates/metas.template%#
+    <style type="text/css">
+      .logreport-feeds {
+        margin: 0 0 16px 0;
+      }
+      .logreport-card {
+        border: 1px solid #ddd;
+        margin: 0 0 16px 0;
+        background: #fff;
+      }
+      .logreport-header {
+        background: #f5f5f5;
+        border-bottom: 1px solid #ddd;
+        padding: 10px 12px;
+      }
+      .logreport-title {
+        font-weight: bold;
+        margin: 0 0 4px 0;
+      }
+      .logreport-meta {
+        color: #666;
+        font-size: 0.9em;
+      }
+      .logreport-body {
+        white-space: pre-wrap;
+        word-break: break-word;
+        max-height: 520px;
+        overflow: auto;
+        margin: 0;
+        padding: 12px;
+        background: #fff;
+        border: 0;
+        font-family: Menlo, Consolas, monospace;
+        font-size: 12px;
+        line-height: 1.45;
+      }
+      .logreport-empty {
+        border: 1px solid #ddd;
+        background: #f9f9f9;
+        padding: 14px;
+      }
+      .logreport-warning {
+        border: 1px solid #f0ad4e;
+        border-left: 6px solid #f0ad4e;
+        background: #fcf8e3;
+        color: #6b4f13;
+        padding: 12px;
+        margin: 10px 0 16px 0;
+      }
+      .logreport-actions {
+        margin: 0 0 16px 0;
+      }
+      .logreport-message {
+        border: 1px solid #5bc0de;
+        border-left: 6px solid #5bc0de;
+        background: #eef9fc;
+        color: #245269;
+        padding: 12px;
+        margin: 10px 0 16px 0;
+      }
+    </style>
+  </head>
+  <body id="LogReports">
+    #%env/templates/header.template%#
+    #%env/templates/submenuComputation.template%#
+
+    <h2>Log Reports</h2>
+
+    <form action="LogReports_p.html" method="get" accept-charset="UTF-8">
+      <fieldset>
+        <label for="count">Reports</label>
+        <input type="number" name="count" id="count" min="0" max="#[maxcount]#" value="#[count]#" />
+        <input type="submit" value="refresh" />
+      </fieldset>
+    </form>
+
+    <form action="LogReports_p.html" method="post" accept-charset="UTF-8" class="logreport-actions">
+      <input type="hidden" name="count" value="#[count]#" />
+      <button type="submit" name="runReportNow" value="1" class="btn btn-primary">run report now</button>
+    </form>
+
+    #(runReportResult)#
+    ::
+    <div class="logreport-message">Current-hour report written: #[runReportFile]# (#[runReportDurationSeconds]# seconds)</div>
+    ::
+    <div class="logreport-warning">No log lines were found for the current hour.</div>
+    ::
+    <div class="logreport-warning">
+      No production model is configured for the log-report role. Assign one in the
+      <a href="LLMSelection_p.html#availableModels">Production Models Matrix</a>.
+    </div>
+    ::
+    <div class="logreport-warning">Report generation failed after #[runReportDurationSeconds]# seconds: #[runReportFile]#</div>
+    #(/runReportResult)#
+
+    #(modelConfigured)#
+    <div class="logreport-warning">
+      No production model is configured for the log-report role. Log report generation stays inactive until a model is assigned in the
+      <a href="LLMSelection_p.html#availableModels">Production Models Matrix</a>.
+    </div>
+    ::
+    #(/modelConfigured)#
+
+    <p class="logreport-feeds">
+      Reports found: #[reportCount]#<br />
+      Directory: #[reportdir]#<br />
+      Feeds:
+      <a href="#[jsonFeed]#">JSON</a>
+      |
+      <a href="#[rssFeed]#">RSS</a>
+    </p>
+
+    #(reportdirMissing)#
+    ::
+    <div class="logreport-empty">The report directory does not exist yet. Reports will appear here after the scheduler has generated the first completed hourly report.</div>
+    #(/reportdirMissing)#
+
+    #{reports}#
+    <div class="logreport-card">
+      <div class="logreport-header">
+        <div class="logreport-title">#[title]#</div>
+        <div class="logreport-meta">
+          #[type]# report - #[filename]# - #[published]# - #[chars]# characters
+        </div>
+      </div>
+      <pre class="logreport-body">#[content]#</pre>
+    </div>
+    #{/reports}#
+
+    #(noReports)#
+    ::
+    <div class="logreport-empty">No generated log reports were found.</div>
+    #(/noReports)#
+
+    #%env/templates/footer.template%#
+  </body>
+</html>
diff --git a/htroot/ViewLog_p.html b/htroot/ViewLog_p.html
index 6ef10300d..6efb12d11 100644
--- a/htroot/ViewLog_p.html
+++ b/htroot/ViewLog_p.html
@@ -11,14 +11,19 @@
     

     <form action="ViewLog_p.html" method="get" accept-charset="UTF-8">

       <fieldset>

-        <input type="text" name="lines" id="lines" size ="6" value="#[lines]#" maxlength="9" /><label for="lines"> Lines (max. #[maxlines]#)</label>

-        <input type="checkbox" name="mode" id="mode" value="reversed" #(reverseChecked)#::checked="checked"#(/reverseChecked)# /><label for="mode">reversed order</label>

-        <input type="text" name="filter" id="filter" size ="30" value="#[filter]#" maxlength="60" />

-        <input type="submit" value="refresh" />

-      </fieldset>

-    </form>

-    

-    <div class="log">

+        <input type="text" name="lines" id="lines" size ="6" value="#[lines]#" maxlength="9" /><label for="lines"> Lines (max. #[maxlines]#)</label>
+        <input type="checkbox" name="mode" id="mode" value="reversed" #(reverseChecked)#::checked="checked"#(/reverseChecked)# /><label for="mode">reversed order</label>
+        <select name="filterMode" id="filterMode">
+          <option value="regex" #(filterModeRegex)#::selected="selected"#(/filterModeRegex)#>regex</option>
+          <option value="terms" #(filterModeTerms)#::selected="selected"#(/filterModeTerms)#>terms</option>
+        </select>
+        <input type="text" name="filter" id="filter" size ="42" value="#[filter]#" maxlength="160" />
+        <input type="submit" value="refresh" />
+      </fieldset>
+    </form>
+    #(filterError)#::<p class="error">Invalid regular expression filter.</p>#(/filterError)#
+    
+    <div class="log">
     #{log}#<p class="log#(level)#Debug::Info::System::Warning::Severe#(/level)#" style="font-family:Monospace;margin-top:0px;margin-bottom:0px;">#[line]#</p>#{/log}#

 	</div>

     

diff --git a/htroot/api/logreports.json b/htroot/api/logreports.json
new file mode 100644
index 000000000..4b495dada
--- /dev/null
+++ b/htroot/api/logreports.json
@@ -0,0 +1,17 @@
+{
+  #(authorized)#
+  "error": "authorization required",
+  ::
+  #(/authorized)#
+  "reports": [
+    #{reports}#
+    {
+      "filename": "#[filename]#",
+      "title": "#[title]#",
+      "published": "#[published]#",
+      "daily": #(daily)#false::true#(/daily)#,
+      "content": "#[content]#"
+    }#(comma)#::,#(/comma)#
+    #{/reports}#
+  ]
+}
diff --git a/htroot/api/logreports.rss b/htroot/api/logreports.rss
new file mode 100644
index 000000000..8a0d9f2ed
--- /dev/null
+++ b/htroot/api/logreports.rss
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<rss version="2.0">
+<channel>
+<title>#[channel_title]#</title>
+<description>#[channel_description]#</description>
+<pubDate>#[channel_pubDate]#</pubDate>
+#(authorized)#
+<item>
+<title>Authorization required</title>
+<description>Administrative authentication is required to read YaCy log reports.</description>
+<guid isPermaLink="false">authorization-required</guid>
+</item>
+::
+#(/authorized)#
+#{reports}#
+<item>
+<title>#[title-rss]#</title>
+<description>#[description-rss]#</description>
+<pubDate>#[pubDate-rss]#</pubDate>
+<guid isPermaLink="false">#[guid-rss]#</guid>
+</item>
+#{/reports}#
+</channel>
+</rss>
diff --git a/htroot/env/templates/submenuAI.template b/htroot/env/templates/submenuAI.template
index 8265b9ff0..b8fff30a0 100644
--- a/htroot/env/templates/submenuAI.template
+++ b/htroot/env/templates/submenuAI.template
@@ -5,6 +5,7 @@
     <li><a href="LLMSelection_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">LLM Selection</a></li>
     <li><a href="RAGConfig_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">RAG Config</a></li>
     <li><a href="ToolsConfig_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Tools Config</a></li>
+    <li><a href="LogReports_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Log Reports</a></li>
     <li><a href="AIShield_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">AI Shield</a></li>
     <li><a href="yacychat.html" class="MenuItemLink">Chat</a></li>
   </ul>
diff --git a/htroot/env/templates/submenuComputation.template b/htroot/env/templates/submenuComputation.template
index 9b24c0226..99d1a75de 100644
--- a/htroot/env/templates/submenuComputation.template
+++ b/htroot/env/templates/submenuComputation.template
@@ -12,6 +12,7 @@
   <h3>Processes</h3>
   <ul class="SubMenu">
     <li><a href="ViewLog_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Server Log</a></li>
+    <li><a href="LogReports_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Log Reports</a></li>
     <li><a href="Threaddump_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Thread Dump</a></li>
     <li><a href="PerformanceConcurrency_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Concurrent Indexing</a></li>
     <li><a href="PerformanceMemory_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Memory Usage</a></li>