another report fix

2 files changed, 58 insertions, 24 deletions

diff --git a/source/net/yacy/ai/LLM.java b/source/net/yacy/ai/LLM.java
index 6bdeaef38..71ce2b598 100644
--- a/source/net/yacy/ai/LLM.java
+++ b/source/net/yacy/ai/LLM.java
@@ -480,6 +480,7 @@ public class LLM {
                 throw new IOException("Request failed with response code " + responseCode);
             }
             final StringBuilder full = new StringBuilder();
+            String finishReason = "";
             try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8"))) {
                 String line;
                 while ((line = br.readLine()) != null) {
@@ -490,14 +491,29 @@ public class LLM {
                     final JSONObject event = new JSONObject(payload);
                     final JSONArray choices = event.optJSONArray("choices");
                     if (choices == null || choices.length() == 0) continue;
-                    final JSONObject delta = choices.getJSONObject(0).optJSONObject("delta");
+                    final JSONObject choice = choices.getJSONObject(0);
+                    final JSONObject delta = choice.optJSONObject("delta");
                     final String content = delta == null ? "" : delta.optString("content", "");
                     if (!content.isEmpty()) {
                         full.append(content);
                         if (onDelta != null) onDelta.accept(content);
                     }
+                    // the terminal chunk carries the finish_reason; remember the last non-empty one
+                    final String reason = choice.optString("finish_reason", "");
+                    if (!reason.isEmpty()) finishReason = reason;
                 }
             }
+            // A truncated answer is not an abort on our side but a generation limit. Unlike
+            // chat() the streaming path used to drop this signal, so a report that ends
+            // mid-word (prompt fills the context window, leaving no room to generate) was
+            // invisible in the logs; surface it explicitly here.
+            if ("length".equals(finishReason)) {
+                log.warn("chatStream response was truncated by the max_tokens limit (" + max_tokens
+                        + "), model=" + LogRedaction.redact(model)
+                        + ", contentChars=" + full.length()
+                        + ". The prompt likely fills the context window, leaving no room to generate;"
+                        + " reduce the prompt size or raise the model context/max_tokens.");
+            }
             return full.toString();
         } catch (JSONException | URISyntaxException e) {
             throw new IOException(e.getMessage());
diff --git a/source/net/yacy/ai/LogReportService.java b/source/net/yacy/ai/LogReportService.java
index 3e0792223..00141d3d0 100644
--- a/source/net/yacy/ai/LogReportService.java
+++ b/source/net/yacy/ai/LogReportService.java
@@ -352,10 +352,10 @@ public class LogReportService {
             }
             try {
                 final NoiseSummary noiseSummary = classifyNoise(bucket.getValue());
-                final String prompt = hourlyPrompt(bucket.getKey(), bucket.getValue(), noiseSummary);
-                log.info("runId=" + runId + " event=hourly-report phase=classify-noise bucket=" + bucket.getKey() + " inputLines=" + bucket.getValue().size() + " noiseLines=" + noiseSummary.classifiedLines() + " noiseCategories=" + noiseSummary.buckets.size());
                 final int configuredMaxTokens = this.sb.getConfigInt(CONFIG_MAX_TOKENS, model.llm.max_tokens);
                 final int maxTokens = Math.max(1, Math.min(model.llm.max_tokens, configuredMaxTokens));
+                final String prompt = hourlyPrompt(bucket.getKey(), bucket.getValue(), noiseSummary, promptPayloadCharBudget(model, maxTokens));
+                log.info("runId=" + runId + " event=hourly-report phase=classify-noise bucket=" + bucket.getKey() + " inputLines=" + bucket.getValue().size() + " noiseLines=" + noiseSummary.classifiedLines() + " noiseCategories=" + noiseSummary.buckets.size());
                 log.info("runId=" + runId + " event=hourly-report phase=model-call bucket=" + bucket.getKey() + " model=" + LogRedaction.redact(model.model) + " backend=" + LogRedaction.redact(model.llm.hoststub) + " inputLines=" + bucket.getValue().size() + " promptChars=" + prompt.length() + " maxTokens=" + maxTokens);
                 final long modelStart = System.currentTimeMillis();
                 final String report = model.llm.chatStream(model.model, SYSTEM_PROMPT, prompt, maxTokens, null);
@@ -412,10 +412,10 @@ public class LogReportService {
 
         final File reportFile = new File(reportDirectory, hourlyReportFilename(currentHour));
         final NoiseSummary noiseSummary = classifyNoise(bucketLines);
-        final String prompt = hourlyPrompt(currentHour, bucketLines, noiseSummary);
-        log.info("runId=" + runId + " event=current-hour-report phase=classify-noise bucket=" + currentHour + " inputLines=" + bucketLines.size() + " noiseLines=" + noiseSummary.classifiedLines() + " noiseCategories=" + noiseSummary.buckets.size());
         final int configuredMaxTokens = this.sb.getConfigInt(CONFIG_MAX_TOKENS, model.llm.max_tokens);
         final int maxTokens = Math.max(1, Math.min(model.llm.max_tokens, configuredMaxTokens));
+        final String prompt = hourlyPrompt(currentHour, bucketLines, noiseSummary, promptPayloadCharBudget(model, maxTokens));
+        log.info("runId=" + runId + " event=current-hour-report phase=classify-noise bucket=" + currentHour + " inputLines=" + bucketLines.size() + " noiseLines=" + noiseSummary.classifiedLines() + " noiseCategories=" + noiseSummary.buckets.size());
         log.info("runId=" + runId + " event=current-hour-report phase=model-call bucket=" + currentHour + " model=" + LogRedaction.redact(model.model) + " backend=" + LogRedaction.redact(model.llm.hoststub) + " inputLines=" + bucketLines.size() + " promptChars=" + prompt.length() + " maxTokens=" + maxTokens);
         final long modelStart = System.currentTimeMillis();
         final String report = model.llm.chatStream(model.model, SYSTEM_PROMPT, prompt, maxTokens, onDelta);
@@ -476,9 +476,9 @@ public class LogReportService {
                 continue;
             }
             try {
-                final String prompt = dailyPrompt(day.getKey(), day.getValue());
                 final int configuredMaxTokens = this.sb.getConfigInt(CONFIG_MAX_TOKENS, model.llm.max_tokens);
                 final int maxTokens = Math.max(1, Math.min(model.llm.max_tokens, configuredMaxTokens));
+                final String prompt = dailyPrompt(day.getKey(), day.getValue(), promptPayloadCharBudget(model, maxTokens));
                 log.info("runId=" + runId + " event=daily-report phase=model-call day=" + day.getKey() + " model=" + LogRedaction.redact(model.model) + " backend=" + LogRedaction.redact(model.llm.hoststub) + " sourceReports=" + day.getValue().size() + " promptChars=" + prompt.length() + " maxTokens=" + maxTokens);
                 final long modelStart = System.currentTimeMillis();
                 final String report = model.llm.chatStream(model.model, SYSTEM_PROMPT, prompt, maxTokens, null);
@@ -585,11 +585,7 @@ public class LogReportService {
         return System.currentTimeMillis() - start;
     }
 
-    private static String hourlyPrompt(final LocalDateTime bucket, final List<String> lines) {
-        return hourlyPrompt(bucket, lines, classifyNoise(lines));
-    }
-
-    private static String hourlyPrompt(final LocalDateTime bucket, final List<String> lines, final NoiseSummary noiseSummary) {
+    private static String hourlyPrompt(final LocalDateTime bucket, final List<String> lines, final NoiseSummary noiseSummary, final int maxPayloadChars) {
         final StringBuilder prompt = new StringBuilder(1024 + lines.size() * 120);
         prompt.append("Create a YaCy self-enhancement log report for hour ")
                 .append(bucket)
@@ -624,28 +620,50 @@ public class LogReportService {
         if (omittedNoise > 0) {
             prompt.append("(").append(omittedNoise).append(" noise lines omitted)\n");
         }
-        appendWithPromptBudget(prompt, linesText);
+        appendWithPromptBudget(prompt, linesText, maxPayloadChars);
         return prompt.toString();
     }
 
+    /** rough characters-per-token ratio used to budget the prompt against the model's token window */
+    private static final int CHARS_PER_TOKEN = 4;
+    /** never shrink the prompt payload below this many tokens, even for tiny model windows */
+    private static final int MIN_PROMPT_PAYLOAD_TOKENS = 512;
+
     /**
-     * Upper bound for the variable part of a report prompt. Local models process the
-     * prompt token by token before they generate anything, so an unbounded prompt
-     * makes the report generation arbitrarily slow. 64k chars are roughly 16k tokens.
+     * Character budget for the variable part of a report prompt (log lines or hourly
+     * reports), derived from the model's token window. A local model must ingest the
+     * entire prompt before it emits a single output token, and the prompt and the
+     * generated report share one context window: prompt + output has to fit into
+     * model.llm.max_tokens. The payload is therefore sized to (window - output reserve)
+     * tokens, converted to characters.
+     * <p>
+     * A fixed budget (previously 64k chars ≈ 16k tokens) overflows small windows: with a
+     * default 4k-token model the ~16k-token hourly prompt filled the whole window, left no
+     * room to generate, and the report stopped after one or two tokens. When the output
+     * cap already fills the window (the common case where max_tokens equals the context
+     * length) the payload falls back to a quarter of the window so a report is still
+     * produced; if the output then hits its cap it is truncated and logged
+     * (finish_reason=length) instead of the prompt silently overflowing.
      */
-    private static final int MAX_PROMPT_PAYLOAD_CHARS = 65536;
+    private static int promptPayloadCharBudget(final LLMModel model, final int maxTokens) {
+        final int contextTokens = Math.max(1, model.llm.max_tokens);
+        final int promptTokens = Math.max(
+                Math.max(MIN_PROMPT_PAYLOAD_TOKENS, contextTokens / 4),
+                contextTokens - maxTokens);
+        return promptTokens * CHARS_PER_TOKEN;
+    }
 
     /**
-     * Append the payload to the prompt, truncated to MAX_PROMPT_PAYLOAD_CHARS.
-     * When truncating, the most recent part (the tail) is kept because the newest
-     * log lines are the most relevant ones for the report.
+     * Append the payload to the prompt, truncated to maxPayloadChars. When truncating,
+     * the most recent part (the tail) is kept because the newest log lines are the most
+     * relevant ones for the report.
      */
-    private static void appendWithPromptBudget(final StringBuilder prompt, final StringBuilder payload) {
-        if (payload.length() <= MAX_PROMPT_PAYLOAD_CHARS) {
+    private static void appendWithPromptBudget(final StringBuilder prompt, final StringBuilder payload, final int maxPayloadChars) {
+        if (payload.length() <= maxPayloadChars) {
             prompt.append(payload);
             return;
         }
-        int cut = payload.length() - MAX_PROMPT_PAYLOAD_CHARS;
+        int cut = payload.length() - maxPayloadChars;
         final int lineStart = payload.indexOf("\n", cut);
         if (lineStart >= 0) cut = lineStart + 1; // do not start with a partial line
         prompt.append("(older content truncated to fit the prompt budget)\n")
@@ -798,7 +816,7 @@ public class LogReportService {
         return new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8);
     }
 
-    private static String dailyPrompt(final LocalDate day, final List<File> hourlyReports) throws IOException {
+    private static String dailyPrompt(final LocalDate day, final List<File> hourlyReports, final int maxPayloadChars) throws IOException {
         final StringBuilder prompt = new StringBuilder(4096);
         prompt.append("Create one consolidated YaCy self-enhancement report for ")
                 .append(day)
@@ -816,7 +834,7 @@ public class LogReportService {
             reportsText.append("\n\n## ").append(hourlyReport.getName()).append("\n\n")
                     .append(new String(Files.readAllBytes(hourlyReport.toPath()), StandardCharsets.UTF_8));
         }
-        appendWithPromptBudget(prompt, reportsText);
+        appendWithPromptBudget(prompt, reportsText, maxPayloadChars);
         return prompt.toString();
     }