summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--defaults/yacy.init8
-rw-r--r--htroot/LLMSelection_p.html195
-rw-r--r--source/net/yacy/ai/LLM.java70
-rw-r--r--source/net/yacy/ai/LogReportService.java38
-rw-r--r--source/net/yacy/htroot/LLMSelection_p.java58
5 files changed, 282 insertions, 87 deletions
diff --git a/defaults/yacy.init b/defaults/yacy.init
index 03d79eb3e..a086c5326 100644
--- a/defaults/yacy.init
+++ b/defaults/yacy.init
@@ -1434,6 +1434,10 @@ decoration.simpleheadernavbar = navbar-default
# ai settings
ai.production_models = []
+# per-service context window (num_ctx), JSON object mapping a service hoststub to its
+# token window. Advisory: must match the window the backend actually serves (e.g. Ollama
+# OLLAMA_CONTEXT_LENGTH). Used to budget prompts; unset services default to 4096 tokens.
+ai.service_num_ctx = {}
ai.system-prompt = You are a smart and helpful chatbot. If possible, use friendly emojies.
ai.llm-system-prefix = \n\nYou may receive additional expert knowledge in the user prompt after a 'Additional Information' headline to enhance your knowledge. Use it only if applicable.
ai.llm-user-prefix = \n\nAdditional Information:\n\nbelow you find a collection of texts that might be useful to generate a response. Do not discuss these documents, just use them to answer the question above.\n\n
@@ -1443,10 +1447,6 @@ ai.logreport.dir = DATA/REPORTS/log
ai.logreport.initial_delay_minutes = 5
ai.logreport.period_minutes = 60
ai.logreport.max_bucket_lines = 100000
-# Output-token cap for a generated report. This is the report length only, NOT the
-# context window: the prompt budget is (model max_tokens - this value), so a smaller
-# value here leaves more of the model's context window for the log lines in the prompt.
-ai.logreport.max_tokens = 16384
ai.logreport.daily_compression.enabled = true
ai.logreport.feed.max_entries = 100
ai.shield.allow-nonlocalhost = false
diff --git a/htroot/LLMSelection_p.html b/htroot/LLMSelection_p.html
index 30e3f3155..f125c10b6 100644
--- a/htroot/LLMSelection_p.html
+++ b/htroot/LLMSelection_p.html
@@ -5,7 +5,7 @@
<title>YaCy '#[clientname]#': LLM Selection</title>
#%env/templates/metas.template%#
</head>
- <body id="IndexControl" data-llm-service="#[llm_service]#" data-llm-hoststub="#[llm_hoststub]#" data-llm-apikey="#[llm_apikey]#" data-model-capabilities="#[model_capabilities]#">
+ <body id="IndexControl" data-llm-service="#[llm_service]#" data-llm-hoststub="#[llm_hoststub]#" data-llm-apikey="#[llm_apikey]#" data-model-capabilities="#[model_capabilities]#" data-service-num-ctx="#[service_num_ctx_json]#">
#%env/templates/header.template%#
#%env/templates/submenuAI.template%#
<script>
@@ -580,6 +580,13 @@
}
}
+ // Service dropdown handler: reset the hoststub to the service default, then
+ // reflect the resulting endpoint in the Services table.
+ function serviceChanged() {
+ setHoststub();
+ syncSelectedServiceRow();
+ }
+
function applyPresetInference() {
const serviceSelect = document.getElementById("service");
const hoststubInput = document.getElementById("hoststub");
@@ -882,8 +889,10 @@
upsertProductionModel(modelName);
}
- const MAX_TOKEN_OPTIONS = ["4096", "8192", "16384", "32768", "65536", "131072", "262440"];
- const DEFAULT_MAX_TOKENS = "16384";
+ // max_tokens (num_predict) options start at 2048 = half of the default num_ctx (4096),
+ // so a freshly deployed model reserves output room that fits inside the default window.
+ const MAX_TOKEN_OPTIONS = ["2048", "4096", "8192", "16384", "32768", "65536", "131072", "262440"];
+ const DEFAULT_MAX_TOKENS = "2048";
function normalizeMaxTokenValue(value) {
const text = (value == null ? "" : String(value)).trim();
@@ -939,13 +948,13 @@
const serviceField = document.getElementById("service");
const hoststubField = document.getElementById("hoststub");
const apikeyField = document.getElementById("apikey");
- const maxTokenField = document.getElementById("maxtoken");
const service = serviceField ? serviceField.value : "";
const hoststub = hoststubField ? hoststubField.value.trim() : "";
const apikey = apikeyField ? apikeyField.value.trim() : "";
- const maxToken = maxTokenField ? maxTokenField.value : "";
const persistedCaps = getPersistedCapabilitiesForModel(service, hoststub, modelName);
+ // a newly deployed endpoint should also show up in the Services table
+ ensureServiceRow(service, hoststub);
let targetRow = Array.from(tbody.querySelectorAll("tr")).find(row => {
const modelCell = row.cells && row.cells[PRODUCTION_MODEL_MODEL_COLUMN_INDEX];
@@ -968,10 +977,10 @@
}
const cells = targetRow.cells;
- // for a new row take the max_tokens from the top selection; for an
- // existing row keep the per-row value the user may have edited
+ // a new row starts at the default max_tokens; an existing row keeps the
+ // per-row value the user may have edited in the Production Models Matrix
const effectiveMaxToken = isNewRow
- ? maxToken
+ ? DEFAULT_MAX_TOKENS
: readMaxTokenValue(cells[PRODUCTION_MODEL_MAX_TOKENS_COLUMN_INDEX]);
const values = [service, modelName, hoststub, apikey, effectiveMaxToken];
values.forEach((value, index) => {
@@ -1057,6 +1066,94 @@
});
}
+ const SERVICE_DEFAULT_NUM_CTX = 4096;
+ // fixed num_ctx choices (4k..256k); max_tokens additionally offers 2048 below this range
+ const NUM_CTX_OPTIONS = ["4096", "8192", "16384", "32768", "65536", "131072", "262440"];
+ let storedServiceNumCtx = {};
+
+ function normalizeHoststubJs(hoststub) {
+ return (hoststub == null ? "" : String(hoststub)).trim().replace(/\/+$/, "");
+ }
+
+ // Build the fixed-option num_ctx <select> for a Services-table cell. Auto-saves on
+ // change, like the other inputs on this page.
+ function createNumCtxSelect(hoststub, value) {
+ const normalized = String(value || SERVICE_DEFAULT_NUM_CTX);
+ const select = document.createElement("select");
+ select.className = "form-control service-num-ctx";
+ select.dataset.hoststub = hoststub;
+ select.style.width = "140px";
+ const optionValues = NUM_CTX_OPTIONS.includes(normalized) ? NUM_CTX_OPTIONS : [normalized, ...NUM_CTX_OPTIONS];
+ optionValues.forEach(v => {
+ const option = document.createElement("option");
+ option.value = v;
+ option.textContent = v;
+ select.appendChild(option);
+ });
+ select.value = normalized;
+ select.addEventListener("change", () => saveServiceNumCtx());
+ return select;
+ }
+
+ // Populate the server-rendered num_ctx cells (.num-ctx-cell) with their selects.
+ function initServiceRows() {
+ document.querySelectorAll("#servicesTable tbody .num-ctx-cell").forEach(cell => {
+ if (cell.querySelector("select")) return;
+ cell.appendChild(createNumCtxSelect(cell.dataset.hoststub || "", cell.dataset.numCtx));
+ });
+ }
+
+ // Ensure the Services table has a row for the given endpoint. Server-rendered rows
+ // exist for deployed endpoints; this adds a row for a service that is selected in
+ // the Service Selection box but not yet deployed, prefilled with its stored window
+ // or the default. Existing rows (and any edits) are left untouched.
+ function ensureServiceRow(service, hoststub) {
+ const tbody = document.querySelector("#servicesTable tbody");
+ const key = normalizeHoststubJs(hoststub);
+ if (!tbody || !key) return;
+ const present = Array.from(tbody.querySelectorAll(".service-num-ctx"))
+ .some(sel => normalizeHoststubJs(sel.dataset.hoststub) === key);
+ if (present) return;
+ const value = storedServiceNumCtx[key] || SERVICE_DEFAULT_NUM_CTX;
+ const tr = document.createElement("tr");
+ const tdService = document.createElement("td");
+ tdService.textContent = service || "";
+ const tdHost = document.createElement("td");
+ tdHost.textContent = key;
+ const tdCtx = document.createElement("td");
+ tdCtx.className = "num-ctx-cell";
+ tdCtx.dataset.hoststub = key;
+ tdCtx.dataset.numCtx = value;
+ tdCtx.appendChild(createNumCtxSelect(key, value));
+ tr.appendChild(tdService);
+ tr.appendChild(tdHost);
+ tr.appendChild(tdCtx);
+ tbody.appendChild(tr);
+ }
+
+ // Reflect the currently selected Service Selection endpoint in the Services table.
+ function syncSelectedServiceRow() {
+ const serviceSelect = document.getElementById("service");
+ const hoststubInput = document.getElementById("hoststub");
+ if (!serviceSelect || !hoststubInput) return;
+ ensureServiceRow(serviceSelect.value, hoststubInput.value);
+ }
+
+ function saveServiceNumCtx() {
+ const service_num_ctx = {};
+ document.querySelectorAll("#servicesTable .service-num-ctx").forEach(sel => {
+ const hoststub = (sel.dataset.hoststub || "").trim();
+ const value = parseInt(sel.value, 10);
+ if (hoststub && Number.isFinite(value) && value > 0) service_num_ctx[hoststub] = value;
+ });
+ fetch(PRODUCTION_MODEL_SUBMIT_URL, {
+ method: "POST", headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ service_num_ctx })
+ }).catch(err => {
+ console.error("Failed to persist service context windows", err);
+ });
+ }
+
function ensureProductionRowUsageCells(row, defaultChecked) {
if (!row) return;
// ensure existence of checkboxes
@@ -1965,28 +2062,18 @@
return downloadBtn;
}
- // Mirror the most recently deployed row's max_tokens into the top select so
- // the field reflects the current configuration instead of always resetting
- // to the hardcoded default. The per-row selects remain the source of truth.
- function prefillTopMaxTokenFromTable() {
- const select = document.getElementById("maxtoken");
- const tbody = getProductionTableBody();
- if (!select || !tbody) return;
- const rows = tbody.querySelectorAll("tr");
- if (!rows.length) return;
- const lastRow = rows[rows.length - 1];
- const value = readMaxTokenValue(lastRow.cells[PRODUCTION_MODEL_MAX_TOKENS_COLUMN_INDEX]);
- if (Array.from(select.options).some(o => o.value === value)) {
- select.value = value;
- }
- }
-
document.addEventListener("DOMContentLoaded", () => {
try {
+ try {
+ storedServiceNumCtx = JSON.parse(document.body.dataset.serviceNumCtx || "{}") || {};
+ } catch (e) {
+ storedServiceNumCtx = {};
+ }
persistedModelCapabilities = readPersistedModelCapabilities();
normalizeProductionModelRows();
- prefillTopMaxTokenFromTable();
+ initServiceRows();
applyPresetInference();
+ syncSelectedServiceRow();
resumePendingPulls();
// auto-show available models if a preset inference exists
const body = document.body;
@@ -2031,7 +2118,7 @@
<dl>
<dt class="TableCellDark">service</dt>
<dd>
- <select name="service" id="service" class="form-control" onchange="setHoststub()">
+ <select name="service" id="service" class="form-control" onchange="serviceChanged()">
<option value="OLLAMA" selected="selected">Ollama</option>
<option value="LMSTUDIO">LMStudio</option>
<option value="OPENAI">OpenAI</option>
@@ -2040,34 +2127,16 @@
</dd>
<dt class="TableCellDark">hoststub</dt>
- <dd><input type="text" name="hoststub" id="hoststub" value="http://localhost:11434" size="30" maxlength="60" class="form-control"/>&nbsp; you can probably leave this to the default value
+ <dd><input type="text" name="hoststub" id="hoststub" value="http://localhost:11434" size="30" maxlength="60" class="form-control" onchange="syncSelectedServiceRow()"/>&nbsp; you can probably leave this to the default value
</dd>
<dt class="TableCellDark">api_key</dt>
<dd><input type="text" name="apikey" id="apikey" value="" disabled=true size="30" maxlength="120" class="form-control"/>&nbsp; (not required for Ollama or LMStudio)
- </dd>
-
- <dt class="TableCellDark">max_tokens</dt>
- <dd>
- <select id="maxtoken" name="maxtoken" class="form-control">
- <option>4096</option>
- <option>8192</option>
- <option selected="selected">16384</option>
- <option>32768</option>
- <option>65536</option>
- <option>131072</option>
- <option>262440</option>
- </select>
- <span class="info"><img src="env/grafics/i16.gif" width="16" height="16" alt="info"/><span>
- This is the default max_tokens applied to a model when you deploy it; you can change it per model afterwards in the Production Models Matrix below.
- </span></span>
<br/>
<small>
- <b>max_tokens</b> caps the number of <i>generated</i> tokens (sent as the OpenAI <code>max_tokens</code>, i.e. Ollama <code>num_predict</code>).
- It does <b>not</b> enlarge the model's context window: Ollama defaults <code>num_ctx</code> to 4096 regardless of this value,
- and its OpenAI-compatible endpoint (<code>/v1/chat/completions</code>) cannot set <code>num_ctx</code> per request.
- To actually use a large context in Ollama, raise the context length once via the environment variable
- <code>OLLAMA_CONTEXT_LENGTH</code>, a Modelfile <code>PARAMETER num_ctx &lt;n&gt;</code>, or the Context Length slider in the Ollama app settings.
+ The selected service's context window (<code>num_ctx</code>) is shown and editable in the
+ <a href="#services">Services</a> table below. A model's generated-token cap (<code>max_tokens</code> =
+ Ollama <code>num_predict</code>) is set per model in the Production Models Matrix.
</small>
</dd>
@@ -2078,6 +2147,36 @@
</fieldset>
</form>
+ <fieldset id="servicesContainer" style="display: block;"><a name="services"></a><legend>Services</legend>
+ <p>
+ <b>num_ctx</b> is the context window (in tokens) of the inference service &mdash; a per-service
+ value, shared by all models on that endpoint. It is the total budget for prompt <i>plus</i>
+ generated output; YaCy uses it to size prompts so they leave room to generate. The row for the
+ service selected above appears here automatically with its stored (or default) window.
+ This value is <b>advisory</b>: set it to match the window your backend actually serves
+ (Ollama: <code>OLLAMA_CONTEXT_LENGTH</code>, a Modelfile <code>PARAMETER num_ctx</code>, or the
+ Context Length setting). YaCy does not enforce it on the backend.
+ </p>
+ <table class="table table-striped" id="servicesTable">
+ <thead class="thead-dark">
+ <tr>
+ <td>service</td>
+ <td>hoststub</td>
+ <td>num_ctx</td>
+ </tr>
+ </thead>
+ <tbody>
+ #{services}#
+ <tr>
+ <td>#[service]#</td>
+ <td>#[hoststub]#</td>
+ <td class="num-ctx-cell" data-hoststub="#[hoststub]#" data-num-ctx="#[num_ctx]#"></td>
+ </tr>
+ #{/services}#
+ </tbody>
+ </table>
+ </fieldset>
+
<fieldset id="loadModelContainer" style="display:none"></fieldset>
<fieldset id="downloadActivityContainer" style="display:none">
<legend>Model Downloads</legend>
diff --git a/source/net/yacy/ai/LLM.java b/source/net/yacy/ai/LLM.java
index 71ce2b598..3e6d74783 100644
--- a/source/net/yacy/ai/LLM.java
+++ b/source/net/yacy/ai/LLM.java
@@ -47,6 +47,8 @@ public class LLM {
private static final ConcurrentLog log = new ConcurrentLog("LLM");
private static final String MODEL_CAPABILITIES_CONFIG = "ai.model_capabilities";
+ /** config key: JSON object mapping a service hoststub to its context window (num_ctx) */
+ public static final String SERVICE_NUM_CTX_CONFIG = "ai.service_num_ctx";
private static String[] STOPTOKENS = new String[]{"[/INST]", "<|im_end|>", "<|end_of_turn|>", "<|eot_id|>", "<|end_header_id|>", "<EOS_TOKEN>", "</s>", "<|end|>"};
public static enum LLMType {
@@ -85,15 +87,31 @@ public class LLM {
}
}
+ /** Ollama's out-of-the-box context window; used when a service has no num_ctx configured. */
+ public static final int DEFAULT_NUM_CTX = 4096;
+ /**
+ * Sole fallback for a model's max_tokens when a config row is missing the field
+ * (legacy/malformed rows only). The Production Models Matrix is the place where
+ * max_tokens is defined; this must match the matrix UI default (DEFAULT_MAX_TOKENS
+ * in LLMSelection_p.html) so no divergent default exists.
+ */
+ public static final int DEFAULT_MAX_TOKENS = 2048;
+
public final String hoststub;
public final String api_key;
- public final int max_tokens; // the max_tokens as configured by the endpoint for all models
+ public final int max_tokens; // output-token cap (OpenAI max_tokens = Ollama num_predict)
+ public final int num_ctx; // context window of the inference service (per-service, advisory)
public final LLMType type;
-
+
public LLM(final String hoststub, final String api_key, final int max_tokens, final LLMType type) {
+ this(hoststub, api_key, max_tokens, DEFAULT_NUM_CTX, type);
+ }
+
+ public LLM(final String hoststub, final String api_key, final int max_tokens, final int num_ctx, final LLMType type) {
this.hoststub = hoststub.endsWith("/") ? hoststub.substring(0, hoststub.length() - 1) : hoststub;
this.api_key = api_key == null ? "" : api_key;
- this.max_tokens = max_tokens <= 0 ? 4096 : max_tokens;
+ this.max_tokens = max_tokens <= 0 ? DEFAULT_MAX_TOKENS : max_tokens;
+ this.num_ctx = num_ctx <= 0 ? DEFAULT_NUM_CTX : num_ctx;
this.type = type;
}
@@ -133,7 +151,7 @@ public class LLM {
// found one that shall be used for this use case
final String hoststub = row.optString("hoststub", "");
final String api_key = row.optString("api_key", "");
- final int max_tokens = Integer.parseInt(row.optString("max_tokens", "4096"));
+ final int max_tokens = Integer.parseInt(row.optString("max_tokens", String.valueOf(DEFAULT_MAX_TOKENS)));
final String model = row.optString("model", "");
final LLMType type = LLMType.valueOf(row.optString("service", "OLLAMA"));
boolean tooling = row.optBoolean("tooling", false);
@@ -145,10 +163,11 @@ public class LLM {
if (!thinking) thinking = "supported".equals(capabilityEntry.optString("thinking", ""));
}
}
- LLM llm = new LLM(hoststub, api_key, max_tokens, type);
+ final int num_ctx = serviceNumCtx(sb, hoststub);
+ LLM llm = new LLM(hoststub, api_key, max_tokens, num_ctx, type);
LLMModel llmmodel = new LLMModel(llm, model, tooling, thinking);
if (logRouting) {
- log.info(routePrefix(runId, caller) + "event=model-routing phase=select usage=" + llmUsage + " row=" + i + " service=" + type.name() + " model=" + LogRedaction.redact(model) + " backend=" + LogRedaction.redact(llm.hoststub) + " maxTokens=" + llm.max_tokens + " tooling=" + tooling + " thinking=" + thinking + " productionRows=" + production_models.length() + " durationMs=" + elapsed(start));
+ log.info(routePrefix(runId, caller) + "event=model-routing phase=select usage=" + llmUsage + " row=" + i + " service=" + type.name() + " model=" + LogRedaction.redact(model) + " backend=" + LogRedaction.redact(llm.hoststub) + " maxTokens=" + llm.max_tokens + " numCtx=" + llm.num_ctx + " tooling=" + tooling + " thinking=" + thinking + " productionRows=" + production_models.length() + " durationMs=" + elapsed(start));
}
return llmmodel;
}
@@ -187,6 +206,31 @@ public class LLM {
return normalizedType + "|" + normalizedHoststub + "|" + normalizedModel;
}
+ /** Normalize a hoststub for use as a service key: trim and drop trailing slashes. */
+ public static String normalizeHoststub(final String hoststub) {
+ if (hoststub == null) return "";
+ return hoststub.trim().replaceAll("/+$", "");
+ }
+
+ /**
+ * Context window (num_ctx) configured for the inference service at the given hoststub.
+ * num_ctx is a per-service setting (a self-hosted server exposes one context length for
+ * all its models via OLLAMA_CONTEXT_LENGTH), stored under SERVICE_NUM_CTX_CONFIG keyed by
+ * hoststub. The value is advisory: YaCy uses it to budget the prompt against the window,
+ * it does not enforce it on the backend. Falls back to DEFAULT_NUM_CTX when unset.
+ */
+ public static int serviceNumCtx(final Switchboard sb, final String hoststub) {
+ if (sb == null) return DEFAULT_NUM_CTX;
+ final String json = sb.getConfig(SERVICE_NUM_CTX_CONFIG, "{}");
+ try {
+ final JSONObject map = new JSONObject(new JSONTokener(json));
+ final int value = map.optInt(normalizeHoststub(hoststub), 0);
+ return value <= 0 ? DEFAULT_NUM_CTX : value;
+ } catch (final JSONException e) {
+ return DEFAULT_NUM_CTX;
+ }
+ }
+
private static JSONObject readModelCapabilities() {
final Switchboard sb = Switchboard.getSwitchboard();
if (sb == null) return new JSONObject(true);
@@ -370,12 +414,12 @@ public class LLM {
data.put("model", model);
data.put("temperature", 0.1);
data.put("max_tokens", max_tokens);
- // Best-effort hint for Ollama's context window (num_ctx). Ollama's
- // OpenAI-compatible endpoint does not read this today and pure-OpenAI
- // backends ignore unknown fields, so it is a harmless forward-looking
- // hedge; the reliable way to raise the context window remains
+ // Best-effort hint for Ollama's context window (num_ctx), taken from the
+ // per-service configuration. Ollama's OpenAI-compatible endpoint does not read
+ // this today and pure-OpenAI backends ignore unknown fields, so it is a harmless
+ // forward-looking hedge; the reliable way to raise the context window remains
// OLLAMA_CONTEXT_LENGTH or a Modelfile PARAMETER num_ctx.
- data.put("num_ctx", max_tokens);
+ data.put("num_ctx", this.num_ctx);
data.put("messages", context);
data.put("stop", new JSONArray(STOPTOKENS));
data.put("stream", false);
@@ -454,8 +498,8 @@ public class LLM {
data.put("model", model);
data.put("temperature", 0.1);
data.put("max_tokens", max_tokens);
- // best-effort num_ctx hint, see chat(); harmless to non-Ollama backends
- data.put("num_ctx", max_tokens);
+ // best-effort num_ctx hint from the per-service config, see chat(); harmless to non-Ollama backends
+ data.put("num_ctx", this.num_ctx);
data.put("messages", context);
data.put("stop", new JSONArray(STOPTOKENS));
data.put("stream", true);
diff --git a/source/net/yacy/ai/LogReportService.java b/source/net/yacy/ai/LogReportService.java
index 00141d3d0..af1aaab9c 100644
--- a/source/net/yacy/ai/LogReportService.java
+++ b/source/net/yacy/ai/LogReportService.java
@@ -61,7 +61,6 @@ public class LogReportService {
public static final String CONFIG_MAX_BUCKET_LINES = "ai.logreport.max_bucket_lines";
public static final String CONFIG_INITIAL_DELAY_MINUTES = "ai.logreport.initial_delay_minutes";
public static final String CONFIG_PERIOD_MINUTES = "ai.logreport.period_minutes";
- public static final String CONFIG_MAX_TOKENS = "ai.logreport.max_tokens";
public static final String CONFIG_DAILY_COMPRESSION_ENABLED = "ai.logreport.daily_compression.enabled";
public static final String CONFIG_FEED_MAX_ENTRIES = "ai.logreport.feed.max_entries";
public static final String DEFAULT_REPORT_DIR = "DATA/REPORTS/log";
@@ -352,8 +351,8 @@ public class LogReportService {
}
try {
final NoiseSummary noiseSummary = classifyNoise(bucket.getValue());
- final int configuredMaxTokens = this.sb.getConfigInt(CONFIG_MAX_TOKENS, model.llm.max_tokens);
- final int maxTokens = Math.max(1, Math.min(model.llm.max_tokens, configuredMaxTokens));
+ // output cap follows the model's configured max_tokens (Production Models Matrix)
+ final int maxTokens = Math.max(1, model.llm.max_tokens);
final String prompt = hourlyPrompt(bucket.getKey(), bucket.getValue(), noiseSummary, promptPayloadCharBudget(model, maxTokens));
log.info("runId=" + runId + " event=hourly-report phase=classify-noise bucket=" + bucket.getKey() + " inputLines=" + bucket.getValue().size() + " noiseLines=" + noiseSummary.classifiedLines() + " noiseCategories=" + noiseSummary.buckets.size());
log.info("runId=" + runId + " event=hourly-report phase=model-call bucket=" + bucket.getKey() + " model=" + LogRedaction.redact(model.model) + " backend=" + LogRedaction.redact(model.llm.hoststub) + " inputLines=" + bucket.getValue().size() + " promptChars=" + prompt.length() + " maxTokens=" + maxTokens);
@@ -412,8 +411,8 @@ public class LogReportService {
final File reportFile = new File(reportDirectory, hourlyReportFilename(currentHour));
final NoiseSummary noiseSummary = classifyNoise(bucketLines);
- final int configuredMaxTokens = this.sb.getConfigInt(CONFIG_MAX_TOKENS, model.llm.max_tokens);
- final int maxTokens = Math.max(1, Math.min(model.llm.max_tokens, configuredMaxTokens));
+ // output cap follows the model's configured max_tokens (Production Models Matrix)
+ final int maxTokens = Math.max(1, model.llm.max_tokens);
final String prompt = hourlyPrompt(currentHour, bucketLines, noiseSummary, promptPayloadCharBudget(model, maxTokens));
log.info("runId=" + runId + " event=current-hour-report phase=classify-noise bucket=" + currentHour + " inputLines=" + bucketLines.size() + " noiseLines=" + noiseSummary.classifiedLines() + " noiseCategories=" + noiseSummary.buckets.size());
log.info("runId=" + runId + " event=current-hour-report phase=model-call bucket=" + currentHour + " model=" + LogRedaction.redact(model.model) + " backend=" + LogRedaction.redact(model.llm.hoststub) + " inputLines=" + bucketLines.size() + " promptChars=" + prompt.length() + " maxTokens=" + maxTokens);
@@ -476,8 +475,8 @@ public class LogReportService {
continue;
}
try {
- final int configuredMaxTokens = this.sb.getConfigInt(CONFIG_MAX_TOKENS, model.llm.max_tokens);
- final int maxTokens = Math.max(1, Math.min(model.llm.max_tokens, configuredMaxTokens));
+ // output cap follows the model's configured max_tokens (Production Models Matrix)
+ final int maxTokens = Math.max(1, model.llm.max_tokens);
final String prompt = dailyPrompt(day.getKey(), day.getValue(), promptPayloadCharBudget(model, maxTokens));
log.info("runId=" + runId + " event=daily-report phase=model-call day=" + day.getKey() + " model=" + LogRedaction.redact(model.model) + " backend=" + LogRedaction.redact(model.llm.hoststub) + " sourceReports=" + day.getValue().size() + " promptChars=" + prompt.length() + " maxTokens=" + maxTokens);
final long modelStart = System.currentTimeMillis();
@@ -631,22 +630,23 @@ public class LogReportService {
/**
* Character budget for the variable part of a report prompt (log lines or hourly
- * reports), derived from the model's token window. A local model must ingest the
- * entire prompt before it emits a single output token, and the prompt and the
- * generated report share one context window: prompt + output has to fit into
- * model.llm.max_tokens. The payload is therefore sized to (window - output reserve)
- * tokens, converted to characters.
+ * reports), derived from the service's context window (num_ctx). A local model must
+ * ingest the entire prompt before it emits a single output token, and the prompt and
+ * the generated report share one context window: prompt + output has to fit into
+ * num_ctx. The payload is therefore sized to (num_ctx - output reserve) tokens,
+ * converted to characters.
* <p>
* A fixed budget (previously 64k chars ≈ 16k tokens) overflows small windows: with a
- * default 4k-token model the ~16k-token hourly prompt filled the whole window, left no
- * room to generate, and the report stopped after one or two tokens. When the output
- * cap already fills the window (the common case where max_tokens equals the context
- * length) the payload falls back to a quarter of the window so a report is still
- * produced; if the output then hits its cap it is truncated and logged
- * (finish_reason=length) instead of the prompt silently overflowing.
+ * default 4k-token window the ~16k-token hourly prompt filled the whole window, left no
+ * room to generate, and the report stopped after one or two tokens. When the output cap
+ * already fills the window the payload falls back to a quarter of the window so a report
+ * is still produced; if the output then hits its cap it is truncated and logged
+ * (finish_reason=length) instead of the prompt silently overflowing. Raise the service's
+ * num_ctx (on /LLMSelection_p.html, matching the backend's OLLAMA_CONTEXT_LENGTH) to give
+ * the prompt more room.
*/
private static int promptPayloadCharBudget(final LLMModel model, final int maxTokens) {
- final int contextTokens = Math.max(1, model.llm.max_tokens);
+ final int contextTokens = Math.max(1, model.llm.num_ctx);
final int promptTokens = Math.max(
Math.max(MIN_PROMPT_PAYLOAD_TOKENS, contextTokens / 4),
contextTokens - maxTokens);
diff --git a/source/net/yacy/htroot/LLMSelection_p.java b/source/net/yacy/htroot/LLMSelection_p.java
index 960a3e985..405bcf773 100644
--- a/source/net/yacy/htroot/LLMSelection_p.java
+++ b/source/net/yacy/htroot/LLMSelection_p.java
@@ -87,7 +87,7 @@ public class LLMSelection_p {
normalized.put("model", row.optString("model", ""));
normalized.put("hoststub", row.optString("hoststub", ""));
normalized.put("api_key", row.optString("api_key", ""));
- normalized.put("max_tokens", row.optString("max_tokens", "4096"));
+ normalized.put("max_tokens", row.optString("max_tokens", String.valueOf(net.yacy.ai.LLM.DEFAULT_MAX_TOKENS)));
normalized.put("search", false);
normalized.put("chat", row.optBoolean("chat", false));
@@ -146,13 +146,30 @@ public class LLMSelection_p {
if (inferenceSystem != null) {
sb.setConfig("ai.inference_system", inferenceSystem.toString());
}
+
+ JSONObject serviceNumCtx = bodyj.optJSONObject("service_num_ctx");
+ if (serviceNumCtx != null) {
+ // per-service context window (num_ctx), keyed by normalized hoststub
+ try {
+ final JSONObject normalized = new JSONObject(true);
+ for (final String hoststub : serviceNumCtx.keySet()) {
+ final String key = net.yacy.ai.LLM.normalizeHoststub(hoststub);
+ if (key.isEmpty()) continue;
+ final int value = serviceNumCtx.optInt(hoststub, 0);
+ if (value > 0) normalized.put(key, value);
+ }
+ sb.setConfig(net.yacy.ai.LLM.SERVICE_NUM_CTX_CONFIG, normalized.toString());
+ } catch (JSONException e) {
+ //e.printStackTrace();
+ }
+ }
/*
{"production_models":[{
"service":"OLLAMA",
"model":"hf.co\/janhq\/Jan-v1-edge-gguf:Q4_K_M",
"hoststub":"http:\/\/localhost:11434",
"api_key":"",
- "max_tokens":"4096",
+ "max_tokens":"2048",
"answers":true,
"chat":true,
"translation":true,
@@ -183,7 +200,7 @@ public class LLMSelection_p {
prop.put("productionmodels_" + i + "_model", row.optString("model", ""));
prop.put("productionmodels_" + i + "_hoststub", row.optString("hoststub", ""));
prop.put("productionmodels_" + i + "_api_key", row.optString("api_key", ""));
- prop.put("productionmodels_" + i + "_max_tokens", row.optString("max_tokens", "4096"));
+ prop.put("productionmodels_" + i + "_max_tokens", row.optString("max_tokens", String.valueOf(net.yacy.ai.LLM.DEFAULT_MAX_TOKENS)));
prop.put("productionmodels_" + i + "_search", row.optBoolean("search", false));
prop.put("productionmodels_" + i + "_chat", row.optBoolean("chat", false));
@@ -218,6 +235,37 @@ public class LLMSelection_p {
e.printStackTrace();
}
+ // build the per-service table: one row per distinct hoststub found in the
+ // production models, with its configured context window (num_ctx)
+ try {
+ JSONObject numCtxMap = new JSONObject(true);
+ try {
+ numCtxMap = new JSONObject(new JSONTokener(sb.getConfig(net.yacy.ai.LLM.SERVICE_NUM_CTX_CONFIG, "{}")));
+ } catch (JSONException e) {
+ numCtxMap = new JSONObject(true);
+ }
+ final java.util.LinkedHashMap<String, String> serviceByHoststub = new java.util.LinkedHashMap<>();
+ if (production_models != null) {
+ for (int i = 0; i < production_models.length(); i++) {
+ final JSONObject row = production_models.getJSONObject(i);
+ final String hoststub = net.yacy.ai.LLM.normalizeHoststub(row.optString("hoststub", ""));
+ if (hoststub.isEmpty() || serviceByHoststub.containsKey(hoststub)) continue;
+ serviceByHoststub.put(hoststub, row.optString("service", "OLLAMA"));
+ }
+ }
+ int s = 0;
+ for (final java.util.Map.Entry<String, String> service : serviceByHoststub.entrySet()) {
+ final int numCtx = numCtxMap.optInt(service.getKey(), net.yacy.ai.LLM.DEFAULT_NUM_CTX);
+ prop.put("services_" + s + "_service", service.getValue());
+ prop.putHTML("services_" + s + "_hoststub", service.getKey());
+ prop.put("services_" + s + "_num_ctx", numCtx);
+ s++;
+ }
+ prop.put("services", s);
+ } catch (JSONException e) {
+ prop.put("services", 0);
+ }
+
try {
if (production_models != null) {
for (int i = 0; i < production_models.length(); i++) {
@@ -244,6 +292,10 @@ public class LLMSelection_p {
prop.putHTML("model_capabilities", "{}");
}
+ // expose the stored per-service num_ctx map to the page so the Services
+ // table can prefill the window for a selected-but-not-yet-deployed endpoint
+ prop.putHTML("service_num_ctx_json", sb.getConfig(net.yacy.ai.LLM.SERVICE_NUM_CTX_CONFIG, "{}"));
+
// prefill inference system configuration if present
final String inferenceJson = sb.getConfig("ai.inference_system", "{}");
try {
generated by cgit v1.2.3 (git 2.46.0) at 2026-08-08 08:24:55 +0000