summaryrefslogtreecommitdiff
path: root/source
diff options
context:
space:
mode:
Diffstat (limited to 'source')
-rw-r--r--source/net/yacy/ai/LLM.java70
-rw-r--r--source/net/yacy/ai/LogReportService.java38
-rw-r--r--source/net/yacy/htroot/LLMSelection_p.java58
3 files changed, 131 insertions, 35 deletions
diff --git a/source/net/yacy/ai/LLM.java b/source/net/yacy/ai/LLM.java
index 71ce2b598..3e6d74783 100644
--- a/source/net/yacy/ai/LLM.java
+++ b/source/net/yacy/ai/LLM.java
@@ -47,6 +47,8 @@ public class LLM {
private static final ConcurrentLog log = new ConcurrentLog("LLM");
private static final String MODEL_CAPABILITIES_CONFIG = "ai.model_capabilities";
+ /** config key: JSON object mapping a service hoststub to its context window (num_ctx) */
+ public static final String SERVICE_NUM_CTX_CONFIG = "ai.service_num_ctx";
private static String[] STOPTOKENS = new String[]{"[/INST]", "<|im_end|>", "<|end_of_turn|>", "<|eot_id|>", "<|end_header_id|>", "<EOS_TOKEN>", "</s>", "<|end|>"};
public static enum LLMType {
@@ -85,15 +87,31 @@ public class LLM {
}
}
+ /** Ollama's out-of-the-box context window; used when a service has no num_ctx configured. */
+ public static final int DEFAULT_NUM_CTX = 4096;
+ /**
+ * Sole fallback for a model's max_tokens when a config row is missing the field
+ * (legacy/malformed rows only). The Production Models Matrix is the place where
+ * max_tokens is defined; this must match the matrix UI default (DEFAULT_MAX_TOKENS
+ * in LLMSelection_p.html) so no divergent default exists.
+ */
+ public static final int DEFAULT_MAX_TOKENS = 2048;
+
public final String hoststub;
public final String api_key;
- public final int max_tokens; // the max_tokens as configured by the endpoint for all models
+ public final int max_tokens; // output-token cap (OpenAI max_tokens = Ollama num_predict)
+ public final int num_ctx; // context window of the inference service (per-service, advisory)
public final LLMType type;
-
+
public LLM(final String hoststub, final String api_key, final int max_tokens, final LLMType type) {
+ this(hoststub, api_key, max_tokens, DEFAULT_NUM_CTX, type);
+ }
+
+ public LLM(final String hoststub, final String api_key, final int max_tokens, final int num_ctx, final LLMType type) {
this.hoststub = hoststub.endsWith("/") ? hoststub.substring(0, hoststub.length() - 1) : hoststub;
this.api_key = api_key == null ? "" : api_key;
- this.max_tokens = max_tokens <= 0 ? 4096 : max_tokens;
+ this.max_tokens = max_tokens <= 0 ? DEFAULT_MAX_TOKENS : max_tokens;
+ this.num_ctx = num_ctx <= 0 ? DEFAULT_NUM_CTX : num_ctx;
this.type = type;
}
@@ -133,7 +151,7 @@ public class LLM {
// found one that shall be used for this use case
final String hoststub = row.optString("hoststub", "");
final String api_key = row.optString("api_key", "");
- final int max_tokens = Integer.parseInt(row.optString("max_tokens", "4096"));
+ final int max_tokens = Integer.parseInt(row.optString("max_tokens", String.valueOf(DEFAULT_MAX_TOKENS)));
final String model = row.optString("model", "");
final LLMType type = LLMType.valueOf(row.optString("service", "OLLAMA"));
boolean tooling = row.optBoolean("tooling", false);
@@ -145,10 +163,11 @@ public class LLM {
if (!thinking) thinking = "supported".equals(capabilityEntry.optString("thinking", ""));
}
}
- LLM llm = new LLM(hoststub, api_key, max_tokens, type);
+ final int num_ctx = serviceNumCtx(sb, hoststub);
+ LLM llm = new LLM(hoststub, api_key, max_tokens, num_ctx, type);
LLMModel llmmodel = new LLMModel(llm, model, tooling, thinking);
if (logRouting) {
- log.info(routePrefix(runId, caller) + "event=model-routing phase=select usage=" + llmUsage + " row=" + i + " service=" + type.name() + " model=" + LogRedaction.redact(model) + " backend=" + LogRedaction.redact(llm.hoststub) + " maxTokens=" + llm.max_tokens + " tooling=" + tooling + " thinking=" + thinking + " productionRows=" + production_models.length() + " durationMs=" + elapsed(start));
+ log.info(routePrefix(runId, caller) + "event=model-routing phase=select usage=" + llmUsage + " row=" + i + " service=" + type.name() + " model=" + LogRedaction.redact(model) + " backend=" + LogRedaction.redact(llm.hoststub) + " maxTokens=" + llm.max_tokens + " numCtx=" + llm.num_ctx + " tooling=" + tooling + " thinking=" + thinking + " productionRows=" + production_models.length() + " durationMs=" + elapsed(start));
}
return llmmodel;
}
@@ -187,6 +206,31 @@ public class LLM {
return normalizedType + "|" + normalizedHoststub + "|" + normalizedModel;
}
+ /** Normalize a hoststub for use as a service key: trim and drop trailing slashes. */
+ public static String normalizeHoststub(final String hoststub) {
+ if (hoststub == null) return "";
+ return hoststub.trim().replaceAll("/+$", "");
+ }
+
+ /**
+ * Context window (num_ctx) configured for the inference service at the given hoststub.
+ * num_ctx is a per-service setting (a self-hosted server exposes one context length for
+ * all its models via OLLAMA_CONTEXT_LENGTH), stored under SERVICE_NUM_CTX_CONFIG keyed by
+ * hoststub. The value is advisory: YaCy uses it to budget the prompt against the window,
+ * it does not enforce it on the backend. Falls back to DEFAULT_NUM_CTX when unset.
+ */
+ public static int serviceNumCtx(final Switchboard sb, final String hoststub) {
+ if (sb == null) return DEFAULT_NUM_CTX;
+ final String json = sb.getConfig(SERVICE_NUM_CTX_CONFIG, "{}");
+ try {
+ final JSONObject map = new JSONObject(new JSONTokener(json));
+ final int value = map.optInt(normalizeHoststub(hoststub), 0);
+ return value <= 0 ? DEFAULT_NUM_CTX : value;
+ } catch (final JSONException e) {
+ return DEFAULT_NUM_CTX;
+ }
+ }
+
private static JSONObject readModelCapabilities() {
final Switchboard sb = Switchboard.getSwitchboard();
if (sb == null) return new JSONObject(true);
@@ -370,12 +414,12 @@ public class LLM {
data.put("model", model);
data.put("temperature", 0.1);
data.put("max_tokens", max_tokens);
- // Best-effort hint for Ollama's context window (num_ctx). Ollama's
- // OpenAI-compatible endpoint does not read this today and pure-OpenAI
- // backends ignore unknown fields, so it is a harmless forward-looking
- // hedge; the reliable way to raise the context window remains
+ // Best-effort hint for Ollama's context window (num_ctx), taken from the
+ // per-service configuration. Ollama's OpenAI-compatible endpoint does not read
+ // this today and pure-OpenAI backends ignore unknown fields, so it is a harmless
+ // forward-looking hedge; the reliable way to raise the context window remains
// OLLAMA_CONTEXT_LENGTH or a Modelfile PARAMETER num_ctx.
- data.put("num_ctx", max_tokens);
+ data.put("num_ctx", this.num_ctx);
data.put("messages", context);
data.put("stop", new JSONArray(STOPTOKENS));
data.put("stream", false);
@@ -454,8 +498,8 @@ public class LLM {
data.put("model", model);
data.put("temperature", 0.1);
data.put("max_tokens", max_tokens);
- // best-effort num_ctx hint, see chat(); harmless to non-Ollama backends
- data.put("num_ctx", max_tokens);
+ // best-effort num_ctx hint from the per-service config, see chat(); harmless to non-Ollama backends
+ data.put("num_ctx", this.num_ctx);
data.put("messages", context);
data.put("stop", new JSONArray(STOPTOKENS));
data.put("stream", true);
diff --git a/source/net/yacy/ai/LogReportService.java b/source/net/yacy/ai/LogReportService.java
index 00141d3d0..af1aaab9c 100644
--- a/source/net/yacy/ai/LogReportService.java
+++ b/source/net/yacy/ai/LogReportService.java
@@ -61,7 +61,6 @@ public class LogReportService {
public static final String CONFIG_MAX_BUCKET_LINES = "ai.logreport.max_bucket_lines";
public static final String CONFIG_INITIAL_DELAY_MINUTES = "ai.logreport.initial_delay_minutes";
public static final String CONFIG_PERIOD_MINUTES = "ai.logreport.period_minutes";
- public static final String CONFIG_MAX_TOKENS = "ai.logreport.max_tokens";
public static final String CONFIG_DAILY_COMPRESSION_ENABLED = "ai.logreport.daily_compression.enabled";
public static final String CONFIG_FEED_MAX_ENTRIES = "ai.logreport.feed.max_entries";
public static final String DEFAULT_REPORT_DIR = "DATA/REPORTS/log";
@@ -352,8 +351,8 @@ public class LogReportService {
}
try {
final NoiseSummary noiseSummary = classifyNoise(bucket.getValue());
- final int configuredMaxTokens = this.sb.getConfigInt(CONFIG_MAX_TOKENS, model.llm.max_tokens);
- final int maxTokens = Math.max(1, Math.min(model.llm.max_tokens, configuredMaxTokens));
+ // output cap follows the model's configured max_tokens (Production Models Matrix)
+ final int maxTokens = Math.max(1, model.llm.max_tokens);
final String prompt = hourlyPrompt(bucket.getKey(), bucket.getValue(), noiseSummary, promptPayloadCharBudget(model, maxTokens));
log.info("runId=" + runId + " event=hourly-report phase=classify-noise bucket=" + bucket.getKey() + " inputLines=" + bucket.getValue().size() + " noiseLines=" + noiseSummary.classifiedLines() + " noiseCategories=" + noiseSummary.buckets.size());
log.info("runId=" + runId + " event=hourly-report phase=model-call bucket=" + bucket.getKey() + " model=" + LogRedaction.redact(model.model) + " backend=" + LogRedaction.redact(model.llm.hoststub) + " inputLines=" + bucket.getValue().size() + " promptChars=" + prompt.length() + " maxTokens=" + maxTokens);
@@ -412,8 +411,8 @@ public class LogReportService {
final File reportFile = new File(reportDirectory, hourlyReportFilename(currentHour));
final NoiseSummary noiseSummary = classifyNoise(bucketLines);
- final int configuredMaxTokens = this.sb.getConfigInt(CONFIG_MAX_TOKENS, model.llm.max_tokens);
- final int maxTokens = Math.max(1, Math.min(model.llm.max_tokens, configuredMaxTokens));
+ // output cap follows the model's configured max_tokens (Production Models Matrix)
+ final int maxTokens = Math.max(1, model.llm.max_tokens);
final String prompt = hourlyPrompt(currentHour, bucketLines, noiseSummary, promptPayloadCharBudget(model, maxTokens));
log.info("runId=" + runId + " event=current-hour-report phase=classify-noise bucket=" + currentHour + " inputLines=" + bucketLines.size() + " noiseLines=" + noiseSummary.classifiedLines() + " noiseCategories=" + noiseSummary.buckets.size());
log.info("runId=" + runId + " event=current-hour-report phase=model-call bucket=" + currentHour + " model=" + LogRedaction.redact(model.model) + " backend=" + LogRedaction.redact(model.llm.hoststub) + " inputLines=" + bucketLines.size() + " promptChars=" + prompt.length() + " maxTokens=" + maxTokens);
@@ -476,8 +475,8 @@ public class LogReportService {
continue;
}
try {
- final int configuredMaxTokens = this.sb.getConfigInt(CONFIG_MAX_TOKENS, model.llm.max_tokens);
- final int maxTokens = Math.max(1, Math.min(model.llm.max_tokens, configuredMaxTokens));
+ // output cap follows the model's configured max_tokens (Production Models Matrix)
+ final int maxTokens = Math.max(1, model.llm.max_tokens);
final String prompt = dailyPrompt(day.getKey(), day.getValue(), promptPayloadCharBudget(model, maxTokens));
log.info("runId=" + runId + " event=daily-report phase=model-call day=" + day.getKey() + " model=" + LogRedaction.redact(model.model) + " backend=" + LogRedaction.redact(model.llm.hoststub) + " sourceReports=" + day.getValue().size() + " promptChars=" + prompt.length() + " maxTokens=" + maxTokens);
final long modelStart = System.currentTimeMillis();
@@ -631,22 +630,23 @@ public class LogReportService {
/**
* Character budget for the variable part of a report prompt (log lines or hourly
- * reports), derived from the model's token window. A local model must ingest the
- * entire prompt before it emits a single output token, and the prompt and the
- * generated report share one context window: prompt + output has to fit into
- * model.llm.max_tokens. The payload is therefore sized to (window - output reserve)
- * tokens, converted to characters.
+ * reports), derived from the service's context window (num_ctx). A local model must
+ * ingest the entire prompt before it emits a single output token, and the prompt and
+ * the generated report share one context window: prompt + output has to fit into
+ * num_ctx. The payload is therefore sized to (num_ctx - output reserve) tokens,
+ * converted to characters.
* <p>
* A fixed budget (previously 64k chars ≈ 16k tokens) overflows small windows: with a
- * default 4k-token model the ~16k-token hourly prompt filled the whole window, left no
- * room to generate, and the report stopped after one or two tokens. When the output
- * cap already fills the window (the common case where max_tokens equals the context
- * length) the payload falls back to a quarter of the window so a report is still
- * produced; if the output then hits its cap it is truncated and logged
- * (finish_reason=length) instead of the prompt silently overflowing.
+ * default 4k-token window the ~16k-token hourly prompt filled the whole window, left no
+ * room to generate, and the report stopped after one or two tokens. When the output cap
+ * already fills the window the payload falls back to a quarter of the window so a report
+ * is still produced; if the output then hits its cap it is truncated and logged
+ * (finish_reason=length) instead of the prompt silently overflowing. Raise the service's
+ * num_ctx (on /LLMSelection_p.html, matching the backend's OLLAMA_CONTEXT_LENGTH) to give
+ * the prompt more room.
*/
private static int promptPayloadCharBudget(final LLMModel model, final int maxTokens) {
- final int contextTokens = Math.max(1, model.llm.max_tokens);
+ final int contextTokens = Math.max(1, model.llm.num_ctx);
final int promptTokens = Math.max(
Math.max(MIN_PROMPT_PAYLOAD_TOKENS, contextTokens / 4),
contextTokens - maxTokens);
diff --git a/source/net/yacy/htroot/LLMSelection_p.java b/source/net/yacy/htroot/LLMSelection_p.java
index 960a3e985..405bcf773 100644
--- a/source/net/yacy/htroot/LLMSelection_p.java
+++ b/source/net/yacy/htroot/LLMSelection_p.java
@@ -87,7 +87,7 @@ public class LLMSelection_p {
normalized.put("model", row.optString("model", ""));
normalized.put("hoststub", row.optString("hoststub", ""));
normalized.put("api_key", row.optString("api_key", ""));
- normalized.put("max_tokens", row.optString("max_tokens", "4096"));
+ normalized.put("max_tokens", row.optString("max_tokens", String.valueOf(net.yacy.ai.LLM.DEFAULT_MAX_TOKENS)));
normalized.put("search", false);
normalized.put("chat", row.optBoolean("chat", false));
@@ -146,13 +146,30 @@ public class LLMSelection_p {
if (inferenceSystem != null) {
sb.setConfig("ai.inference_system", inferenceSystem.toString());
}
+
+ JSONObject serviceNumCtx = bodyj.optJSONObject("service_num_ctx");
+ if (serviceNumCtx != null) {
+ // per-service context window (num_ctx), keyed by normalized hoststub
+ try {
+ final JSONObject normalized = new JSONObject(true);
+ for (final String hoststub : serviceNumCtx.keySet()) {
+ final String key = net.yacy.ai.LLM.normalizeHoststub(hoststub);
+ if (key.isEmpty()) continue;
+ final int value = serviceNumCtx.optInt(hoststub, 0);
+ if (value > 0) normalized.put(key, value);
+ }
+ sb.setConfig(net.yacy.ai.LLM.SERVICE_NUM_CTX_CONFIG, normalized.toString());
+ } catch (JSONException e) {
+ //e.printStackTrace();
+ }
+ }
/*
{"production_models":[{
"service":"OLLAMA",
"model":"hf.co\/janhq\/Jan-v1-edge-gguf:Q4_K_M",
"hoststub":"http:\/\/localhost:11434",
"api_key":"",
- "max_tokens":"4096",
+ "max_tokens":"2048",
"answers":true,
"chat":true,
"translation":true,
@@ -183,7 +200,7 @@ public class LLMSelection_p {
prop.put("productionmodels_" + i + "_model", row.optString("model", ""));
prop.put("productionmodels_" + i + "_hoststub", row.optString("hoststub", ""));
prop.put("productionmodels_" + i + "_api_key", row.optString("api_key", ""));
- prop.put("productionmodels_" + i + "_max_tokens", row.optString("max_tokens", "4096"));
+ prop.put("productionmodels_" + i + "_max_tokens", row.optString("max_tokens", String.valueOf(net.yacy.ai.LLM.DEFAULT_MAX_TOKENS)));
prop.put("productionmodels_" + i + "_search", row.optBoolean("search", false));
prop.put("productionmodels_" + i + "_chat", row.optBoolean("chat", false));
@@ -218,6 +235,37 @@ public class LLMSelection_p {
e.printStackTrace();
}
+ // build the per-service table: one row per distinct hoststub found in the
+ // production models, with its configured context window (num_ctx)
+ try {
+ JSONObject numCtxMap = new JSONObject(true);
+ try {
+ numCtxMap = new JSONObject(new JSONTokener(sb.getConfig(net.yacy.ai.LLM.SERVICE_NUM_CTX_CONFIG, "{}")));
+ } catch (JSONException e) {
+ numCtxMap = new JSONObject(true);
+ }
+ final java.util.LinkedHashMap<String, String> serviceByHoststub = new java.util.LinkedHashMap<>();
+ if (production_models != null) {
+ for (int i = 0; i < production_models.length(); i++) {
+ final JSONObject row = production_models.getJSONObject(i);
+ final String hoststub = net.yacy.ai.LLM.normalizeHoststub(row.optString("hoststub", ""));
+ if (hoststub.isEmpty() || serviceByHoststub.containsKey(hoststub)) continue;
+ serviceByHoststub.put(hoststub, row.optString("service", "OLLAMA"));
+ }
+ }
+ int s = 0;
+ for (final java.util.Map.Entry<String, String> service : serviceByHoststub.entrySet()) {
+ final int numCtx = numCtxMap.optInt(service.getKey(), net.yacy.ai.LLM.DEFAULT_NUM_CTX);
+ prop.put("services_" + s + "_service", service.getValue());
+ prop.putHTML("services_" + s + "_hoststub", service.getKey());
+ prop.put("services_" + s + "_num_ctx", numCtx);
+ s++;
+ }
+ prop.put("services", s);
+ } catch (JSONException e) {
+ prop.put("services", 0);
+ }
+
try {
if (production_models != null) {
for (int i = 0; i < production_models.length(); i++) {
@@ -244,6 +292,10 @@ public class LLMSelection_p {
prop.putHTML("model_capabilities", "{}");
}
+ // expose the stored per-service num_ctx map to the page so the Services
+ // table can prefill the window for a selected-but-not-yet-deployed endpoint
+ prop.putHTML("service_num_ctx_json", sb.getConfig(net.yacy.ai.LLM.SERVICE_NUM_CTX_CONFIG, "{}"));
+
// prefill inference system configuration if present
final String inferenceJson = sb.getConfig("ai.inference_system", "{}");
try {
generated by cgit v1.2.3 (git 2.46.0) at 2026-08-08 09:34:56 +0000