diff options
| -rw-r--r-- | htroot/LLMSelection_p.html | 96 | ||||
| -rw-r--r-- | source/net/yacy/ai/LLM.java | 8 |
2 files changed, 101 insertions, 3 deletions
diff --git a/htroot/LLMSelection_p.html b/htroot/LLMSelection_p.html index fce0fcc45..30e3f3155 100644 --- a/htroot/LLMSelection_p.html +++ b/htroot/LLMSelection_p.html @@ -55,6 +55,7 @@ const PRODUCTION_MODEL_TOTAL_COLUMNS = 18; const PRODUCTION_MODEL_MODEL_COLUMN_INDEX = 1; + const PRODUCTION_MODEL_MAX_TOKENS_COLUMN_INDEX = 4; const PRODUCTION_MODEL_USAGE_COLUMN_START = 5; const PRODUCTION_MODEL_USAGE_COLUMN_END = 12; // including const PRODUCTION_MODEL_FEATURE_COLUMN_START = 13; @@ -881,6 +882,55 @@ upsertProductionModel(modelName); } + const MAX_TOKEN_OPTIONS = ["4096", "8192", "16384", "32768", "65536", "131072", "262440"]; + const DEFAULT_MAX_TOKENS = "16384"; + + function normalizeMaxTokenValue(value) { + const text = (value == null ? "" : String(value)).trim(); + return text || DEFAULT_MAX_TOKENS; + } + + // The per-row max_tokens cell is an editable <select> so the value can be + // changed per model after deployment; it is the persisted source of truth + // (the max_tokens field at the top of the page is only a default for the + // next deploy). This ensures such a select exists in the cell and selects + // the given value, adding a bespoke option if the value is not a preset. + function ensureMaxTokenSelect(cell, value) { + if (!cell) return; + const normalized = normalizeMaxTokenValue(value); + let select = cell.querySelector('select[data-role="max-tokens"]'); + if (!select) { + select = document.createElement("select"); + select.className = "form-control"; + select.dataset.role = "max-tokens"; + const optionValues = MAX_TOKEN_OPTIONS.includes(normalized) + ? MAX_TOKEN_OPTIONS + : [normalized, ...MAX_TOKEN_OPTIONS]; + optionValues.forEach(v => { + const option = document.createElement("option"); + option.value = v; + option.textContent = v; + select.appendChild(option); + }); + cell.textContent = ""; + cell.appendChild(select); + select.addEventListener("change", () => persistProductionModels()); + } else if (!Array.from(select.options).some(o => o.value === normalized)) { + const option = document.createElement("option"); + option.value = normalized; + option.textContent = normalized; + select.appendChild(option); + } + select.value = normalized; + } + + function readMaxTokenValue(cell) { + if (!cell) return DEFAULT_MAX_TOKENS; + const select = cell.querySelector('select[data-role="max-tokens"]'); + if (select) return normalizeMaxTokenValue(select.value); + return normalizeMaxTokenValue(cell.textContent); + } + function upsertProductionModel(modelName) { const tbody = getProductionTableBody(); if (!tbody) return; @@ -918,9 +968,17 @@ } const cells = targetRow.cells; - const values = [service, modelName, hoststub, apikey, maxToken]; + // for a new row take the max_tokens from the top selection; for an + // existing row keep the per-row value the user may have edited + const effectiveMaxToken = isNewRow + ? maxToken + : readMaxTokenValue(cells[PRODUCTION_MODEL_MAX_TOKENS_COLUMN_INDEX]); + const values = [service, modelName, hoststub, apikey, effectiveMaxToken]; values.forEach((value, index) => { - if (cells[index]) { + if (!cells[index]) return; + if (index === PRODUCTION_MODEL_MAX_TOKENS_COLUMN_INDEX) { + ensureMaxTokenSelect(cells[index], value); + } else { cells[index].textContent = value || ""; } }); @@ -948,6 +1006,8 @@ for (let i = 0; i < missingCells; i += 1) { row.appendChild(document.createElement("td")); } + const maxTokenCell = row.cells[PRODUCTION_MODEL_MAX_TOKENS_COLUMN_INDEX]; + ensureMaxTokenSelect(maxTokenCell, readMaxTokenValue(maxTokenCell)); ensureProductionRowUsageCells(row, false); ensureProductionRowActionButton(row); scheduleCapabilityVerificationForRow(row); @@ -1155,6 +1215,8 @@ } else if (index >= PRODUCTION_MODEL_USAGE_COLUMN_START && index <= PRODUCTION_MODEL_USAGE_COLUMN_END) { const checkbox = row.cells[index] ? row.cells[index].querySelector('input[type="checkbox"]') : null; rowData[columnName] = checkbox && isSelectableUsageColumn(index) ? checkbox.checked : false; + } else if (index === PRODUCTION_MODEL_MAX_TOKENS_COLUMN_INDEX) { + rowData[columnName] = readMaxTokenValue(row.cells[index]); } else { rowData[columnName] = row.cells[index] ? row.cells[index].textContent.trim() : ""; } @@ -1903,10 +1965,27 @@ return downloadBtn; } + // Mirror the most recently deployed row's max_tokens into the top select so + // the field reflects the current configuration instead of always resetting + // to the hardcoded default. The per-row selects remain the source of truth. + function prefillTopMaxTokenFromTable() { + const select = document.getElementById("maxtoken"); + const tbody = getProductionTableBody(); + if (!select || !tbody) return; + const rows = tbody.querySelectorAll("tr"); + if (!rows.length) return; + const lastRow = rows[rows.length - 1]; + const value = readMaxTokenValue(lastRow.cells[PRODUCTION_MODEL_MAX_TOKENS_COLUMN_INDEX]); + if (Array.from(select.options).some(o => o.value === value)) { + select.value = value; + } + } + document.addEventListener("DOMContentLoaded", () => { try { persistedModelCapabilities = readPersistedModelCapabilities(); normalizeProductionModelRows(); + prefillTopMaxTokenFromTable(); applyPresetInference(); resumePendingPulls(); // auto-show available models if a preset inference exists @@ -1978,7 +2057,18 @@ <option>65536</option> <option>131072</option> <option>262440</option> - </select> You must set the Context Length in the LLM service to fit to your selected max_tokens; in Ollama you find a Context Length slider in the settings + </select> + <span class="info"><img src="env/grafics/i16.gif" width="16" height="16" alt="info"/><span> + This is the default max_tokens applied to a model when you deploy it; you can change it per model afterwards in the Production Models Matrix below. + </span></span> + <br/> + <small> + <b>max_tokens</b> caps the number of <i>generated</i> tokens (sent as the OpenAI <code>max_tokens</code>, i.e. Ollama <code>num_predict</code>). + It does <b>not</b> enlarge the model's context window: Ollama defaults <code>num_ctx</code> to 4096 regardless of this value, + and its OpenAI-compatible endpoint (<code>/v1/chat/completions</code>) cannot set <code>num_ctx</code> per request. + To actually use a large context in Ollama, raise the context length once via the environment variable + <code>OLLAMA_CONTEXT_LENGTH</code>, a Modelfile <code>PARAMETER num_ctx <n></code>, or the Context Length slider in the Ollama app settings. + </small> </dd> <dt> </dt> diff --git a/source/net/yacy/ai/LLM.java b/source/net/yacy/ai/LLM.java index 5af3cd276..6bdeaef38 100644 --- a/source/net/yacy/ai/LLM.java +++ b/source/net/yacy/ai/LLM.java @@ -370,6 +370,12 @@ public class LLM { data.put("model", model); data.put("temperature", 0.1); data.put("max_tokens", max_tokens); + // Best-effort hint for Ollama's context window (num_ctx). Ollama's + // OpenAI-compatible endpoint does not read this today and pure-OpenAI + // backends ignore unknown fields, so it is a harmless forward-looking + // hedge; the reliable way to raise the context window remains + // OLLAMA_CONTEXT_LENGTH or a Modelfile PARAMETER num_ctx. + data.put("num_ctx", max_tokens); data.put("messages", context); data.put("stop", new JSONArray(STOPTOKENS)); data.put("stream", false); @@ -448,6 +454,8 @@ public class LLM { data.put("model", model); data.put("temperature", 0.1); data.put("max_tokens", max_tokens); + // best-effort num_ctx hint, see chat(); harmless to non-Ollama backends + data.put("num_ctx", max_tokens); data.put("messages", context); data.put("stop", new JSONArray(STOPTOKENS)); data.put("stream", true); |