diff options
| author | Michael Peter Christen <mc@yacy.net> | 2026-07-04 10:57:22 +0200 |
|---|---|---|
| committer | Michael Peter Christen <mc@yacy.net> | 2026-07-04 10:57:22 +0200 |
| commit | a8b90eb3653cea8322a9e0101eda6c14c3f25cc8 (patch) | |
| tree | 721d222657f6d463be40fd0ae30a847f821cb0a8 /source/net | |
| parent | 177cc87b8967fc3f25b2f22e88611b99e61c02aa (diff) | |
| parent | e1c988b5c38972ae855efdabbc5e24586e938733 (diff) | |
Merge branch 'master' of https://github.com/yacy/yacy_search_server.git
Diffstat (limited to 'source/net')
| -rw-r--r-- | source/net/yacy/ai/LLM.java | 50 | ||||
| -rw-r--r-- | source/net/yacy/ai/LogReportService.java | 732 | ||||
| -rw-r--r-- | source/net/yacy/ai/RAGAugmentor.java | 72 | ||||
| -rw-r--r-- | source/net/yacy/ai/ToolCallProtocol.java | 57 | ||||
| -rw-r--r-- | source/net/yacy/cora/util/LogRedaction.java | 54 | ||||
| -rw-r--r-- | source/net/yacy/htroot/AILab.java | 2 | ||||
| -rw-r--r-- | source/net/yacy/htroot/LLMSelection_p.java | 7 | ||||
| -rw-r--r-- | source/net/yacy/htroot/LogReports_p.java | 94 | ||||
| -rw-r--r-- | source/net/yacy/htroot/ViewLog_p.java | 139 | ||||
| -rw-r--r-- | source/net/yacy/htroot/api/logreports.java | 64 | ||||
| -rw-r--r-- | source/net/yacy/http/servlets/RAGProxyServlet.java | 46 | ||||
| -rw-r--r-- | source/net/yacy/kelondro/logging/GuiHandler.java | 2 | ||||
| -rw-r--r-- | source/net/yacy/yacy.java | 63 |
13 files changed, 1278 insertions, 104 deletions
diff --git a/source/net/yacy/ai/LLM.java b/source/net/yacy/ai/LLM.java index 7f74943d0..2363230ab 100644 --- a/source/net/yacy/ai/LLM.java +++ b/source/net/yacy/ai/LLM.java @@ -39,10 +39,13 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONTokener; +import net.yacy.cora.util.ConcurrentLog; +import net.yacy.cora.util.LogRedaction; import net.yacy.search.Switchboard; public class LLM { + private static final ConcurrentLog log = new ConcurrentLog("LLM"); private static final String MODEL_CAPABILITIES_CONFIG = "ai.model_capabilities"; private static String[] STOPTOKENS = new String[]{"[/INST]", "<|im_end|>", "<|end_of_turn|>", "<|eot_id|>", "<|end_header_id|>", "<EOS_TOKEN>", "</s>", "<|end|>"}; @@ -65,7 +68,8 @@ public class LLM { classification, query, qapairs, - tldr + tldr, + logreport } public static class LLMModel { @@ -99,8 +103,25 @@ public class LLM { * @return */ public static LLMModel llmFromUsage(LLMUsage llmUsage) { - Switchboard sb = Switchboard.getSwitchboard(); - String pms = sb.getConfig("ai.production_models", "[]"); + return llmFromUsage(llmUsage, null, null); + } + + public static LLMModel llmFromUsage(final LLMUsage llmUsage, final String runId, final String caller) { + return llmFromUsage(llmUsage, runId, caller, true); + } + + public static LLMModel llmFromUsageQuiet(final LLMUsage llmUsage) { + return llmFromUsage(llmUsage, null, null, false); + } + + private static LLMModel llmFromUsage(final LLMUsage llmUsage, final String runId, final String caller, final boolean logRouting) { + final long start = System.currentTimeMillis(); + final Switchboard sb = Switchboard.getSwitchboard(); + if (sb == null) { + if (logRouting) log.warn(routePrefix(runId, caller) + "event=model-routing phase=fail usage=" + llmUsage + " reason=switchboard-unavailable durationMs=" + elapsed(start)); + return null; + } + final String pms = sb.getConfig("ai.production_models", "[]"); JSONObject model_capabilities = readModelCapabilities(); try { JSONArray production_models = new JSONArray(new JSONTokener(pms)); @@ -126,15 +147,34 @@ public class LLM { } LLM llm = new LLM(hoststub, api_key, max_tokens, type); LLMModel llmmodel = new LLMModel(llm, model, tooling, thinking); + if (logRouting) { + log.info(routePrefix(runId, caller) + "event=model-routing phase=select usage=" + llmUsage + " row=" + i + " service=" + type.name() + " model=" + LogRedaction.redact(model) + " backend=" + LogRedaction.redact(llm.hoststub) + " maxTokens=" + llm.max_tokens + " tooling=" + tooling + " thinking=" + thinking + " productionRows=" + production_models.length() + " durationMs=" + elapsed(start)); + } return llmmodel; } } - } catch (JSONException | NumberFormatException e) { - e.printStackTrace(); + if (logRouting) { + log.info(routePrefix(runId, caller) + "event=model-routing phase=miss usage=" + llmUsage + " productionRows=" + production_models.length() + " durationMs=" + elapsed(start)); + } + } catch (JSONException | IllegalArgumentException e) { + if (logRouting) { + log.warn(routePrefix(runId, caller) + "event=model-routing phase=fail usage=" + llmUsage + " errorClass=" + e.getClass().getName() + " reason=" + LogRedaction.redactMessage(e) + " durationMs=" + elapsed(start)); + } } // so if we don't find a model for that specific usage, we purposely return null to show that there is a missing configuration return null; } + + private static String routePrefix(final String runId, final String caller) { + final StringBuilder prefix = new StringBuilder(); + if (runId != null && !runId.isEmpty()) prefix.append("runId=").append(runId).append(' '); + if (caller != null && !caller.isEmpty()) prefix.append("caller=").append(caller).append(' '); + return prefix.toString(); + } + + private static long elapsed(final long start) { + return System.currentTimeMillis() - start; + } public String getHoststub() { return this.hoststub; diff --git a/source/net/yacy/ai/LogReportService.java b/source/net/yacy/ai/LogReportService.java new file mode 100644 index 000000000..ddee5db9c --- /dev/null +++ b/source/net/yacy/ai/LogReportService.java @@ -0,0 +1,732 @@ +/** + * LogReportService + * Copyright 2026 by contributors to the YaCy project + * First released 26.06.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.ai; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; +import java.util.logging.Handler; +import java.util.logging.Logger; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import net.yacy.ai.LLM.LLMModel; +import net.yacy.ai.LLM.LLMUsage; +import net.yacy.cora.util.ConcurrentLog; +import net.yacy.cora.util.LogRedaction; +import net.yacy.kelondro.logging.GuiHandler; +import net.yacy.search.Switchboard; + +public class LogReportService { + + public static final String CONFIG_REPORT_DIR = "ai.logreport.dir"; + public static final String CONFIG_ENABLED = "ai.logreport.enabled"; + public static final String CONFIG_MAX_BUCKET_LINES = "ai.logreport.max_bucket_lines"; + public static final String CONFIG_INITIAL_DELAY_MINUTES = "ai.logreport.initial_delay_minutes"; + public static final String CONFIG_PERIOD_MINUTES = "ai.logreport.period_minutes"; + public static final String CONFIG_MAX_TOKENS = "ai.logreport.max_tokens"; + public static final String CONFIG_DAILY_COMPRESSION_ENABLED = "ai.logreport.daily_compression.enabled"; + public static final String CONFIG_DELETE_HOURLY_AFTER_DAILY = "ai.logreport.daily_compression.delete_hourly"; + public static final String CONFIG_FEED_MAX_ENTRIES = "ai.logreport.feed.max_entries"; + public static final String DEFAULT_REPORT_DIR = "DATA/REPORTS/log"; + public static final int DEFAULT_MAX_BUCKET_LINES = 100000; + public static final int DEFAULT_FEED_MAX_ENTRIES = 100; + public static final long DEFAULT_INITIAL_DELAY_MINUTES = 5L; + public static final long DEFAULT_PERIOD_MINUTES = 60L; + + private static final ConcurrentLog log = new ConcurrentLog("LOGREPORT"); + private static final DateTimeFormatter LOG_TIMESTAMP_FORMATTER = DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss"); + private static final DateTimeFormatter HOURLY_REPORT_FORMATTER = DateTimeFormatter.ofPattern("'report-'yyyy-MM-dd-HH'.md'"); + private static final DateTimeFormatter DAILY_REPORT_FORMATTER = DateTimeFormatter.ofPattern("'report-'yyyy-MM-dd'.md'"); + private static final ZoneId REPORT_ZONE = ZoneId.systemDefault(); + private static final String SYSTEM_PROMPT = + "You evaluate YaCy runtime logs for operator-facing self-enhancement reports. " + + "Do not suggest automatic code changes. Identify operational evidence and practical development opportunities."; + private static final int NOISE_EXAMPLE_LIMIT = 3; + + private final Switchboard sb; + + public static class ReportEntry implements Comparable<ReportEntry> { + public final String filename; + public final String title; + public final ZonedDateTime published; + public final String content; + public final boolean daily; + + public ReportEntry(final String filename, final String title, final ZonedDateTime published, final String content, final boolean daily) { + this.filename = filename; + this.title = title; + this.published = published; + this.content = content; + this.daily = daily; + } + + @Override + public int compareTo(final ReportEntry other) { + final int dateCompare = other.published.compareTo(this.published); + if (dateCompare != 0) return dateCompare; + return other.filename.compareTo(this.filename); + } + } + + private static class NoiseBucket { + int count; + final List<String> examples = new ArrayList<>(NOISE_EXAMPLE_LIMIT); + } + + private static class NoiseSummary { + final int totalLines; + final Map<String, NoiseBucket> buckets = new LinkedHashMap<>(); + + NoiseSummary(final int totalLines) { + this.totalLines = totalLines; + } + + int classifiedLines() { + int count = 0; + for (final NoiseBucket bucket : this.buckets.values()) count += bucket.count; + return count; + } + } + + public LogReportService(final Switchboard sb) { + this.sb = sb; + } + + public static boolean hasConfiguredLogReportModel() { + return LLM.llmFromUsageQuiet(LLMUsage.logreport) != null; + } + + public static ScheduledExecutorService startScheduler(final Switchboard sb) { + if (sb == null) return null; + if (!sb.getConfigBool(CONFIG_ENABLED, true)) { + log.info("Log report scheduler is disabled by " + CONFIG_ENABLED + "."); + return null; + } + if (!hasConfiguredLogReportModel()) { + log.info("Log report scheduler is inactive because no production model is configured for logreport usage."); + return null; + } + + final LogReportService service = new LogReportService(sb); + final long initialDelay = Math.max(0L, sb.getConfigLong(CONFIG_INITIAL_DELAY_MINUTES, DEFAULT_INITIAL_DELAY_MINUTES)); + final long period = Math.max(1L, sb.getConfigLong(CONFIG_PERIOD_MINUTES, DEFAULT_PERIOD_MINUTES)); + final AtomicBoolean missingModelLogged = new AtomicBoolean(false); + final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> { + final Thread thread = new Thread(r, "LogReportService.scheduler"); + thread.setDaemon(true); + return thread; + }); + scheduler.scheduleAtFixedRate(() -> { + final String runId = newRunId(); + final long tickStart = System.currentTimeMillis(); + try { + if (!hasConfiguredLogReportModel()) { + if (missingModelLogged.compareAndSet(false, true)) { + log.info("runId=" + runId + " event=scheduler-tick phase=skip reason=no-logreport-model"); + } + return; + } + missingModelLogged.set(false); + log.info("runId=" + runId + " event=scheduler-tick phase=start periodMinutes=" + period); + final List<File> hourlyReports = service.generateHourlyReports(runId, "scheduler"); + final List<File> dailyReports = service.generateDailyReports(runId, "scheduler"); + log.info("runId=" + runId + " event=scheduler-tick phase=end result=success hourlyWritten=" + hourlyReports.size() + " dailyWritten=" + dailyReports.size() + " durationMs=" + elapsed(tickStart)); + } catch (final Throwable e) { + log.warn("runId=" + runId + " event=scheduler-tick phase=end result=failure errorClass=" + e.getClass().getName() + " reason=" + LogRedaction.redactMessage(e) + " durationMs=" + elapsed(tickStart)); + } + }, initialDelay, period, TimeUnit.MINUTES); + log.info("event=scheduler-start result=success initialDelayMinutes=" + initialDelay + " periodMinutes=" + period); + return scheduler; + } + + public static void stopScheduler(final ScheduledExecutorService scheduler) { + if (scheduler == null) return; + scheduler.shutdownNow(); + log.info("Log report scheduler stopped."); + } + + public File getReportDirectory() { + return this.sb.getDataPath(CONFIG_REPORT_DIR, DEFAULT_REPORT_DIR); + } + + public List<ReportEntry> discoverReports(final int maxEntries) { + final File reportDirectory = getReportDirectory(); + if (!reportDirectory.isDirectory()) { + log.info("No log reports discovered because report directory does not exist: " + reportDirectory.getAbsolutePath()); + return Collections.emptyList(); + } + final File[] files = reportDirectory.listFiles(); + if (files == null || files.length == 0) { + log.info("No log reports discovered in " + reportDirectory.getAbsolutePath() + "."); + return Collections.emptyList(); + } + + final List<ReportEntry> reports = new ArrayList<>(); + int candidates = 0; + for (final File file : files) { + if (!file.isFile()) continue; + candidates++; + final ReportEntry report = reportEntry(file); + if (report != null) reports.add(report); + } + Collections.sort(reports); + log.info("Discovered " + reports.size() + " log report(s) from " + candidates + " file(s) in " + reportDirectory.getAbsolutePath() + "."); + if (maxEntries >= 0 && reports.size() > maxEntries) { + return new ArrayList<>(reports.subList(0, maxEntries)); + } + return reports; + } + + public List<File> generateHourlyReports() { + return generateHourlyReports(newRunId(), "direct"); + } + + private List<File> generateHourlyReports(final String runId, final String trigger) { + final long start = System.currentTimeMillis(); + log.info("runId=" + runId + " event=hourly-reports phase=start trigger=" + trigger); + final LLMModel model = LLM.llmFromUsage(LLMUsage.logreport, runId, "hourly-reports"); + if (model == null) { + log.info("runId=" + runId + " event=hourly-reports phase=skip reason=no-logreport-model durationMs=" + elapsed(start)); + return Collections.emptyList(); + } + + final List<String> logLines = readRuntimeLogLines(); + final Map<LocalDateTime, List<String>> buckets = completedHourlyBuckets(logLines); + log.info("runId=" + runId + " event=hourly-reports phase=bucket-scan runtimeLines=" + logLines.size() + " buckets=" + buckets.size()); + if (buckets.isEmpty()) { + log.info("runId=" + runId + " event=hourly-reports phase=skip reason=no-completed-buckets durationMs=" + elapsed(start)); + return Collections.emptyList(); + } + + final File reportDirectory = getReportDirectory(); + if (!reportDirectory.exists() && !reportDirectory.mkdirs()) { + log.warn("runId=" + runId + " event=hourly-reports phase=prepare-directory result=failure path=" + reportDirectory.getAbsolutePath()); + return Collections.emptyList(); + } + + final List<File> writtenReports = new ArrayList<>(); + int skippedExisting = 0; + int failedReports = 0; + for (final Map.Entry<LocalDateTime, List<String>> bucket : buckets.entrySet()) { + final File reportFile = new File(reportDirectory, hourlyReportFilename(bucket.getKey())); + if (reportFile.exists()) { + skippedExisting++; + log.info("runId=" + runId + " event=hourly-report phase=skip bucket=" + bucket.getKey() + " reason=report-exists file=" + reportFile.getAbsolutePath()); + continue; + } + try { + final NoiseSummary noiseSummary = classifyNoise(bucket.getValue()); + final String prompt = hourlyPrompt(bucket.getKey(), bucket.getValue(), noiseSummary); + log.info("runId=" + runId + " event=hourly-report phase=classify-noise bucket=" + bucket.getKey() + " inputLines=" + bucket.getValue().size() + " noiseLines=" + noiseSummary.classifiedLines() + " noiseCategories=" + noiseSummary.buckets.size()); + final int configuredMaxTokens = this.sb.getConfigInt(CONFIG_MAX_TOKENS, model.llm.max_tokens); + final int maxTokens = Math.max(1, Math.min(model.llm.max_tokens, configuredMaxTokens)); + log.info("runId=" + runId + " event=hourly-report phase=model-call bucket=" + bucket.getKey() + " model=" + LogRedaction.redact(model.model) + " backend=" + LogRedaction.redact(model.llm.hoststub) + " inputLines=" + bucket.getValue().size() + " promptChars=" + prompt.length() + " maxTokens=" + maxTokens); + final long modelStart = System.currentTimeMillis(); + final String report = model.llm.chat(model.model, SYSTEM_PROMPT, prompt, maxTokens); + final long modelDuration = elapsed(modelStart); + log.info("runId=" + runId + " event=hourly-report phase=model-return bucket=" + bucket.getKey() + " durationMs=" + modelDuration + " outputChars=" + (report == null ? 0 : report.length())); + if (report == null || report.trim().isEmpty()) { + throw new IOException("model returned an empty report"); + } + writeReport(reportFile, reportDocument(bucket.getKey(), bucket.getValue().size(), report)); + writtenReports.add(reportFile); + log.info("runId=" + runId + " event=hourly-report phase=write result=success bucket=" + bucket.getKey() + " inputLines=" + bucket.getValue().size() + " outputChars=" + report.length() + " file=" + reportFile.getAbsolutePath()); + } catch (final IOException e) { + failedReports++; + log.warn("runId=" + runId + " event=hourly-report phase=write result=failure bucket=" + bucket.getKey() + " file=" + LogRedaction.redact(reportFile.getAbsolutePath()) + " errorClass=" + e.getClass().getName() + " reason=" + LogRedaction.redactMessage(e)); + } + } + log.info("runId=" + runId + " event=hourly-reports phase=end result=success buckets=" + buckets.size() + " written=" + writtenReports.size() + " skippedExisting=" + skippedExisting + " failed=" + failedReports + " durationMs=" + elapsed(start)); + return writtenReports; + } + + public File generateCurrentHourReportOverwrite() throws IOException { + final String runId = newRunId(); + final long start = System.currentTimeMillis(); + log.info("runId=" + runId + " event=current-hour-report phase=start mode=manual"); + final LLMModel model = LLM.llmFromUsage(LLMUsage.logreport, runId, "current-hour-report"); + if (model == null) { + log.info("runId=" + runId + " event=current-hour-report phase=skip reason=no-logreport-model durationMs=" + elapsed(start)); + return null; + } + + final LocalDateTime currentHour = LocalDateTime.now().truncatedTo(ChronoUnit.HOURS); + final List<String> logLines = readRuntimeLogLines(); + final List<String> bucketLines = hourlyBucket(logLines, currentHour); + log.info("runId=" + runId + " event=current-hour-report phase=bucket-scan bucket=" + currentHour + " runtimeLines=" + logLines.size() + " inputLines=" + bucketLines.size()); + if (bucketLines.isEmpty()) { + log.info("runId=" + runId + " event=current-hour-report phase=skip bucket=" + currentHour + " reason=no-matching-log-lines durationMs=" + elapsed(start)); + return null; + } + + final File reportDirectory = getReportDirectory(); + if (!reportDirectory.exists() && !reportDirectory.mkdirs()) { + log.warn("runId=" + runId + " event=current-hour-report phase=prepare-directory result=failure path=" + reportDirectory.getAbsolutePath()); + throw new IOException("cannot create log report directory " + reportDirectory.getAbsolutePath()); + } + + final File reportFile = new File(reportDirectory, hourlyReportFilename(currentHour)); + final NoiseSummary noiseSummary = classifyNoise(bucketLines); + final String prompt = hourlyPrompt(currentHour, bucketLines, noiseSummary); + log.info("runId=" + runId + " event=current-hour-report phase=classify-noise bucket=" + currentHour + " inputLines=" + bucketLines.size() + " noiseLines=" + noiseSummary.classifiedLines() + " noiseCategories=" + noiseSummary.buckets.size()); + final int configuredMaxTokens = this.sb.getConfigInt(CONFIG_MAX_TOKENS, model.llm.max_tokens); + final int maxTokens = Math.max(1, Math.min(model.llm.max_tokens, configuredMaxTokens)); + log.info("runId=" + runId + " event=current-hour-report phase=model-call bucket=" + currentHour + " model=" + LogRedaction.redact(model.model) + " backend=" + LogRedaction.redact(model.llm.hoststub) + " inputLines=" + bucketLines.size() + " promptChars=" + prompt.length() + " maxTokens=" + maxTokens); + final long modelStart = System.currentTimeMillis(); + final String report = model.llm.chat(model.model, SYSTEM_PROMPT, prompt, maxTokens); + log.info("runId=" + runId + " event=current-hour-report phase=model-return bucket=" + currentHour + " durationMs=" + elapsed(modelStart) + " outputChars=" + (report == null ? 0 : report.length())); + if (report == null || report.trim().isEmpty()) { + log.warn("runId=" + runId + " event=current-hour-report phase=model-return result=failure bucket=" + currentHour + " reason=empty-report"); + throw new IOException("model returned an empty report"); + } + if (reportFile.exists()) { + log.info("runId=" + runId + " event=current-hour-report phase=write action=overwrite file=" + reportFile.getAbsolutePath()); + } + writeReport(reportFile, reportDocument(currentHour, bucketLines.size(), report), true); + log.info("runId=" + runId + " event=current-hour-report phase=end result=success bucket=" + currentHour + " inputLines=" + bucketLines.size() + " outputChars=" + report.length() + " file=" + reportFile.getAbsolutePath() + " durationMs=" + elapsed(start)); + return reportFile; + } + + public List<File> generateDailyReports() { + return generateDailyReports(newRunId(), "direct"); + } + + private List<File> generateDailyReports(final String runId, final String trigger) { + final long start = System.currentTimeMillis(); + log.info("runId=" + runId + " event=daily-reports phase=start trigger=" + trigger); + if (!this.sb.getConfigBool(CONFIG_DAILY_COMPRESSION_ENABLED, true)) { + log.info("runId=" + runId + " event=daily-reports phase=skip reason=daily-compression-disabled durationMs=" + elapsed(start)); + return Collections.emptyList(); + } + final LLMModel model = LLM.llmFromUsage(LLMUsage.logreport, runId, "daily-reports"); + if (model == null) { + log.info("runId=" + runId + " event=daily-reports phase=skip reason=no-logreport-model durationMs=" + elapsed(start)); + return Collections.emptyList(); + } + + final File reportDirectory = getReportDirectory(); + if (!reportDirectory.isDirectory()) { + log.info("runId=" + runId + " event=daily-reports phase=skip reason=report-directory-missing path=" + reportDirectory.getAbsolutePath() + " durationMs=" + elapsed(start)); + return Collections.emptyList(); + } + + final Map<LocalDate, List<File>> completeDays = completeHourlyReportSets(reportDirectory); + log.info("runId=" + runId + " event=daily-reports phase=scan completeDays=" + completeDays.size() + " path=" + reportDirectory.getAbsolutePath()); + if (completeDays.isEmpty()) { + log.info("runId=" + runId + " event=daily-reports phase=skip reason=no-complete-day durationMs=" + elapsed(start)); + return Collections.emptyList(); + } + + final List<File> writtenReports = new ArrayList<>(); + int skippedExisting = 0; + int failedReports = 0; + for (final Map.Entry<LocalDate, List<File>> day : completeDays.entrySet()) { + final File dailyReportFile = new File(reportDirectory, dailyReportFilename(day.getKey())); + if (dailyReportFile.exists()) { + skippedExisting++; + log.info("runId=" + runId + " event=daily-report phase=skip day=" + day.getKey() + " reason=report-exists file=" + dailyReportFile.getAbsolutePath()); + continue; + } + try { + final String prompt = dailyPrompt(day.getKey(), day.getValue()); + final int configuredMaxTokens = this.sb.getConfigInt(CONFIG_MAX_TOKENS, model.llm.max_tokens); + final int maxTokens = Math.max(1, Math.min(model.llm.max_tokens, configuredMaxTokens)); + log.info("runId=" + runId + " event=daily-report phase=model-call day=" + day.getKey() + " model=" + LogRedaction.redact(model.model) + " backend=" + LogRedaction.redact(model.llm.hoststub) + " sourceReports=" + day.getValue().size() + " promptChars=" + prompt.length() + " maxTokens=" + maxTokens); + final long modelStart = System.currentTimeMillis(); + final String report = model.llm.chat(model.model, SYSTEM_PROMPT, prompt, maxTokens); + log.info("runId=" + runId + " event=daily-report phase=model-return day=" + day.getKey() + " durationMs=" + elapsed(modelStart) + " outputChars=" + (report == null ? 0 : report.length())); + if (report == null || report.trim().isEmpty()) { + throw new IOException("model returned an empty daily report"); + } + writeReport(dailyReportFile, dailyReportDocument(day.getKey(), day.getValue().size(), report)); + writtenReports.add(dailyReportFile); + log.info("runId=" + runId + " event=daily-report phase=write result=success day=" + day.getKey() + " sourceReports=" + day.getValue().size() + " outputChars=" + report.length() + " file=" + dailyReportFile.getAbsolutePath()); + if (this.sb.getConfigBool(CONFIG_DELETE_HOURLY_AFTER_DAILY, false)) { + deleteHourlyReports(day.getValue()); + log.info("runId=" + runId + " event=daily-report phase=delete-hourly day=" + day.getKey() + " sourceReports=" + day.getValue().size()); + } + } catch (final IOException e) { + failedReports++; + log.warn("runId=" + runId + " event=daily-report phase=write result=failure day=" + day.getKey() + " file=" + LogRedaction.redact(dailyReportFile.getAbsolutePath()) + " errorClass=" + e.getClass().getName() + " reason=" + LogRedaction.redactMessage(e)); + } + } + log.info("runId=" + runId + " event=daily-reports phase=end result=success completeDays=" + completeDays.size() + " written=" + writtenReports.size() + " skippedExisting=" + skippedExisting + " failed=" + failedReports + " durationMs=" + elapsed(start)); + return writtenReports; + } + + public Map<LocalDateTime, List<String>> completedHourlyBuckets(final List<String> logLines) { + final int maxBucketLines = this.sb.getConfigInt(CONFIG_MAX_BUCKET_LINES, DEFAULT_MAX_BUCKET_LINES); + final LocalDateTime currentHour = LocalDateTime.now().truncatedTo(ChronoUnit.HOURS); + final Map<LocalDateTime, List<String>> buckets = new TreeMap<>(); + + for (final String line : logLines) { + final LocalDateTime timestamp = parseLogTimestamp(line); + if (timestamp == null) continue; + final LocalDateTime bucket = timestamp.truncatedTo(ChronoUnit.HOURS); + if (!bucket.isBefore(currentHour)) continue; + List<String> bucketLines = buckets.get(bucket); + if (bucketLines == null) { + bucketLines = new ArrayList<>(); + buckets.put(bucket, bucketLines); + } + bucketLines.add(line); + while (bucketLines.size() > maxBucketLines) { + bucketLines.remove(0); + } + } + return buckets; + } + + private List<String> hourlyBucket(final List<String> logLines, final LocalDateTime selectedHour) { + final int maxBucketLines = this.sb.getConfigInt(CONFIG_MAX_BUCKET_LINES, DEFAULT_MAX_BUCKET_LINES); + final List<String> bucketLines = new ArrayList<>(); + for (final String line : logLines) { + final LocalDateTime timestamp = parseLogTimestamp(line); + if (timestamp == null || !selectedHour.equals(timestamp.truncatedTo(ChronoUnit.HOURS))) continue; + bucketLines.add(line); + while (bucketLines.size() > maxBucketLines) { + bucketLines.remove(0); + } + } + return bucketLines; + } + + public List<String> readRuntimeLogLines() { + final GuiHandler handler = findGuiHandler(); + if (handler == null) return Collections.emptyList(); + final String[] lines = handler.getLogLines(false, handler.getSize()); + final List<String> result = new ArrayList<>(lines.length); + for (final String line : lines) { + if (line == null) continue; + final String trimmed = line.trim(); + if (!trimmed.isEmpty()) result.add(trimmed); + } + return result; + } + + public static String hourlyReportFilename(final LocalDateTime bucket) { + return HOURLY_REPORT_FORMATTER.format(bucket); + } + + public static String dailyReportFilename(final LocalDate day) { + return DAILY_REPORT_FORMATTER.format(day); + } + + static LocalDateTime parseLogTimestamp(final String line) { + if (line == null || line.length() < 21) return null; + try { + return LocalDateTime.parse(line.substring(2, 21), LOG_TIMESTAMP_FORMATTER); + } catch (final RuntimeException e) { + return null; + } + } + + private static GuiHandler findGuiHandler() { + final Logger rootLogger = Logger.getLogger(""); + final Handler[] handlers = rootLogger.getHandlers(); + for (final Handler handler : handlers) { + if (handler instanceof GuiHandler) return (GuiHandler) handler; + } + return null; + } + + private static String newRunId() { + return UUID.randomUUID().toString(); + } + + private static long elapsed(final long start) { + return System.currentTimeMillis() - start; + } + + private static String hourlyPrompt(final LocalDateTime bucket, final List<String> lines) { + return hourlyPrompt(bucket, lines, classifyNoise(lines)); + } + + private static String hourlyPrompt(final LocalDateTime bucket, final List<String> lines, final NoiseSummary noiseSummary) { + final StringBuilder prompt = new StringBuilder(1024 + lines.size() * 120); + prompt.append("Create a YaCy self-enhancement log report for hour ") + .append(bucket) + .append(".\n\n") + .append("Noise classification:\n") + .append(noiseSummaryText(noiseSummary)) + .append("\nGuidance: repeated peer availability, remote search miss, and logreport self-observation noise should be summarized as operational background unless it correlates with user-visible failure, long latency, or a repeated code exception.\n\n") + .append("Use these fixed sections:\n") + .append("1. Summary / key takeaway\n") + .append("2. Usage types\n") + .append("3. Challenges\n") + .append("4. Errors and risks\n") + .append("5. Possible improvements\n") + .append("6. Concrete actions\n\n") + .append("Each concrete action must identify the log trigger and expected effect. ") + .append("Do not include secrets, credentials, or raw user content beyond what already appears in the log metadata.\n\n") + .append("Log lines:\n"); + for (final String line : lines) { + prompt.append(line).append('\n'); + } + return prompt.toString(); + } + + private static NoiseSummary classifyNoise(final List<String> lines) { + final NoiseSummary summary = new NoiseSummary(lines == null ? 0 : lines.size()); + if (lines == null || lines.isEmpty()) return summary; + for (final String line : lines) { + final String category = noiseCategory(line); + if (category == null) continue; + NoiseBucket bucket = summary.buckets.get(category); + if (bucket == null) { + bucket = new NoiseBucket(); + summary.buckets.put(category, bucket); + } + bucket.count++; + if (bucket.examples.size() < NOISE_EXAMPLE_LIMIT) { + bucket.examples.add(LogRedaction.redact(line)); + } + } + return summary; + } + + private static String noiseCategory(final String line) { + if (line == null || line.isEmpty()) return null; + final String normalized = line.toLowerCase(); + if (normalized.contains("network") && normalized.contains("publish: disconnected")) { + return "peer-publish-disconnected"; + } + if (normalized.contains("remote search - no answer from remote peer")) { + return "remote-search-no-answer"; + } + if (normalized.contains("remote search - interrupted search to remote peer")) { + return "remote-search-interrupted"; + } + if (normalized.contains("transfer to peer") && normalized.contains("failed")) { + return "peer-transfer-failed"; + } + if (normalized.contains("found not enough") && normalized.contains("peers for distribution")) { + return "dht-not-enough-peers"; + } + if (normalized.contains("logreport") && (normalized.contains("discovered ") || normalized.contains("no log reports discovered"))) { + return "logreport-self-observation"; + } + if (normalized.contains("event=model-routing phase=select") || normalized.contains("event=model-routing phase=miss")) { + return "llm-routing-observation"; + } + return null; + } + + private static String noiseSummaryText(final NoiseSummary summary) { + if (summary == null || summary.buckets.isEmpty()) { + return "- Classified noise lines: 0 / " + (summary == null ? 0 : summary.totalLines) + "\n"; + } + final StringBuilder text = new StringBuilder(512); + text.append("- Classified noise lines: ") + .append(summary.classifiedLines()) + .append(" / ") + .append(summary.totalLines) + .append('\n'); + for (final Map.Entry<String, NoiseBucket> entry : summary.buckets.entrySet()) { + text.append("- ") + .append(entry.getKey()) + .append(": ") + .append(entry.getValue().count) + .append('\n'); + for (final String example : entry.getValue().examples) { + text.append(" example: ").append(example).append('\n'); + } + } + return text.toString(); + } + + private static Map<LocalDate, List<File>> completeHourlyReportSets(final File reportDirectory) { + final File[] files = reportDirectory.listFiles(); + if (files == null || files.length == 0) return Collections.emptyMap(); + + final Map<LocalDate, List<File>> candidates = new TreeMap<>(); + for (final File file : files) { + if (!file.isFile()) continue; + final LocalDateTime hour = parseHourlyReportFilename(file.getName()); + if (hour == null) continue; + final LocalDate day = hour.toLocalDate(); + List<File> dayFiles = candidates.get(day); + if (dayFiles == null) { + dayFiles = new ArrayList<>(); + candidates.put(day, dayFiles); + } + dayFiles.add(file); + } + + final Map<LocalDate, List<File>> complete = new TreeMap<>(); + for (final Map.Entry<LocalDate, List<File>> entry : candidates.entrySet()) { + if (entry.getValue().size() != 24) continue; + final List<File> ordered = hourlyFilesForDay(reportDirectory, entry.getKey()); + if (ordered.size() == 24) complete.put(entry.getKey(), ordered); + } + return complete; + } + + private static LocalDateTime parseHourlyReportFilename(final String filename) { + if (filename == null || !filename.matches("report-\\d{4}-\\d{2}-\\d{2}-\\d{2}\\.md")) return null; + try { + return LocalDateTime.parse(filename, HOURLY_REPORT_FORMATTER); + } catch (final RuntimeException e) { + log.warn("Could not parse hourly log report filename " + filename + ": " + e.getMessage()); + return null; + } + } + + private static LocalDate parseDailyReportFilename(final String filename) { + if (filename == null || !filename.matches("report-\\d{4}-\\d{2}-\\d{2}\\.md")) return null; + try { + return LocalDate.parse(filename, DAILY_REPORT_FORMATTER); + } catch (final RuntimeException e) { + log.warn("Could not parse daily log report filename " + filename + ": " + e.getMessage()); + return null; + } + } + + private static ReportEntry reportEntry(final File file) { + final String filename = file.getName(); + try { + final LocalDate daily = parseDailyReportFilename(filename); + if (daily != null) { + final ZonedDateTime published = daily.atTime(LocalTime.MAX).atZone(REPORT_ZONE); + return new ReportEntry(filename, "YaCy daily log report " + daily, published, readFile(file), true); + } + final LocalDateTime hourly = parseHourlyReportFilename(filename); + if (hourly != null) { + final ZonedDateTime published = hourly.atZone(REPORT_ZONE); + return new ReportEntry(filename, "YaCy hourly log report " + hourly, published, readFile(file), false); + } + } catch (final IOException e) { + log.warn("Could not read log report " + file.getAbsolutePath() + ": " + e.getMessage()); + } + log.info("Ignoring file in log report directory because it does not match a report filename: " + file.getAbsolutePath()); + return null; + } + + private static String readFile(final File file) throws IOException { + return new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8); + } + + private static List<File> hourlyFilesForDay(final File reportDirectory, final LocalDate day) { + final List<File> files = new ArrayList<>(24); + for (int hour = 0; hour < 24; hour++) { + final File file = new File(reportDirectory, hourlyReportFilename(day.atTime(hour, 0))); + if (!file.isFile()) return Collections.emptyList(); + files.add(file); + } + return files; + } + + private static String dailyPrompt(final LocalDate day, final List<File> hourlyReports) throws IOException { + final StringBuilder prompt = new StringBuilder(4096); + prompt.append("Create one consolidated YaCy self-enhancement report for ") + .append(day) + .append(" from the following hourly log reports.\n\n") + .append("Use these fixed sections:\n") + .append("1. Summary / key takeaway\n") + .append("2. Usage types\n") + .append("3. Challenges\n") + .append("4. Errors and risks\n") + .append("5. Possible improvements\n") + .append("6. Concrete actions\n\n") + .append("Find common patterns across the day. Each concrete action must identify the repeated log trigger and expected effect.\n\n"); + for (final File hourlyReport : hourlyReports) { + prompt.append("\n\n## ").append(hourlyReport.getName()).append("\n\n") + .append(new String(Files.readAllBytes(hourlyReport.toPath()), StandardCharsets.UTF_8)); + } + return prompt.toString(); + } + + private static String reportDocument(final LocalDateTime bucket, final int lineCount, final String report) { + final StringBuilder document = new StringBuilder(report.length() + 256); + document.append("# YaCy Log Report\n\n") + .append("- Bucket: ").append(bucket).append('\n') + .append("- Lines: ").append(lineCount).append('\n') + .append("- Generated: ").append(LocalDateTime.now()).append("\n\n") + .append(report.trim()) + .append('\n'); + return document.toString(); + } + + private static String dailyReportDocument(final LocalDate day, final int hourlyReportCount, final String report) { + final StringBuilder document = new StringBuilder(report.length() + 256); + document.append("# YaCy Daily Log Report\n\n") + .append("- Day: ").append(day).append('\n') + .append("- Hourly reports: ").append(hourlyReportCount).append('\n') + .append("- Generated: ").append(LocalDateTime.now()).append("\n\n") + .append(report.trim()) + .append('\n'); + return document.toString(); + } + + private static void deleteHourlyReports(final List<File> hourlyReports) { + for (final File hourlyReport : hourlyReports) { + if (!hourlyReport.delete()) { + log.warn("Could not delete hourly log report after daily compression: " + hourlyReport.getAbsolutePath()); + } + } + } + + private static void writeReport(final File reportFile, final String report) throws IOException { + writeReport(reportFile, report, false); + } + + private static void writeReport(final File reportFile, final String report, final boolean replaceExisting) throws IOException { + final File parent = reportFile.getParentFile(); + if (parent != null && !parent.exists() && !parent.mkdirs()) { + throw new IOException("cannot create directory " + parent.getAbsolutePath()); + } + final File tmpFile = File.createTempFile(reportFile.getName() + ".", ".tmp", parent); + Files.write(tmpFile.toPath(), report.getBytes(StandardCharsets.UTF_8)); + try { + if (replaceExisting) { + Files.move(tmpFile.toPath(), reportFile.toPath(), StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } else { + Files.move(tmpFile.toPath(), reportFile.toPath(), StandardCopyOption.ATOMIC_MOVE); + } + } catch (final AtomicMoveNotSupportedException e) { + if (replaceExisting) { + Files.move(tmpFile.toPath(), reportFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + } else { + Files.move(tmpFile.toPath(), reportFile.toPath()); + } + } + } +} diff --git a/source/net/yacy/ai/RAGAugmentor.java b/source/net/yacy/ai/RAGAugmentor.java index a2dde813c..d2f38520d 100644 --- a/source/net/yacy/ai/RAGAugmentor.java +++ b/source/net/yacy/ai/RAGAugmentor.java @@ -41,6 +41,7 @@ import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.lod.vocabulary.Tagging; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.util.ConcurrentLog; +import net.yacy.cora.util.LogRedaction; import net.yacy.kelondro.data.meta.URIMetadataNode; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; @@ -99,8 +100,12 @@ public final class RAGAugmentor { * @return JSON array with {@code url,title[,text]} entries */ public static JSONArray searchResults(String query, int count, final boolean includeSnippet) { + return searchResults(query, count, includeSnippet, null); + } + + public static JSONArray searchResults(String query, int count, final boolean includeSnippet, final String runId) { final QueryParams theQuery = buildTextQueryParams(query, count, QueryParams.Searchdom.LOCAL); - return searchResults(theQuery, count, includeSnippet); + return searchResults(theQuery, count, includeSnippet, runId); } /** @@ -112,9 +117,13 @@ public final class RAGAugmentor { * @return markdown context block */ public static String searchResultsAsMarkdown(String query, int count, boolean global) { + return searchResultsAsMarkdown(query, count, global, null); + } + + public static String searchResultsAsMarkdown(String query, int count, boolean global, final String runId) { final long searchStart = System.currentTimeMillis(); - JSONArray searchResults = global ? searchResultsGlobal(query, count, true) : searchResults(query, count, true); - ConcurrentLog.info("RAGProxy", "searchResults=" + searchResults.length() + " global=" + global + " searchMs=" + (System.currentTimeMillis() - searchStart)); + JSONArray searchResults = global ? searchResultsGlobal(query, count, true, runId) : searchResults(query, count, true, runId); + ConcurrentLog.info("RAGProxy", prefix(runId) + "event=rag-search phase=results resultCount=" + searchResults.length() + " global=" + global + " searchMs=" + (System.currentTimeMillis() - searchStart)); StringBuilder sb = new StringBuilder(); for (int i = 0; i < searchResults.length(); i++) { @@ -135,7 +144,7 @@ public final class RAGAugmentor { } final String markdown = truncateSearchDocument(sb.toString()); - ConcurrentLog.info("RAGProxy", "markdownChars=" + markdown.length() + " resultCount=" + searchResults.length()); + ConcurrentLog.info("RAGProxy", prefix(runId) + "event=rag-search phase=markdown markdownChars=" + markdown.length() + " resultCount=" + searchResults.length()); return markdown; } @@ -184,8 +193,12 @@ public final class RAGAugmentor { * @return JSON array with normalized result objects */ public static JSONArray searchResultsGlobal(String query, int count, final boolean includeSnippet) { + return searchResultsGlobal(query, count, includeSnippet, null); + } + + public static JSONArray searchResultsGlobal(String query, int count, final boolean includeSnippet, final String runId) { final QueryParams theQuery = buildTextQueryParams(query, count, QueryParams.Searchdom.GLOBAL); - return searchResults(theQuery, count, includeSnippet); + return searchResults(theQuery, count, includeSnippet, runId); } /** @@ -199,9 +212,23 @@ public final class RAGAugmentor { * @return JSON array with normalized result objects */ private static JSONArray searchResults(final QueryParams theQuery, final int count, final boolean includeSnippet) { + return searchResults(theQuery, count, includeSnippet, null); + } + + private static JSONArray searchResults(final QueryParams theQuery, final int count, final boolean includeSnippet, final String runId) { + final long start = System.currentTimeMillis(); final JSONArray results = new JSONArray(); - if (theQuery == null || count == 0) return results; + if (theQuery == null || count == 0) { + ConcurrentLog.info("RAGProxy", prefix(runId) + "event=rag-search phase=skip reason=empty-query count=" + count); + return results; + } final Switchboard sb = Switchboard.getSwitchboard(); + if (sb == null) { + ConcurrentLog.warn("RAGProxy", prefix(runId) + "event=rag-search phase=fail reason=switchboard-unavailable"); + return results; + } + final boolean globalSearch = !theQuery.isLocal(); + ConcurrentLog.info("RAGProxy", prefix(runId) + "event=rag-search phase=start global=" + globalSearch + " count=" + count + " includeSnippet=" + includeSnippet); final SearchEvent theSearch = SearchEventCache.getEvent( theQuery, sb.peers, @@ -218,7 +245,6 @@ public final class RAGAugmentor { final long timeout = sb.getConfigLong( SwitchboardConstants.REMOTESEARCH_MAXTIME_USER, sb.getConfigLong(SwitchboardConstants.REMOTESEARCH_MAXTIME_DEFAULT, 3000)); - final boolean globalSearch = !theQuery.isLocal(); if (globalSearch) { theSearch.resortCachedResults(); } else { @@ -255,8 +281,10 @@ public final class RAGAugmentor { results.put(result); resultIndex++; } catch (JSONException e) { + ConcurrentLog.warn("RAGProxy", prefix(runId) + "event=rag-search phase=result result=failure errorClass=" + e.getClass().getName() + " reason=" + LogRedaction.redactMessage(e)); } } + ConcurrentLog.info("RAGProxy", prefix(runId) + "event=rag-search phase=end result=success global=" + globalSearch + " requested=" + count + " returned=" + results.length() + " includeSnippet=" + includeSnippet + " durationMs=" + (System.currentTimeMillis() - start)); return results; } @@ -332,17 +360,29 @@ public final class RAGAugmentor { * @return space-separated lowercase term list or {@code null} on failure */ public static String searchWordsForPrompt(final LLM llm, final String model, final String prompt, final String systemPrompt) { + return searchWordsForPrompt(llm, model, prompt, systemPrompt, null); + } + + public static String searchWordsForPrompt(final LLM llm, final String model, final String prompt, final String systemPrompt, final String runId) { + final long start = System.currentTimeMillis(); final String question = prompt == null ? "" : prompt.trim(); final String instruction = systemPrompt == null || systemPrompt.trim().isEmpty() ? "Compress the user prompt into a short search description. Return only a JSON array of concise, discriminative search terms in lowercase." : systemPrompt.trim(); - if (llm == null || model == null || model.isEmpty()) return null; + if (llm == null || model == null || model.isEmpty()) { + ConcurrentLog.warn("RAGProxy", prefix(runId) + "event=rag-query-generation phase=skip reason=no-model promptChars=" + question.length()); + return null; + } try { + ConcurrentLog.info("RAGProxy", prefix(runId) + "event=rag-query-generation phase=model-call model=" + LogRedaction.redact(model) + " backend=" + LogRedaction.redact(llm.hoststub) + " promptChars=" + question.length() + " promptWords=" + wordCount(question)); LLM.Context context = new LLM.Context(instruction); context.addPrompt(question); Set<String> singlewords = new LinkedHashSet<>(); String[] a = LLM.stringsFromChat(llm.chat(model, context, LLM.listSchema, 200)); - if (a == null || a.length == 0) return null; + if (a == null || a.length == 0) { + ConcurrentLog.warn("RAGProxy", prefix(runId) + "event=rag-query-generation phase=model-return result=empty durationMs=" + (System.currentTimeMillis() - start)); + return null; + } for (String s: a) { if (s == null) continue; // Flatten model output into unique lowercased tokens. @@ -353,13 +393,25 @@ public final class RAGAugmentor { for (String s: singlewords) query.append(s).append(' '); String querys = query.toString().trim(); if (querys.length() == 0) return null; + ConcurrentLog.info("RAGProxy", prefix(runId) + "event=rag-query-generation phase=end result=success terms=" + singlewords.size() + " queryChars=" + querys.length() + " durationMs=" + (System.currentTimeMillis() - start)); return querys; } catch (IOException | JSONException e) { - e.printStackTrace(); + ConcurrentLog.warn("RAGProxy", prefix(runId) + "event=rag-query-generation phase=end result=failure errorClass=" + e.getClass().getName() + " reason=" + LogRedaction.redactMessage(e) + " durationMs=" + (System.currentTimeMillis() - start)); return null; } } + private static String prefix(final String runId) { + return runId == null || runId.isEmpty() ? "" : "runId=" + runId + " "; + } + + private static int wordCount(final String text) { + if (text == null) return 0; + final String trimmed = text.trim(); + if (trimmed.isEmpty()) return 0; + return trimmed.split("\\s+").length; + } + /** * Splits text into sentence-aware chunks around a target max length. * diff --git a/source/net/yacy/ai/ToolCallProtocol.java b/source/net/yacy/ai/ToolCallProtocol.java index 2eab71687..d3e33dbaf 100644 --- a/source/net/yacy/ai/ToolCallProtocol.java +++ b/source/net/yacy/ai/ToolCallProtocol.java @@ -43,6 +43,8 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONTokener; +import net.yacy.cora.util.ConcurrentLog; +import net.yacy.cora.util.LogRedaction; /** * Implements the protocol glue for streamed chat completions that include tool @@ -63,6 +65,8 @@ import org.json.JSONTokener; */ public final class ToolCallProtocol { + private static final ConcurrentLog log = new ConcurrentLog("TOOLCALL"); + /** * Hard stop to prevent infinite tool loops (assistant requests tool calls * repeatedly without converging to a final answer). @@ -117,16 +121,24 @@ public final class ToolCallProtocol { * @throws IOException on network/stream/protocol errors */ public static int proxyToolLifecycle(ServletOutputStream out, LLM.LLMModel llm4Chat, JSONObject originalBody, JSONArray messages, JSONObject initialMetadata) throws IOException { + return proxyToolLifecycle(out, llm4Chat, originalBody, messages, initialMetadata, null); + } + + public static int proxyToolLifecycle(ServletOutputStream out, LLM.LLMModel llm4Chat, JSONObject originalBody, JSONArray messages, JSONObject initialMetadata, final String runId) throws IOException { + final long start = System.currentTimeMillis(); final JSONObject preparedBody = prepareToolRequestBody(originalBody, false, llm4Chat != null && llm4Chat.tooling); if (llm4Chat != null && llm4Chat.thinking) { LLM.applyNoThinkingParameters(preparedBody); } + log.info(prefix(runId) + "event=tool-lifecycle phase=start tooling=" + (llm4Chat != null && llm4Chat.tooling) + " thinking=" + (llm4Chat != null && llm4Chat.thinking) + " messages=" + (messages == null ? 0 : messages.length()) + " metadata=" + (initialMetadata == null ? 0 : initialMetadata.length())); final HttpURLConnection conn = openChatCompletionConnection(llm4Chat, preparedBody); final int status = conn.getResponseCode(); //final String message = conn.getResponseMessage(); if (status == 200) { - handleInitialStreamAndContinue(out, conn, llm4Chat, preparedBody, messages, initialMetadata); + handleInitialStreamAndContinue(out, conn, llm4Chat, preparedBody, messages, initialMetadata, runId); + log.info(prefix(runId) + "event=tool-lifecycle phase=end result=success status=" + status + " durationMs=" + (System.currentTimeMillis() - start)); } else { + log.warn(prefix(runId) + "event=tool-lifecycle phase=end result=upstream-error status=" + status + " reason=" + LogRedaction.redact(conn.getResponseMessage()) + " durationMs=" + (System.currentTimeMillis() - start)); Logger.getLogger("ToolCallProtocoll").severe(status + " " + conn.getResponseMessage()); } return status; @@ -196,6 +208,11 @@ public final class ToolCallProtocol { * @throws IOException on I/O errors */ private static void handleInitialStreamAndContinue(ServletOutputStream out, HttpURLConnection conn, LLM.LLMModel llm4Chat, JSONObject originalBody, JSONArray messages, JSONObject initialMetadata) throws IOException { + handleInitialStreamAndContinue(out, conn, llm4Chat, originalBody, messages, initialMetadata, null); + } + + private static void handleInitialStreamAndContinue(ServletOutputStream out, HttpURLConnection conn, LLM.LLMModel llm4Chat, JSONObject originalBody, JSONArray messages, JSONObject initialMetadata, final String runId) throws IOException { + final long start = System.currentTimeMillis(); final StringBuilder assistantContent = new StringBuilder(); final Map<Integer, ToolCall> toolCalls = new HashMap<>(); final boolean[] sawToolCalls = new boolean[]{false}; @@ -232,8 +249,9 @@ public final class ToolCallProtocol { conn.disconnect(); } + log.info(prefix(runId) + "event=tool-stream phase=initial-end sawToolCalls=" + sawToolCalls[0] + " toolCalls=" + toolCalls.size() + " assistantChars=" + assistantContent.length() + " durationMs=" + (System.currentTimeMillis() - start)); if (sawToolCalls[0]) { - handleToolCallsAndContinue(out, llm4Chat, originalBody, messages, assistantContent.toString(), toolCalls); + handleToolCallsAndContinue(out, llm4Chat, originalBody, messages, assistantContent.toString(), toolCalls, runId); } } @@ -259,6 +277,11 @@ public final class ToolCallProtocol { * @throws IOException when network I/O or JSON processing fails */ public static void handleToolCallsAndContinue(ServletOutputStream out, LLM.LLMModel llm4Chat, JSONObject originalBody, JSONArray messages, String assistantContent, Map<Integer, ToolCall> toolCalls) throws IOException { + handleToolCallsAndContinue(out, llm4Chat, originalBody, messages, assistantContent, toolCalls, null); + } + + public static void handleToolCallsAndContinue(ServletOutputStream out, LLM.LLMModel llm4Chat, JSONObject originalBody, JSONArray messages, String assistantContent, Map<Integer, ToolCall> toolCalls, final String runId) throws IOException { + final long start = System.currentTimeMillis(); try { // Work on a copy so caller-owned message arrays are not modified unexpectedly. JSONArray newMessages = new JSONArray(); @@ -272,9 +295,10 @@ public final class ToolCallProtocol { for (int round = 0; round < MAX_TOOL_ROUNDS; round++) { // Convert captured tool calls into assistant/tool messages and execute tools. - ToolRoundData roundData = appendAssistantAndToolMessages(newMessages, roundAssistantContent, roundToolCalls, toolCallCounters); + ToolRoundData roundData = appendAssistantAndToolMessages(newMessages, roundAssistantContent, roundToolCalls, toolCallCounters, runId, round); if (roundData == null || roundData.toolResults.length() == 0) { // No executable tool calls; terminate stream cleanly. + log.info(prefix(runId) + "event=tool-round phase=end round=" + round + " result=no-executable-tool-calls requested=" + (roundToolCalls == null ? 0 : roundToolCalls.size())); out.println("data: [DONE]"); out.flush(); return; @@ -286,9 +310,11 @@ public final class ToolCallProtocol { LLM.applyNoThinkingParameters(followup); } followup.put("messages", newMessages); + log.info(prefix(runId) + "event=tool-round phase=followup-call round=" + round + " toolResults=" + roundData.toolResults.length() + " messages=" + newMessages.length()); final HttpURLConnection followConn = openChatCompletionConnection(llm4Chat, followup); if (followConn.getResponseCode() != 200) { // Upstream error: close downstream stream instead of sending broken chunks. + log.warn(prefix(runId) + "event=tool-round phase=followup-return round=" + round + " result=upstream-error status=" + followConn.getResponseCode() + " reason=" + LogRedaction.redact(followConn.getResponseMessage())); out.println("data: [DONE]"); out.flush(); followConn.disconnect(); @@ -337,16 +363,22 @@ public final class ToolCallProtocol { // Ignore phantom rounds where stream signaled tool-calls but no concrete calls were parsed. if (nextToolCalls.isEmpty()) sawToolCalls[0] = false; // Final answer reached; caller already received forwarded stream lines. - if (!sawToolCalls[0]) return; + if (!sawToolCalls[0]) { + log.info(prefix(runId) + "event=tool-round phase=end round=" + round + " result=final-answer toolResults=" + roundData.toolResults.length() + " assistantChars=" + nextAssistantContent.length() + " durationMs=" + (System.currentTimeMillis() - start)); + return; + } // Continue with next round tool request that was found in follow-up stream. + log.info(prefix(runId) + "event=tool-round phase=continue round=" + round + " nextToolCalls=" + nextToolCalls.size() + " assistantChars=" + nextAssistantContent.length()); roundAssistantContent = nextAssistantContent.toString(); roundToolCalls = nextToolCalls; } // Safety fallback when round cap is hit. + log.warn(prefix(runId) + "event=tool-lifecycle phase=end result=max-rounds maxRounds=" + MAX_TOOL_ROUNDS + " durationMs=" + (System.currentTimeMillis() - start)); out.println("data: [DONE]"); out.flush(); } catch (JSONException e) { + log.warn(prefix(runId) + "event=tool-lifecycle phase=end result=failure errorClass=" + e.getClass().getName() + " reason=" + LogRedaction.redactMessage(e) + " durationMs=" + (System.currentTimeMillis() - start)); throw new IOException("JSON processing error in tool handling", e); } } @@ -442,6 +474,10 @@ public final class ToolCallProtocol { * results, or {@code null} when JSON assembly fails */ private static ToolRoundData appendAssistantAndToolMessages(JSONArray messages, String assistantContent, Map<Integer, ToolCall> toolCalls, Map<String, Integer> toolCallCounters) { + return appendAssistantAndToolMessages(messages, assistantContent, toolCalls, toolCallCounters, null, -1); + } + + private static ToolRoundData appendAssistantAndToolMessages(JSONArray messages, String assistantContent, Map<Integer, ToolCall> toolCalls, Map<String, Integer> toolCallCounters, final String runId, final int round) { try { // Keep original model call order stable by sorting call indices. List<Integer> indices = new ArrayList<>(toolCalls.keySet()); @@ -460,7 +496,10 @@ public final class ToolCallProtocol { final String toolName = call.name == null ? "" : call.name.trim(); final int maxCalls = net.yacy.ai.ToolProvider.maxCallsPerTurn(toolName); final int usedCalls = toolCallCounters.getOrDefault(toolName, Integer.valueOf(0)).intValue(); - if (usedCalls >= maxCalls) continue; + if (usedCalls >= maxCalls) { + log.warn(prefix(runId) + "event=tool-execution phase=skip round=" + round + " tool=" + LogRedaction.redact(toolName) + " reason=max-calls used=" + usedCalls + " max=" + maxCalls); + continue; + } // Assistant message representation of requested tool call. JSONObject toolCallJson = new JSONObject(true); @@ -473,7 +512,10 @@ public final class ToolCallProtocol { toolCallsArray.put(toolCallJson); // Execute tool locally and create "tool" role response message. + final long toolStart = System.currentTimeMillis(); + log.info(prefix(runId) + "event=tool-execution phase=start round=" + round + " tool=" + LogRedaction.redact(call.name) + " argumentChars=" + (call.arguments == null ? 0 : call.arguments.length())); String result = net.yacy.ai.ToolProvider.executeTool(call.name, call.arguments); + log.info(prefix(runId) + "event=tool-execution phase=end round=" + round + " tool=" + LogRedaction.redact(call.name) + " resultChars=" + (result == null ? 0 : result.length()) + " durationMs=" + (System.currentTimeMillis() - toolStart)); JSONObject toolMessage = new JSONObject(true); toolMessage.put("role", "tool"); toolMessage.put("tool_call_id", call.id); @@ -506,10 +548,15 @@ public final class ToolCallProtocol { return new ToolRoundData(toolCallsArray, toolResults); } catch (JSONException e) { + log.warn(prefix(runId) + "event=tool-execution phase=end round=" + round + " result=failure errorClass=" + e.getClass().getName() + " reason=" + LogRedaction.redactMessage(e)); return null; } } + private static String prefix(final String runId) { + return runId == null || runId.isEmpty() ? "" : "runId=" + runId + " "; + } + /** * Injects tool metadata into one outbound SSE line if the line contains JSON * data. diff --git a/source/net/yacy/cora/util/LogRedaction.java b/source/net/yacy/cora/util/LogRedaction.java new file mode 100644 index 000000000..9ccc28ccd --- /dev/null +++ b/source/net/yacy/cora/util/LogRedaction.java @@ -0,0 +1,54 @@ +/** + * LogRedaction + * Copyright 2026 by contributors to the YaCy project + * First released 27.06.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.cora.util; + +import java.util.regex.Pattern; + +/** + * Redacts common credential shapes before values are written to operator logs. + */ +public final class LogRedaction { + + private static final String REDACTED = "[REDACTED]"; + private static final Pattern AUTHORIZATION_HEADER = Pattern.compile("(?i)(authorization\\s*[:=]\\s*)(bearer\\s+|basic\\s+)?([^\\s,;]+)"); + private static final Pattern COOKIE_HEADER = Pattern.compile("(?i)(cookie\\s*[:=]\\s*)([^\\r\\n]+)"); + private static final Pattern SENSITIVE_ASSIGNMENT = Pattern.compile("(?i)\\b(api[_-]?key|access[_-]?token|auth[_-]?token|token|secret|password|passwd|pwd|login)\\b\\s*[:=]\\s*([^\\s,;&]+)"); + private static final Pattern SENSITIVE_QUERY = Pattern.compile("(?i)([?&](?:api[_-]?key|access[_-]?token|auth[_-]?token|token|secret|password|passwd|pwd|login)=)([^&#\\s]+)"); + private static final Pattern URL_USERINFO = Pattern.compile("(?i)(https?://)([^/@\\s:]+):([^/@\\s]+)@"); + + private LogRedaction() { + } + + public static String redact(final String value) { + if (value == null || value.isEmpty()) return value; + String redacted = URL_USERINFO.matcher(value).replaceAll("$1" + REDACTED + ":" + REDACTED + "@"); + redacted = AUTHORIZATION_HEADER.matcher(redacted).replaceAll("$1$2" + REDACTED); + redacted = COOKIE_HEADER.matcher(redacted).replaceAll("$1" + REDACTED); + redacted = SENSITIVE_QUERY.matcher(redacted).replaceAll("$1" + REDACTED); + redacted = SENSITIVE_ASSIGNMENT.matcher(redacted).replaceAll("$1=" + REDACTED); + return redacted; + } + + public static String redactMessage(final Throwable error) { + if (error == null) return ""; + return redact(error.getMessage()); + } +} diff --git a/source/net/yacy/htroot/AILab.java b/source/net/yacy/htroot/AILab.java index 3ba072dad..b6aefa3d7 100644 --- a/source/net/yacy/htroot/AILab.java +++ b/source/net/yacy/htroot/AILab.java @@ -12,6 +12,7 @@ import org.json.JSONException; import org.json.JSONObject; import org.json.JSONTokener; +import net.yacy.ai.LogReportService; import net.yacy.cora.protocol.RequestHeader; import net.yacy.search.Switchboard; import net.yacy.server.serverObjects; @@ -87,6 +88,7 @@ public class AILab { prop.put("ailab_rag_status", hasEngine && hasModel && (hasRagRole || ragVisited) ? "ready" : "pending"); prop.put("ailab_tools_status", hasToolsConfig ? "ready" : "pending"); prop.put("ailab_shield_status", hasShield ? "ready" : "pending"); + prop.put("ailab_logreports_status", LogReportService.hasConfiguredLogReportModel() ? "ready" : "pending"); return prop; } diff --git a/source/net/yacy/htroot/LLMSelection_p.java b/source/net/yacy/htroot/LLMSelection_p.java index 03229a9ce..960a3e985 100644 --- a/source/net/yacy/htroot/LLMSelection_p.java +++ b/source/net/yacy/htroot/LLMSelection_p.java @@ -76,6 +76,11 @@ public class LLMSelection_p { return service + "|" + hoststub + "|" + model; } + private static boolean optBooleanRole(final JSONObject row, final String canonicalKey, final String displayKey) { + if (row == null) return false; + return row.optBoolean(canonicalKey, row.optBoolean(displayKey, false)); + } + private static JSONObject normalizeProductionModelRow(final JSONObject row) throws JSONException { final JSONObject normalized = new JSONObject(true); normalized.put("service", row.optString("service", "OLLAMA")); @@ -91,6 +96,7 @@ public class LLMSelection_p { normalized.put("query", false); normalized.put("qapairs", false); normalized.put("tldr", row.optBoolean("tldr", false)); + normalized.put("logreport", optBooleanRole(row, "logreport", "log-report")); normalized.put("thinking", row.optBoolean("thinking", false)); normalized.put("tooling", row.optBoolean("tooling", false)); @@ -186,6 +192,7 @@ public class LLMSelection_p { prop.put("productionmodels_" + i + "_query", row.optBoolean("query", false)); prop.put("productionmodels_" + i + "_qapairs", row.optBoolean("qapairs", false)); prop.put("productionmodels_" + i + "_tldr", row.optBoolean("tldr", false)); + prop.put("productionmodels_" + i + "_logreport", row.optBoolean("logreport", false)); final String key = capabilityKey(row); JSONObject capabilityEntry = key.isEmpty() ? null : capabilities.optJSONObject(key); diff --git a/source/net/yacy/htroot/LogReports_p.java b/source/net/yacy/htroot/LogReports_p.java new file mode 100644 index 000000000..2a62e20d1 --- /dev/null +++ b/source/net/yacy/htroot/LogReports_p.java @@ -0,0 +1,94 @@ +/** + * LogReports_p + * Copyright 2026 by contributors to the YaCy project + * First released 27.06.2026 at https://yacy.net + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +package net.yacy.htroot; + +import java.io.File; +import java.time.format.DateTimeFormatter; +import java.util.List; + +import net.yacy.ai.LogReportService; +import net.yacy.ai.LogReportService.ReportEntry; +import net.yacy.cora.protocol.RequestHeader; +import net.yacy.search.Switchboard; +import net.yacy.server.serverObjects; +import net.yacy.server.serverSwitch; + +public class LogReports_p { + + public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { + final Switchboard sb = (Switchboard) env; + final serverObjects prop = new serverObjects(); + final LogReportService service = new LogReportService(sb); + final File reportDirectory = service.getReportDirectory(); + + final int configuredMax = sb.getConfigInt(LogReportService.CONFIG_FEED_MAX_ENTRIES, LogReportService.DEFAULT_FEED_MAX_ENTRIES); + final int requestedMax = post == null ? Math.min(20, configuredMax) : post.getInt("count", Math.min(20, configuredMax)); + final int maxEntries = Math.max(0, Math.min(requestedMax, configuredMax)); + + sb.setConfig("ui.LogReports_p.visited", "true"); + + prop.put("runReportResult", "0"); + prop.putHTML("runReportFile", ""); + prop.putHTML("runReportResult_runReportFile", ""); + prop.putNum("runReportResult_runReportDurationSeconds", 0); + if (post != null && post.containsKey("runReportNow")) { + final long start = System.currentTimeMillis(); + try { + final File reportFile = service.generateCurrentHourReportOverwrite(); + final long durationSeconds = Math.max(0L, (System.currentTimeMillis() - start) / 1000L); + prop.putNum("runReportResult_runReportDurationSeconds", durationSeconds); + if (reportFile == null) { + prop.put("runReportResult", LogReportService.hasConfiguredLogReportModel() ? "2" : "3"); + } else { + prop.put("runReportResult", "1"); + prop.putHTML("runReportFile", reportFile.getName()); + prop.putHTML("runReportResult_runReportFile", reportFile.getName()); + } + } catch (final Exception e) { + final long durationSeconds = Math.max(0L, (System.currentTimeMillis() - start) / 1000L); + prop.putNum("runReportResult_runReportDurationSeconds", durationSeconds); + prop.put("runReportResult", "4"); + final String message = e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage(); + prop.putHTML("runReportFile", message); + prop.putHTML("runReportResult_runReportFile", message); + } + } + + final List<ReportEntry> reports = service.discoverReports(maxEntries); + + prop.put("modelConfigured", LogReportService.hasConfiguredLogReportModel() ? "1" : "0"); + prop.putNum("count", maxEntries); + prop.putNum("maxcount", configuredMax); + prop.putHTML("reportdir", reportDirectory.getAbsolutePath()); + prop.put("reportdirExists", reportDirectory.isDirectory() ? "1" : "0"); + prop.put("reportdirMissing", reportDirectory.isDirectory() ? "0" : "1"); + prop.put("jsonFeed", "api/logreports.json?count=" + maxEntries); + prop.put("rssFeed", "api/logreports.rss?count=" + maxEntries); + + for (int i = 0; i < reports.size(); i++) { + final ReportEntry report = reports.get(i); + prop.putHTML("reports_" + i + "_filename", report.filename); + prop.putHTML("reports_" + i + "_title", report.title); + prop.putHTML("reports_" + i + "_published", report.published.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + prop.putHTML("reports_" + i + "_type", report.daily ? "daily" : "hourly"); + prop.put("reports_" + i + "_daily", report.daily ? "1" : "0"); + prop.putNum("reports_" + i + "_chars", report.content.length()); + prop.putHTML("reports_" + i + "_content", report.content); + } + prop.put("reports", reports.size()); + prop.putNum("reportCount", reports.size()); + prop.put("hasReports", reports.isEmpty() ? "0" : "1"); + prop.put("noReports", reports.isEmpty() ? "1" : "0"); + + return prop; + } +} diff --git a/source/net/yacy/htroot/ViewLog_p.java b/source/net/yacy/htroot/ViewLog_p.java index 3df1a45d0..06e51255a 100644 --- a/source/net/yacy/htroot/ViewLog_p.java +++ b/source/net/yacy/htroot/ViewLog_p.java @@ -33,17 +33,18 @@ package net.yacy.htroot;
import java.util.logging.Handler;
-import java.util.logging.Logger;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import java.util.regex.PatternSyntaxException;
-
-import net.yacy.cora.protocol.RequestHeader;
-import net.yacy.cora.util.ConcurrentLog;
-import net.yacy.kelondro.logging.GuiHandler;
-import net.yacy.kelondro.logging.LogalizerHandler;
-import net.yacy.server.serverObjects;
-import net.yacy.server.serverSwitch;
+import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +import net.yacy.cora.protocol.RequestHeader; +import net.yacy.cora.util.ConcurrentLog; +import net.yacy.cora.util.LogRedaction; +import net.yacy.kelondro.logging.GuiHandler; +import net.yacy.kelondro.logging.LogalizerHandler; +import net.yacy.server.serverObjects; +import net.yacy.server.serverSwitch; public class ViewLog_p {
@@ -58,20 +59,26 @@ public class ViewLog_p { * for the user to input regexes like ".*FOO.*" in the HTML
* interface.
*/
- String filter = ".*.*";
-
- if (post != null){
- reversed = (post.containsKey("mode") && "reversed".equals(post.get("mode")));
- json = post.containsKey("json");
+ String filter = ".*.*"; + String filterMode = "regex"; + + if (post != null){ + reversed = (post.containsKey("mode") && "reversed".equals(post.get("mode"))); + json = post.containsKey("json"); if(post.containsKey("lines")){
lines = post.getInt("lines", lines);
}
- if(post.containsKey("filter")){
- filter = post.get("filter");
- }
- }
+ if(post.containsKey("filter")){ + filter = post.get("filter"); + } + + if(post.containsKey("filterMode")){ + filterMode = post.get("filterMode"); + if (!"terms".equals(filterMode)) filterMode = "regex"; + } + } final Logger logger = Logger.getLogger("");
final Handler[] handlers = logger.getHandlers();
@@ -88,29 +95,44 @@ public class ViewLog_p { prop.put("submenu", displaySubmenu ? "1" : "0");
prop.put("reverseChecked", reversed ? "1" : "0");
- prop.put("lines", lines);
- prop.put("maxlines", maxlines);
- prop.putHTML("filter", filter);
-
- // trying to compile the regular expression filter expression
- Matcher filterMatcher = null;
- try {
- final Pattern filterPattern = Pattern.compile(filter,Pattern.MULTILINE);
- filterMatcher = filterPattern.matcher("");
- } catch (final PatternSyntaxException e) {
- ConcurrentLog.logException(e);
- }
-
- int level = 0;
- int lc = 0;
- for (final String logLine : log) {
- if (logLine == null) break;
- final String nextLogLine = logLine.trim();
-
- if (filterMatcher != null) {
- filterMatcher.reset(nextLogLine);
- if (!filterMatcher.find()) continue;
- }
+ prop.put("lines", lines); + prop.put("maxlines", maxlines); + prop.putHTML("filter", filter); + prop.put("filterMode", filterMode); + prop.put("filterModeRegex", "regex".equals(filterMode) ? "1" : "0"); + prop.put("filterModeTerms", "terms".equals(filterMode) ? "1" : "0"); + + Matcher filterMatcher = null; + String[] filterTerms = new String[0]; + boolean validFilter = true; + if ("terms".equals(filterMode)) { + filterTerms = parseFilterTerms(filter); + } else { + // trying to compile the regular expression filter expression + try { + final Pattern filterPattern = Pattern.compile(filter,Pattern.MULTILINE); + filterMatcher = filterPattern.matcher(""); + } catch (final PatternSyntaxException e) { + validFilter = false; + ConcurrentLog.warn("ViewLog", "Invalid log regex filter: " + LogRedaction.redactMessage(e)); + } + } + prop.put("filterError", validFilter ? "0" : "1"); + + int level = 0; + int lc = 0; + for (final String logLine : log) { + if (logLine == null) break; + final String nextLogLine = logLine.trim(); + + if ("terms".equals(filterMode)) { + if (!matchesAllTerms(nextLogLine, filterTerms)) continue; + } else if (validFilter && filterMatcher != null) { + filterMatcher.reset(nextLogLine); + if (!filterMatcher.find()) continue; + } else if (!validFilter) { + continue; + } if (nextLogLine.startsWith("E ")) {
level = 4;
@@ -136,7 +158,30 @@ public class ViewLog_p { }
prop.put("log", lc);
- // return rewrite properties
- return prop;
- }
-}
+ // return rewrite properties + return prop; + } + + private static String[] parseFilterTerms(final String filter) { + if (filter == null || filter.trim().isEmpty()) return new String[0]; + final String[] rawTerms = filter.split(","); + int count = 0; + for (int i = 0; i < rawTerms.length; i++) { + final String term = rawTerms[i].trim().toLowerCase(); + if (term.isEmpty()) continue; + rawTerms[count++] = term; + } + final String[] terms = new String[count]; + System.arraycopy(rawTerms, 0, terms, 0, count); + return terms; + } + + private static boolean matchesAllTerms(final String line, final String[] terms) { + if (terms == null || terms.length == 0) return true; + final String normalizedLine = line == null ? "" : line.toLowerCase(); + for (final String term : terms) { + if (!normalizedLine.contains(term)) return false; + } + return true; + } +} diff --git a/source/net/yacy/htroot/api/logreports.java b/source/net/yacy/htroot/api/logreports.java new file mode 100644 index 000000000..c13e293fd --- /dev/null +++ b/source/net/yacy/htroot/api/logreports.java @@ -0,0 +1,64 @@ +/** + * logreports + * Copyright 2026 by contributors to the YaCy project + * First released 26.06.2026 at https://yacy.net + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +package net.yacy.htroot.api; + +import java.time.format.DateTimeFormatter; +import java.util.List; + +import net.yacy.ai.LogReportService; +import net.yacy.ai.LogReportService.ReportEntry; +import net.yacy.cora.protocol.RequestHeader; +import net.yacy.search.Switchboard; +import net.yacy.server.serverObjects; +import net.yacy.server.serverSwitch; + +public class logreports { + + public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { + final serverObjects prop = new serverObjects(); + prop.put("authorized", "0"); + prop.put("reports", "0"); + prop.putXML("channel_title", "YaCy Log Reports"); + prop.putXML("channel_description", "Generated YaCy self-enhancement log reports"); + prop.put("channel_pubDate", ""); + + if (header == null || env == null) return prop; + final Switchboard sb = (Switchboard) env; + if (!sb.verifyAuthentication(header)) return prop; + prop.put("authorized", "1"); + + final int configuredMax = sb.getConfigInt(LogReportService.CONFIG_FEED_MAX_ENTRIES, LogReportService.DEFAULT_FEED_MAX_ENTRIES); + final int requestedMax = post == null ? configuredMax : post.getInt("count", configuredMax); + final int maxEntries = Math.max(0, Math.min(requestedMax, configuredMax)); + final List<ReportEntry> reports = new LogReportService(sb).discoverReports(maxEntries); + + for (int i = 0; i < reports.size(); i++) { + final ReportEntry report = reports.get(i); + prop.putJSON("reports_" + i + "_filename", report.filename); + prop.putJSON("reports_" + i + "_title", report.title); + prop.put("reports_" + i + "_published", report.published.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + prop.putJSON("reports_" + i + "_content", report.content); + prop.put("reports_" + i + "_daily", report.daily ? "1" : "0"); + prop.put("reports_" + i + "_comma", i + 1 < reports.size() ? "1" : "0"); + + prop.putXML("reports_" + i + "_title-rss", report.title); + prop.putXML("reports_" + i + "_description-rss", report.content); + prop.put("reports_" + i + "_pubDate-rss", report.published.format(DateTimeFormatter.RFC_1123_DATE_TIME)); + prop.putXML("reports_" + i + "_guid-rss", report.filename); + } + prop.put("reports", reports.size()); + if (!reports.isEmpty()) { + prop.put("channel_pubDate", reports.get(0).published.format(DateTimeFormatter.RFC_1123_DATE_TIME)); + } + return prop; + } +} diff --git a/source/net/yacy/http/servlets/RAGProxyServlet.java b/source/net/yacy/http/servlets/RAGProxyServlet.java index 5875dc4fd..c41789e9c 100644 --- a/source/net/yacy/http/servlets/RAGProxyServlet.java +++ b/source/net/yacy/http/servlets/RAGProxyServlet.java @@ -28,6 +28,7 @@ import java.util.ArrayList; import java.util.Base64; import java.util.Deque; import java.util.List; +import java.util.UUID; import java.util.concurrent.ConcurrentLinkedDeque; import javax.servlet.ServletException; @@ -48,6 +49,7 @@ import net.yacy.ai.RAGAugmentor; import net.yacy.ai.ToolCallProtocol; import net.yacy.cora.protocol.Domains; import net.yacy.cora.util.ConcurrentLog; +import net.yacy.cora.util.LogRedaction; import net.yacy.search.Switchboard; /** @@ -83,6 +85,8 @@ public class RAGProxyServlet extends HttpServlet { @Override public void service(ServletRequest request, ServletResponse response) throws IOException, ServletException { + final String runId = UUID.randomUUID().toString(); + final long requestStart = System.currentTimeMillis(); response.setContentType("application/json;charset=utf-8"); HttpServletResponse hresponse = (HttpServletResponse) response; @@ -96,15 +100,18 @@ public class RAGProxyServlet extends HttpServlet { final Switchboard sb = Switchboard.getSwitchboard(); final String clientIP = hrequest.getRemoteAddr(); final boolean localhostAccess = Domains.isLocalhost(clientIP); + ConcurrentLog.info("RAGProxy", "runId=" + runId + " event=rag-request phase=start method=" + hrequest.getMethod() + " localhost=" + localhostAccess); if (!localhostAccess) { // obey the allow-nonlocalhost shield setting final boolean allowNonLocal = sb.getConfigBool("ai.shield.allow-nonlocalhost", false); if (!allowNonLocal) { + ConcurrentLog.warn("RAGProxy", "runId=" + runId + " event=rag-request phase=reject reason=nonlocalhost-blocked durationMs=" + elapsed(requestStart)); hresponse.sendError(HttpServletResponse.SC_FORBIDDEN); return; } } if (isRateLimited(sb, clientIP, localhostAccess)) { + ConcurrentLog.warn("RAGProxy", "runId=" + runId + " event=rag-request phase=reject reason=rate-limited localhost=" + localhostAccess + " durationMs=" + elapsed(requestStart)); hresponse.sendError(429, "Too Many Requests"); // standard status for rate limits return; } @@ -114,11 +121,13 @@ public class RAGProxyServlet extends HttpServlet { if (reqMethod == Method.OTHER) { // required to handle CORS hresponse.setStatus(HttpServletResponse.SC_OK); + ConcurrentLog.info("RAGProxy", "runId=" + runId + " event=rag-request phase=end result=options durationMs=" + elapsed(requestStart)); return; } // We expect a POST request if (reqMethod != Method.POST) { + ConcurrentLog.warn("RAGProxy", "runId=" + runId + " event=rag-request phase=reject reason=method-not-allowed method=" + hrequest.getMethod() + " durationMs=" + elapsed(requestStart)); hresponse.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED); return; } @@ -135,6 +144,7 @@ public class RAGProxyServlet extends HttpServlet { bodyBuilder.append(line); } String body = bodyBuilder.toString(); + ConcurrentLog.info("RAGProxy", "runId=" + runId + " event=rag-request phase=body-read bodyChars=" + body.length()); JSONObject bodyObject; try { // get system message and user prompt @@ -148,8 +158,13 @@ public class RAGProxyServlet extends HttpServlet { // resolve true model name from configuration LLM.LLMUsage usage = LLM.LLMUsage.chat; try {usage = LLM.LLMUsage.valueOf(model);} catch (IllegalArgumentException e) {} - LLM.LLMModel llm4Chat = LLM.llmFromUsage(usage); - LLM.LLMModel llm4tldr = LLM.llmFromUsage(LLM.LLMUsage.tldr); + LLM.LLMModel llm4Chat = LLM.llmFromUsage(usage, runId, "rag-chat"); + LLM.LLMModel llm4tldr = LLM.llmFromUsage(LLM.LLMUsage.tldr, runId, "rag-query-generator"); + if (llm4Chat == null) { + ConcurrentLog.warn("RAGProxy", "runId=" + runId + " event=rag-request phase=reject reason=no-chat-model usage=" + usage + " durationMs=" + elapsed(requestStart)); + hresponse.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE, "No chat model configured"); + return; + } bodyObject.put("model", llm4Chat.model); // replace the model with the decoded model name // get messages and prepare user message attachments @@ -182,7 +197,7 @@ public class RAGProxyServlet extends HttpServlet { user = userObject.getContentText(); // this is the latest user prompt ragMode = userObject.getSearchMode(); } - ConcurrentLog.info("RAGProxy", "ragMode=" + ragMode + " userChars=" + (user == null ? 0 : user.length())); + ConcurrentLog.info("RAGProxy", "runId=" + runId + " event=rag-request phase=messages messages=" + messages.length() + " lastUserIndex=" + lastUserIndex + " ragMode=" + ragMode + " userChars=" + (user == null ? 0 : user.length())); //List<DataURL> data_urls = userObject.getContentAttachments(); // this list is a copy of the content data_urls // RAG @@ -194,17 +209,26 @@ public class RAGProxyServlet extends HttpServlet { final long queryStart = System.currentTimeMillis(); if (countWords(user) <= DIRECT_SEARCH_WORD_LIMIT) { searchResultQuery = user; + ConcurrentLog.info("RAGProxy", "runId=" + runId + " event=rag-query phase=select source=direct userWords=" + countWords(user)); } else { - searchResultQuery = RAGAugmentor.searchWordsForPrompt(llm4tldr.llm, llm4tldr.model, user, queryPrefix); // might return null in case any error occurred - if (searchResultQuery == null || searchResultQuery.length() == 0) searchResultQuery = user; // in case there is an error we simply search with the prompt + if (llm4tldr == null) { + searchResultQuery = user; + ConcurrentLog.warn("RAGProxy", "runId=" + runId + " event=rag-query phase=fallback reason=no-tldr-model userWords=" + countWords(user)); + } else { + searchResultQuery = RAGAugmentor.searchWordsForPrompt(llm4tldr.llm, llm4tldr.model, user, queryPrefix, runId); // might return null in case any error occurred + if (searchResultQuery == null || searchResultQuery.length() == 0) { + searchResultQuery = user; // in case there is an error we simply search with the prompt + ConcurrentLog.warn("RAGProxy", "runId=" + runId + " event=rag-query phase=fallback reason=query-generation-empty userWords=" + countWords(user)); + } + } } final long queryElapsed = System.currentTimeMillis() - queryStart; final long searchStart = System.currentTimeMillis(); - searchResultMarkdown = RAGAugmentor.searchResultsAsMarkdown(searchResultQuery, 10, "global".equals(ragMode)); + searchResultMarkdown = RAGAugmentor.searchResultsAsMarkdown(searchResultQuery, 10, "global".equals(ragMode), runId); final long searchElapsed = System.currentTimeMillis() - searchStart; ConcurrentLog.info( "RAGProxy", - "searchQuery=\"" + searchResultQuery + "\" queryMs=" + queryElapsed + " searchMs=" + searchElapsed + + "runId=" + runId + " event=rag-retrieval phase=end ragMode=" + ragMode + " queryChars=" + searchResultQuery.length() + " queryWords=" + countWords(searchResultQuery) + " queryMs=" + queryElapsed + " searchMs=" + searchElapsed + " markdownChars=" + searchResultMarkdown.length()); user += userPrefix; user += searchResultMarkdown; @@ -217,14 +241,20 @@ public class RAGProxyServlet extends HttpServlet { } // ToolCallProtocol owns request preparation, initial stream handling and follow-up tool rounds. - final int status = ToolCallProtocol.proxyToolLifecycle(out, llm4Chat, bodyObject, messages, initialMetadata); + final int status = ToolCallProtocol.proxyToolLifecycle(out, llm4Chat, bodyObject, messages, initialMetadata, runId); hresponse.setStatus(status); out.close(); // close this here to end transmission + ConcurrentLog.info("RAGProxy", "runId=" + runId + " event=rag-request phase=end result=success status=" + status + " durationMs=" + elapsed(requestStart)); } catch (JSONException e) { + ConcurrentLog.warn("RAGProxy", "runId=" + runId + " event=rag-request phase=end result=failure errorClass=" + e.getClass().getName() + " reason=" + LogRedaction.redactMessage(e) + " durationMs=" + elapsed(requestStart)); throw new IOException(e.getMessage()); } } + private static long elapsed(final long start) { + return System.currentTimeMillis() - start; + } + private static int countWords(final String text) { if (text == null) return 0; final String trimmed = text.trim(); diff --git a/source/net/yacy/kelondro/logging/GuiHandler.java b/source/net/yacy/kelondro/logging/GuiHandler.java index c1366e31d..e9c1684b0 100644 --- a/source/net/yacy/kelondro/logging/GuiHandler.java +++ b/source/net/yacy/kelondro/logging/GuiHandler.java @@ -43,7 +43,7 @@ import net.yacy.kelondro.util.MemoryControl; public class GuiHandler extends Handler {
- private final static int DEFAULT_SIZE = 10000; // don't make this too big, it eats up a lot of memory!
+ private final static int DEFAULT_SIZE = 100000; // don't make this too big, it eats up a lot of memory! private static int size = DEFAULT_SIZE;
private static String buffer[];
private static int start, count;
diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index 1425b7976..1a7166eb4 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -42,9 +42,10 @@ import java.util.LinkedHashMap; import java.util.List;
import java.util.Locale;
import java.util.Map;
-import java.util.Properties;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
+import java.util.Properties; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; import org.apache.http.Header;
import org.apache.http.HttpStatus;
@@ -60,9 +61,10 @@ import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.ConnectionInfo;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.TimeoutRequest;
-import net.yacy.cora.protocol.http.HTTPClient;
-import net.yacy.cora.util.ConcurrentLog;
-import net.yacy.data.TransactionManager;
+import net.yacy.cora.protocol.http.HTTPClient; +import net.yacy.cora.util.ConcurrentLog; +import net.yacy.ai.LogReportService; +import net.yacy.data.TransactionManager; import net.yacy.data.Translator;
import net.yacy.gui.YaCyApp;
import net.yacy.gui.framework.Browser;
@@ -290,13 +292,14 @@ public final class yacy { final int deleteOldDownloadsAfterDays = (int) sb.getConfigLong("update.deleteOld", 30);
yacyRelease.deleteOldDownloads(sb.releasePath, deleteOldDownloadsAfterDays );
- // start main threads
- final int port = sb.getLocalPort();
- final String host = sb.getLocalHost();
- try {
- // start http server
- YaCyHttpServer httpServer;
- httpServer = new YaCyHttpServer(port, host);
+ // start main threads + final int port = sb.getLocalPort(); + final String host = sb.getLocalHost(); + ScheduledExecutorService logReportScheduler = null; + try { + // start http server + YaCyHttpServer httpServer; + httpServer = new YaCyHttpServer(port, host); httpServer.startupServer();
sb.setHttpServer(httpServer);
// TODO: this has no effect on Jetty (but needed to reflect configured value and limit is still used)
@@ -371,11 +374,13 @@ public final class yacy { }
}
// initialize number formatter with this locale
- if (!lang.equals("browser")) // "default" is handled by .setLocale()
- Formatter.setLocale(lang);
-
- // registering shutdown hook
- ConcurrentLog.config("STARTUP", "Registering Shutdown Hook");
+ if (!lang.equals("browser")) // "default" is handled by .setLocale() + Formatter.setLocale(lang); + + logReportScheduler = LogReportService.startScheduler(sb); + + // registering shutdown hook + ConcurrentLog.config("STARTUP", "Registering Shutdown Hook"); final Runtime run = Runtime.getRuntime();
run.addShutdownHook(new shutdownHookThread(sb, shutdownSemaphore));
@@ -394,16 +399,18 @@ public final class yacy { } catch (final Exception e) {
ConcurrentLog.severe("MAIN CONTROL LOOP", "PANIC: " + e.getMessage(),e);
}
- // shut down
- ConcurrentLog.config("SHUTDOWN", "caught termination signal");
- httpServer.stop();
-
- ConcurrentLog.config("SHUTDOWN", "server has terminated");
- sb.close();
- } catch (final Exception e) {
- ConcurrentLog.severe("STARTUP", "Unexpected Error: " + e.getClass().getName(),e);
- //System.exit(1);
- }
+ // shut down + ConcurrentLog.config("SHUTDOWN", "caught termination signal"); + LogReportService.stopScheduler(logReportScheduler); + httpServer.stop(); +
+ ConcurrentLog.config("SHUTDOWN", "server has terminated"); + sb.close(); + } catch (final Exception e) { + LogReportService.stopScheduler(logReportScheduler); + ConcurrentLog.severe("STARTUP", "Unexpected Error: " + e.getClass().getName(),e); + //System.exit(1); + } if (lock != null && lock.isValid()) lock.release();
if (channel != null && channel.isOpen()) channel.close();
} catch (final Exception ee) {
|
