From 9f540de0674378ce610594da568925050b6139ec Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Sat, 4 Jul 2026 13:52:47 +0200 Subject: Enhanced Log Reports --- source/net/yacy/ai/LogReportService.java | 57 +++++++++++++++----------------- source/net/yacy/htroot/LogReports_p.java | 47 ++++++++++++++++++-------- 2 files changed, 61 insertions(+), 43 deletions(-) (limited to 'source') diff --git a/source/net/yacy/ai/LogReportService.java b/source/net/yacy/ai/LogReportService.java index ddee5db9c..81b7eb358 100644 --- a/source/net/yacy/ai/LogReportService.java +++ b/source/net/yacy/ai/LogReportService.java @@ -63,7 +63,6 @@ public class LogReportService { public static final String CONFIG_PERIOD_MINUTES = "ai.logreport.period_minutes"; public static final String CONFIG_MAX_TOKENS = "ai.logreport.max_tokens"; public static final String CONFIG_DAILY_COMPRESSION_ENABLED = "ai.logreport.daily_compression.enabled"; - public static final String CONFIG_DELETE_HOURLY_AFTER_DAILY = "ai.logreport.daily_compression.delete_hourly"; public static final String CONFIG_FEED_MAX_ENTRIES = "ai.logreport.feed.max_entries"; public static final String DEFAULT_REPORT_DIR = "DATA/REPORTS/log"; public static final int DEFAULT_MAX_BUCKET_LINES = 100000; @@ -348,21 +347,24 @@ public class LogReportService { return Collections.emptyList(); } - final Map> completeDays = completeHourlyReportSets(reportDirectory); - log.info("runId=" + runId + " event=daily-reports phase=scan completeDays=" + completeDays.size() + " path=" + reportDirectory.getAbsolutePath()); - if (completeDays.isEmpty()) { - log.info("runId=" + runId + " event=daily-reports phase=skip reason=no-complete-day durationMs=" + elapsed(start)); + final Map> pastDays = pastDayHourlyReportSets(reportDirectory); + log.info("runId=" + runId + " event=daily-reports phase=scan pastDays=" + pastDays.size() + " path=" + reportDirectory.getAbsolutePath()); + if (pastDays.isEmpty()) { + log.info("runId=" + runId + " event=daily-reports phase=skip reason=no-past-day-hourly-reports durationMs=" + elapsed(start)); return Collections.emptyList(); } final List writtenReports = new ArrayList<>(); int skippedExisting = 0; int failedReports = 0; - for (final Map.Entry> day : completeDays.entrySet()) { + for (final Map.Entry> day : pastDays.entrySet()) { final File dailyReportFile = new File(reportDirectory, dailyReportFilename(day.getKey())); if (dailyReportFile.exists()) { + // the day was already compressed earlier; only clean up leftover hourly + // reports (e.g. written by a manual "run report now" after the compression) skippedExisting++; - log.info("runId=" + runId + " event=daily-report phase=skip day=" + day.getKey() + " reason=report-exists file=" + dailyReportFile.getAbsolutePath()); + deleteHourlyReports(day.getValue()); + log.info("runId=" + runId + " event=daily-report phase=skip day=" + day.getKey() + " reason=report-exists deletedLeftoverHourly=" + day.getValue().size() + " file=" + dailyReportFile.getAbsolutePath()); continue; } try { @@ -379,16 +381,15 @@ public class LogReportService { writeReport(dailyReportFile, dailyReportDocument(day.getKey(), day.getValue().size(), report)); writtenReports.add(dailyReportFile); log.info("runId=" + runId + " event=daily-report phase=write result=success day=" + day.getKey() + " sourceReports=" + day.getValue().size() + " outputChars=" + report.length() + " file=" + dailyReportFile.getAbsolutePath()); - if (this.sb.getConfigBool(CONFIG_DELETE_HOURLY_AFTER_DAILY, false)) { - deleteHourlyReports(day.getValue()); - log.info("runId=" + runId + " event=daily-report phase=delete-hourly day=" + day.getKey() + " sourceReports=" + day.getValue().size()); - } + // the hourly reports are now bundled into the daily report and must be deleted + deleteHourlyReports(day.getValue()); + log.info("runId=" + runId + " event=daily-report phase=delete-hourly day=" + day.getKey() + " sourceReports=" + day.getValue().size()); } catch (final IOException e) { failedReports++; log.warn("runId=" + runId + " event=daily-report phase=write result=failure day=" + day.getKey() + " file=" + LogRedaction.redact(dailyReportFile.getAbsolutePath()) + " errorClass=" + e.getClass().getName() + " reason=" + LogRedaction.redactMessage(e)); } } - log.info("runId=" + runId + " event=daily-reports phase=end result=success completeDays=" + completeDays.size() + " written=" + writtenReports.size() + " skippedExisting=" + skippedExisting + " failed=" + failedReports + " durationMs=" + elapsed(start)); + log.info("runId=" + runId + " event=daily-reports phase=end result=success pastDays=" + pastDays.size() + " written=" + writtenReports.size() + " skippedExisting=" + skippedExisting + " failed=" + failedReports + " durationMs=" + elapsed(start)); return writtenReports; } @@ -573,16 +574,25 @@ public class LogReportService { return text.toString(); } - private static Map> completeHourlyReportSets(final File reportDirectory) { + /** + * Group all hourly reports of past days (strictly before today) by day, ordered + * chronologically within each day. Every past day that still has hourly reports is + * a compression candidate, no matter how many hours were actually reported - the + * peer may have been offline for parts of the day. Only the current day is excluded + * because its hourly reports are still accumulating. + */ + private static Map> pastDayHourlyReportSets(final File reportDirectory) { final File[] files = reportDirectory.listFiles(); if (files == null || files.length == 0) return Collections.emptyMap(); + final LocalDate today = LocalDate.now(); final Map> candidates = new TreeMap<>(); for (final File file : files) { if (!file.isFile()) continue; final LocalDateTime hour = parseHourlyReportFilename(file.getName()); if (hour == null) continue; final LocalDate day = hour.toLocalDate(); + if (!day.isBefore(today)) continue; List dayFiles = candidates.get(day); if (dayFiles == null) { dayFiles = new ArrayList<>(); @@ -590,14 +600,11 @@ public class LogReportService { } dayFiles.add(file); } - - final Map> complete = new TreeMap<>(); - for (final Map.Entry> entry : candidates.entrySet()) { - if (entry.getValue().size() != 24) continue; - final List ordered = hourlyFilesForDay(reportDirectory, entry.getKey()); - if (ordered.size() == 24) complete.put(entry.getKey(), ordered); + // the hourly filename pattern sorts chronologically + for (final List dayFiles : candidates.values()) { + dayFiles.sort((a, b) -> a.getName().compareTo(b.getName())); } - return complete; + return candidates; } private static LocalDateTime parseHourlyReportFilename(final String filename) { @@ -644,16 +651,6 @@ public class LogReportService { return new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8); } - private static List hourlyFilesForDay(final File reportDirectory, final LocalDate day) { - final List files = new ArrayList<>(24); - for (int hour = 0; hour < 24; hour++) { - final File file = new File(reportDirectory, hourlyReportFilename(day.atTime(hour, 0))); - if (!file.isFile()) return Collections.emptyList(); - files.add(file); - } - return files; - } - private static String dailyPrompt(final LocalDate day, final List hourlyReports) throws IOException { final StringBuilder prompt = new StringBuilder(4096); prompt.append("Create one consolidated YaCy self-enhancement report for ") diff --git a/source/net/yacy/htroot/LogReports_p.java b/source/net/yacy/htroot/LogReports_p.java index 2a62e20d1..5ef559611 100644 --- a/source/net/yacy/htroot/LogReports_p.java +++ b/source/net/yacy/htroot/LogReports_p.java @@ -30,9 +30,8 @@ public class LogReports_p { final LogReportService service = new LogReportService(sb); final File reportDirectory = service.getReportDirectory(); - final int configuredMax = sb.getConfigInt(LogReportService.CONFIG_FEED_MAX_ENTRIES, LogReportService.DEFAULT_FEED_MAX_ENTRIES); - final int requestedMax = post == null ? Math.min(20, configuredMax) : post.getInt("count", Math.min(20, configuredMax)); - final int maxEntries = Math.max(0, Math.min(requestedMax, configuredMax)); + // fixed display limit; the navigation column shows at most this many reports + final int maxEntries = 100; sb.setConfig("ui.LogReports_p.visited", "true"); @@ -66,25 +65,47 @@ public class LogReports_p { final List reports = service.discoverReports(maxEntries); prop.put("modelConfigured", LogReportService.hasConfiguredLogReportModel() ? "1" : "0"); - prop.putNum("count", maxEntries); - prop.putNum("maxcount", configuredMax); prop.putHTML("reportdir", reportDirectory.getAbsolutePath()); prop.put("reportdirExists", reportDirectory.isDirectory() ? "1" : "0"); prop.put("reportdirMissing", reportDirectory.isDirectory() ? "0" : "1"); prop.put("jsonFeed", "api/logreports.json?count=" + maxEntries); prop.put("rssFeed", "api/logreports.rss?count=" + maxEntries); + // the report to show in the main view is selected by filename from the navigation + // column; matching against the discovered list only prevents path traversal. + // default is the newest report (first entry, list is sorted newest first) + final String requestedReport = post == null ? "" : post.get("report", ""); + ReportEntry selected = null; + for (final ReportEntry report : reports) { + if (report.filename.equals(requestedReport)) { + selected = report; + break; + } + } + if (selected == null && !reports.isEmpty()) selected = reports.get(0); + + // navigation list and selected report live inside the #(hasReports)# alternative, + // therefore all template keys carry the "hasReports_" prefix for (int i = 0; i < reports.size(); i++) { final ReportEntry report = reports.get(i); - prop.putHTML("reports_" + i + "_filename", report.filename); - prop.putHTML("reports_" + i + "_title", report.title); - prop.putHTML("reports_" + i + "_published", report.published.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); - prop.putHTML("reports_" + i + "_type", report.daily ? "daily" : "hourly"); - prop.put("reports_" + i + "_daily", report.daily ? "1" : "0"); - prop.putNum("reports_" + i + "_chars", report.content.length()); - prop.putHTML("reports_" + i + "_content", report.content); + final String p = "hasReports_reports_" + i + "_"; + prop.putHTML(p + "filename", report.filename); + prop.putHTML(p + "title", report.title); + prop.putHTML(p + "date", report.published.format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm"))); + prop.putHTML(p + "type", report.daily ? "daily" : "hourly"); + prop.put(p + "selected", report == selected ? "1" : "0"); + } + prop.put("hasReports_reports", reports.size()); + + prop.put("hasReports_selectedReport", selected == null ? "0" : "1"); + if (selected != null) { + prop.putHTML("hasReports_selectedReport_filename", selected.filename); + prop.putHTML("hasReports_selectedReport_title", selected.title); + prop.putHTML("hasReports_selectedReport_published", selected.published.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + prop.putHTML("hasReports_selectedReport_type", selected.daily ? "daily" : "hourly"); + prop.putNum("hasReports_selectedReport_chars", selected.content.length()); + prop.putHTML("hasReports_selectedReport_content", selected.content); } - prop.put("reports", reports.size()); prop.putNum("reportCount", reports.size()); prop.put("hasReports", reports.isEmpty() ? "0" : "1"); prop.put("noReports", reports.isEmpty() ? "1" : "0"); -- cgit v1.2.3