From 6f505d1f10ffb702ac15dabc16b8cf58bc0053ea Mon Sep 17 00:00:00 2001 From: Michael Peter Christen Date: Sat, 28 Mar 2026 11:29:58 +0100 Subject: Align RAG retrieval with web search and disable local post-ranking --- source/net/yacy/ai/LLM.java | 5 + source/net/yacy/ai/RAGAugmentor.java | 7 +- source/net/yacy/search/query/QueryParams.java | 67 ++++++++++--- source/net/yacy/search/query/SearchEvent.java | 38 ++++++-- source/net/yacy/search/snippet/TextSnippet.java | 120 +++++++++++++++--------- 5 files changed, 167 insertions(+), 70 deletions(-) diff --git a/source/net/yacy/ai/LLM.java b/source/net/yacy/ai/LLM.java index df0e82242..8e9530e79 100644 --- a/source/net/yacy/ai/LLM.java +++ b/source/net/yacy/ai/LLM.java @@ -272,6 +272,11 @@ public class LLM { data.put("messages", context); data.put("stop", new JSONArray(STOPTOKENS)); data.put("stream", false); + + if (model.toLowerCase().contains("qwen3.5")) { // we don't think + data.put("reasoning_effort", "none"); + data.put("enable_thinking", false); + } if (schema != null) { System.out.println(schema.toString()); diff --git a/source/net/yacy/ai/RAGAugmentor.java b/source/net/yacy/ai/RAGAugmentor.java index c9b7215a9..1b5723c3f 100644 --- a/source/net/yacy/ai/RAGAugmentor.java +++ b/source/net/yacy/ai/RAGAugmentor.java @@ -189,11 +189,9 @@ public final class RAGAugmentor { String title = node.title(); String text = null; if (includeSnippet) { + TextSnippet snippet = node.textSnippet(); + if (snippet != null && snippet.exists() && !snippet.getErrorCode().fail()) text = snippet.getLineRaw(); if (text == null || text.isEmpty()) text = node.snippet(); - if (text == null || text.isEmpty()) { - TextSnippet snippet = node.textSnippet(); - if (snippet != null && snippet.exists() && !snippet.getErrorCode().fail()) text = snippet.getLineRaw(); - } if (text == null || text.isEmpty()) text = firstFieldString(node.getFieldValue(CollectionSchema.description_txt.getSolrFieldName())); if (text == null || text.isEmpty()) text = firstFieldString(node.getFieldValue(CollectionSchema.text_t.getSolrFieldName())); result.put("text", text == null ? "" : text.trim()); @@ -264,6 +262,7 @@ public final class RAGAugmentor { theQuery.setDateFacetMaxCount(sb.getConfigInt( SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT, QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT)); + theQuery.setSnippetFetchFullText(true); theQuery.getQueryGoal().filterOut(Switchboard.blueList); return theQuery; } diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.java index e3bd008bf..4686f7cf5 100644 --- a/source/net/yacy/search/query/QueryParams.java +++ b/source/net/yacy/search/query/QueryParams.java @@ -188,7 +188,9 @@ public final class QueryParams { public final int maxDistance; public final Bitfield constraint; public final boolean allofconstraint; - protected CacheStrategy snippetCacheStrategy; + protected CacheStrategy snippetCacheStrategy; + private boolean snippetFetchFullText; + private boolean localSolrOnly; public final RankingProfile ranking; private final Segment indexSegment; public final String clienthost; // this is the client host that starts the query, not a site operator @@ -298,9 +300,11 @@ public final class QueryParams { this.zonecode = domainzone; this.constraint = constraint; this.allofconstraint = allofconstraint; - this.siteexcludes = siteexcludes != null && siteexcludes.isEmpty() ? null: siteexcludes; - this.snippetCacheStrategy = snippetCacheStrategy; - this.clienthost = host; + this.siteexcludes = siteexcludes != null && siteexcludes.isEmpty() ? null: siteexcludes; + this.snippetCacheStrategy = snippetCacheStrategy; + this.snippetFetchFullText = false; + this.localSolrOnly = false; + this.clienthost = host; this.remotepeer = null; this.starttime = System.currentTimeMillis(); this.maxtime = 10000; @@ -658,11 +662,11 @@ public final class QueryParams { if (bf.length() > 0) params.setParam("boost", bf); // a boost function extension, see http://wiki.apache.org/solr/ExtendedDisMax#bf_.28Boost_Function.2C_additive.29 // set highlighting query attributes - if (this.contentdom == Classification.ContentDomain.TEXT || this.contentdom == Classification.ContentDomain.ALL) { - params.setHighlight(true); - params.setHighlightFragsize(SearchEvent.SNIPPET_MAX_LENGTH); - //params.setHighlightRequireFieldMatch(); - params.setHighlightSimplePost(""); + if (!this.snippetFetchFullText && (this.contentdom == Classification.ContentDomain.TEXT || this.contentdom == Classification.ContentDomain.ALL)) { + params.setHighlight(true); + params.setHighlightFragsize(SearchEvent.SNIPPET_MAX_LENGTH); + //params.setHighlightRequireFieldMatch(); + params.setHighlightSimplePost(""); params.setHighlightSimplePre(""); params.setHighlightSnippets(5); for (final CollectionSchema field: SOLR_SNIPPET_FIELDS) { @@ -807,9 +811,24 @@ public final class QueryParams { } else { params.setFacet(false); } - params.setFields("*", "score"); // we need the score for post-ranking - return params; - } + params.setFields(buildResultFieldList(this.snippetFetchFullText)); + return params; + } + + private String[] buildResultFieldList(final boolean includeTextField) { + final ArrayList fields = new ArrayList<>(); + for (final CollectionSchema field : CollectionSchema.values()) { + if (!this.solrSchema.contains(field) || !field.isStored()) { + continue; + } + if (!includeTextField && field == CollectionSchema.text_t) { + continue; + } + fields.add(field.getSolrFieldName()); + } + fields.add("score"); + return fields.toArray(new String[fields.size()]); + } long year = 1000L * 60L * 60L * 24L * 365L; @@ -942,7 +961,7 @@ public final class QueryParams { private volatile String idCacheAnon = null, idCache = null; final static private char asterisk = '*'; - public String id(final boolean anonymized) { + public String id(final boolean anonymized) { if (anonymized) { if (this.idCacheAnon != null) return this.idCacheAnon; } else { @@ -989,7 +1008,9 @@ public final class QueryParams { context.append(this.tld).append(asterisk); context.append(this.inlink).append(asterisk); context.append(this.lat).append(asterisk).append(this.lon).append(asterisk).append(this.radius).append(asterisk); - context.append(this.snippetCacheStrategy == null ? "null" : this.snippetCacheStrategy.name()); + context.append(this.snippetCacheStrategy == null ? "null" : this.snippetCacheStrategy.name()).append(asterisk); + context.append(this.snippetFetchFullText).append(asterisk); + context.append(this.localSolrOnly); // Note : this.maxSuggestions search parameter do not need to be part of this id, as it has no impact on results themselves @@ -1001,7 +1022,23 @@ public final class QueryParams { } return result; } - } + } + + public boolean isSnippetFetchFullText() { + return this.snippetFetchFullText; + } + + public void setSnippetFetchFullText(final boolean snippetFetchFullText) { + this.snippetFetchFullText = snippetFetchFullText; + } + + public boolean isLocalSolrOnly() { + return this.localSolrOnly; + } + + public void setLocalSolrOnly(final boolean localSolrOnly) { + this.localSolrOnly = localSolrOnly; + } /** * Build a search query URL from the given parameters. diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java index b7d7e94c2..9ad44a347 100644 --- a/source/net/yacy/search/query/SearchEvent.java +++ b/source/net/yacy/search/query/SearchEvent.java @@ -227,6 +227,8 @@ public final class SearchEvent implements ScoreMapUpdatesListener { /** if this is true, then every entry in result List is polled immediately to prevent a re-ranking in the resultList. This is usefull if there is only one index source. */ private final boolean pollImmediately; + /** Disable post-ranking when operating in explicit local-search mode. */ + private final boolean disablePostRanking; public final boolean excludeintext_image; // the following values are filled during the search process as statistics for the search @@ -430,6 +432,7 @@ public final class SearchEvent implements ScoreMapUpdatesListener { this.feedersTerminated = new AtomicInteger(0); this.snippetFetchAlive = new AtomicInteger(0); this.addRunning = true; + this.disablePostRanking = this.query.isLocal(); this.receivedRemoteReferences = new AtomicInteger(0); this.order = new ReferenceOrder(this.query.ranking, this.query.targetlang); this.urlhashes = new RowHandleSet(Word.commonHashLength, Word.commonHashOrder, 100); @@ -496,7 +499,8 @@ public final class SearchEvent implements ScoreMapUpdatesListener { } else { this.primarySearchThreadsL = null; this.nodeSearchThreads = null; - this.pollImmediately = !query.getSegment().connectedRWI() || !Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW_SEARCH, false); + this.pollImmediately = !query.getSegment().connectedRWI() + || !Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW_SEARCH, false); if ( generateAbstracts ) { // we need the results now try { @@ -1893,9 +1897,11 @@ public final class SearchEvent implements ScoreMapUpdatesListener { this.query.getQueryGoal().getIncludeHashes(), CacheStrategy.CACHEONLY, false, - 180, + snippetMaxLength(true), false); - final TextSnippet selectedSnippet = solrsnippet.getLineRaw().length() > yacysnippet.getLineRaw().length() ? solrsnippet : yacysnippet; + final TextSnippet selectedSnippet = this.query.isSnippetFetchFullText() && node.getText() != null && !node.getText().isEmpty() + ? new TextSnippet(node.url(), node.getText(), false, ResultClass.SOURCE_METADATA, "") + : (solrsnippet.getLineRaw().length() > yacysnippet.getLineRaw().length() ? solrsnippet : yacysnippet); final URIMetadataNode re = node.makeResultEntry(this.query.getSegment(), this.peers, maximizeSnippet(node, selectedSnippet)); addResult(re, localEntryElement.getWeight()); success = true; @@ -1937,7 +1943,7 @@ public final class SearchEvent implements ScoreMapUpdatesListener { */ public void addResult(URIMetadataNode resultEntry, final long score) { if (resultEntry == null) return; - final long ranking = (score * 128) + postRanking(resultEntry, this.ref /*this.getTopicNavigator(MAX_TOPWORDS)*/); + final long ranking = this.disablePostRanking ? score : (score * 128) + postRanking(resultEntry, this.ref /*this.getTopicNavigator(MAX_TOPWORDS)*/); // TODO: above was originally using (see below), but getTopicNavigator returns this.ref and possibliy alters this.ref on first call (this.ref.size < 2 -> this.ref.clear) // TODO: verify and straighten the use of addTopic, getTopic and getTopicNavigator and related score calculation // final long ranking = ((long) (score * 128.f)) + postRanking(resultEntry, this.getTopicNavigator(MAX_TOPWORDS)); @@ -2011,6 +2017,14 @@ public final class SearchEvent implements ScoreMapUpdatesListener { public URIMetadataNode getSnippet(URIMetadataNode page, final CacheStrategy cacheStrategy) { if (page == null) return null; + if (this.query.isSnippetFetchFullText()) { + final String fullText = page.getText(); + if (fullText != null && !fullText.isEmpty()) { + final TextSnippet snippet = new TextSnippet(page.url(), fullText, false, ResultClass.SOURCE_METADATA, ""); + return page.makeResultEntry(this.query.getSegment(), this.peers, snippet); + } + } + if (cacheStrategy == null) { final TextSnippet snippet = new TextSnippet( null, @@ -2019,7 +2033,7 @@ public final class SearchEvent implements ScoreMapUpdatesListener { this.snippetFetchWordHashes, null, ((this.query.constraint != null) && (this.query.constraint.get(Tokenizer.flag_cat_indexof))), - SearchEvent.SNIPPET_MAX_LENGTH, + snippetMaxLength(false), !this.query.isLocal()); return page.makeResultEntry(this.query.getSegment(), this.peers, maximizeSnippet(page, snippet)); // result without snippet } @@ -2036,7 +2050,7 @@ public final class SearchEvent implements ScoreMapUpdatesListener { this.snippetFetchWordHashes, cacheStrategy, ((this.query.constraint != null) && (this.query.constraint.get(Tokenizer.flag_cat_indexof))), - 180, + snippetMaxLength(true), !this.query.isLocal()); SearchEvent.log.info("text snippet load time for " + page.url().toNormalform(true) + ": " + (System.currentTimeMillis() - startTime) + " ms, " + (!snippet.getErrorCode().fail() ? "snippet found" : ("no snippet found (" + snippet.getError() + ")"))); @@ -2082,10 +2096,13 @@ public final class SearchEvent implements ScoreMapUpdatesListener { return page.makeResultEntry(this.query.getSegment(), this.peers, null); // result without snippet } - private static TextSnippet maximizeSnippet(final URIMetadataNode page, final TextSnippet snippet) { + private TextSnippet maximizeSnippet(final URIMetadataNode page, final TextSnippet snippet) { if (page == null || snippet == null || !snippet.exists()) { return snippet; } + if (!this.query.isSnippetFetchFullText()) { + return snippet; + } String rawText = page.getText(); if (rawText == null || rawText.isEmpty()) { final ArrayList descriptions = page.getDescription(); @@ -2099,6 +2116,13 @@ public final class SearchEvent implements ScoreMapUpdatesListener { return new TextSnippet(page.url(), rawText, snippet.getLineRaw(), snippet.isMarked(), snippet.getErrorCode(), snippet.getError()); } + private int snippetMaxLength(final boolean shortSnippetDefault) { + if (this.query.isSnippetFetchFullText()) { + return -1; + } + return shortSnippetDefault ? 180 : SearchEvent.SNIPPET_MAX_LENGTH; + } + /** * This is the access point for the search interface to retrive ranked results. * for display. diff --git a/source/net/yacy/search/snippet/TextSnippet.java b/source/net/yacy/search/snippet/TextSnippet.java index f6c79bec9..383a8a587 100644 --- a/source/net/yacy/search/snippet/TextSnippet.java +++ b/source/net/yacy/search/snippet/TextSnippet.java @@ -197,17 +197,18 @@ public class TextSnippet implements Comparable, Comparator queryTerms, - final HandleSet queryhashes, + public TextSnippet( + final LoaderDispatcher loader, + final URIMetadataNode row, + final Set queryTerms, + final HandleSet queryhashes, final CacheStrategy cacheStrategy, final boolean pre, final int snippetMaxLength, - final boolean reindexing) { - long beginTime = System.currentTimeMillis(); - // heise = "0OQUNU3JSs05" + final boolean reindexing) { + long beginTime = System.currentTimeMillis(); + final boolean fullDocumentSnippet = snippetMaxLength < 0; + // heise = "0OQUNU3JSs05" final DigestURL url = row.url(); if (queryTerms.isEmpty()) { @@ -283,19 +284,24 @@ public class TextSnippet implements Comparable, Comparator, Comparator= snippetMaxLength / 4 * 3) break; - } - if (s.length() > snippetMaxLength) { s.setLength(snippetMaxLength); s.trimToSize(); } - textline = s.toString(); - } - } - } + } + if (textline == null) { + sentences.reset(); + if (fullDocumentSnippet) { + textline = collectAllSentences(sentences); + } else { + final StringBuilder s = new StringBuilder(snippetMaxLength); + for (final StringBuilder t: sentences) { + s.append(t).append(' '); + if (s.length() >= snippetMaxLength / 4 * 3) break; + } + if (s.length() > snippetMaxLength) { s.setLength(snippetMaxLength); s.trimToSize(); } + textline = s.toString(); + } + } + } + } final String fallbackLine = textline.length() > 0 ? textline : this.line; init(url, fallbackLine, fallbackLine, false, ResultClass.SOURCE_METADATA, null, beginTime); return; @@ -397,28 +407,50 @@ public class TextSnippet implements Comparable, Comparator snippetMaxLength) textline = textline.substring(0, snippetMaxLength); + if (!fullDocumentSnippet && textline.length() > snippetMaxLength) textline = textline.substring(0, snippetMaxLength); // finally store this snippet in our own cache if(wordhashes != null) { snippetsCache.put(wordhashes, urlHash, textline); } init(url, textline, textline, false, source, null, beginTime); - } + } + + private static String collectAllSentences(final Iterable sentences) { + if (sentences == null) { + return null; + } + final StringBuilder all = new StringBuilder(); + for (final StringBuilder sentence : sentences) { + if (sentence == null || sentence.length() == 0) { + continue; + } + if (all.length() > 0) { + all.append(' '); + } + all.append(sentence); + } + return all.toString(); + } /** * Init a snippet line for urlhash -- cgit v1.2.3