Align RAG retrieval with web search and disable local post-ranking

5 files changed, 167 insertions, 70 deletions

diff --git a/source/net/yacy/ai/LLM.java b/source/net/yacy/ai/LLM.java
index df0e82242..8e9530e79 100644
--- a/source/net/yacy/ai/LLM.java
+++ b/source/net/yacy/ai/LLM.java
@@ -272,6 +272,11 @@ public class LLM {
             data.put("messages", context);
             data.put("stop", new JSONArray(STOPTOKENS));
             data.put("stream", false);
+            
+            if (model.toLowerCase().contains("qwen3.5")) { // we don't think
+                data.put("reasoning_effort", "none");
+                data.put("enable_thinking", false);
+            }
 
             if (schema != null) {
                 System.out.println(schema.toString());
diff --git a/source/net/yacy/ai/RAGAugmentor.java b/source/net/yacy/ai/RAGAugmentor.java
index c9b7215a9..1b5723c3f 100644
--- a/source/net/yacy/ai/RAGAugmentor.java
+++ b/source/net/yacy/ai/RAGAugmentor.java
@@ -189,11 +189,9 @@ public final class RAGAugmentor {
                 String title = node.title();
                 String text = null;
                 if (includeSnippet) {
+                    TextSnippet snippet = node.textSnippet();
+                    if (snippet != null && snippet.exists() && !snippet.getErrorCode().fail()) text = snippet.getLineRaw();
                     if (text == null || text.isEmpty()) text = node.snippet();
-                    if (text == null || text.isEmpty()) {
-                        TextSnippet snippet = node.textSnippet();
-                        if (snippet != null && snippet.exists() && !snippet.getErrorCode().fail()) text = snippet.getLineRaw();
-                    }
                     if (text == null || text.isEmpty()) text = firstFieldString(node.getFieldValue(CollectionSchema.description_txt.getSolrFieldName()));
                     if (text == null || text.isEmpty()) text = firstFieldString(node.getFieldValue(CollectionSchema.text_t.getSolrFieldName()));
                     result.put("text", text == null ? "" : text.trim());
@@ -264,6 +262,7 @@ public final class RAGAugmentor {
         theQuery.setDateFacetMaxCount(sb.getConfigInt(
                 SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,
                 QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));
+        theQuery.setSnippetFetchFullText(true);
         theQuery.getQueryGoal().filterOut(Switchboard.blueList);
         return theQuery;
     }
diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.java
index e3bd008bf..4686f7cf5 100644
--- a/source/net/yacy/search/query/QueryParams.java
+++ b/source/net/yacy/search/query/QueryParams.java
@@ -188,7 +188,9 @@ public final class QueryParams {
     public final int maxDistance;

     public final Bitfield constraint;

     public final boolean allofconstraint;

-    protected CacheStrategy snippetCacheStrategy;

+    protected CacheStrategy snippetCacheStrategy;
+    private boolean snippetFetchFullText;
+    private boolean localSolrOnly;
     public final RankingProfile ranking;

     private final Segment indexSegment;

     public final String clienthost; // this is the client host that starts the query, not a site operator

@@ -298,9 +300,11 @@ public final class QueryParams {
         this.zonecode = domainzone;

         this.constraint = constraint;

         this.allofconstraint = allofconstraint;

-        this.siteexcludes = siteexcludes != null && siteexcludes.isEmpty() ? null: siteexcludes;

-        this.snippetCacheStrategy = snippetCacheStrategy;

-        this.clienthost = host;

+        this.siteexcludes = siteexcludes != null && siteexcludes.isEmpty() ? null: siteexcludes;
+        this.snippetCacheStrategy = snippetCacheStrategy;
+        this.snippetFetchFullText = false;
+        this.localSolrOnly = false;
+        this.clienthost = host;
         this.remotepeer = null;

         this.starttime = System.currentTimeMillis();

         this.maxtime = 10000;

@@ -658,11 +662,11 @@ public final class QueryParams {
         if (bf.length() > 0) params.setParam("boost", bf); // a boost function extension, see http://wiki.apache.org/solr/ExtendedDisMax#bf_.28Boost_Function.2C_additive.29

         

         // set highlighting query attributes

-        if (this.contentdom == Classification.ContentDomain.TEXT || this.contentdom == Classification.ContentDomain.ALL) {

-        	params.setHighlight(true);

-        	params.setHighlightFragsize(SearchEvent.SNIPPET_MAX_LENGTH);

-            //params.setHighlightRequireFieldMatch();

-        	params.setHighlightSimplePost("</b>");

+        if (!this.snippetFetchFullText && (this.contentdom == Classification.ContentDomain.TEXT || this.contentdom == Classification.ContentDomain.ALL)) {
+        	params.setHighlight(true);
+        	params.setHighlightFragsize(SearchEvent.SNIPPET_MAX_LENGTH);
+            //params.setHighlightRequireFieldMatch();
+        	params.setHighlightSimplePost("</b>");
         	params.setHighlightSimplePre("<b>");

         	params.setHighlightSnippets(5);

             for (final CollectionSchema field: SOLR_SNIPPET_FIELDS) {

@@ -807,9 +811,24 @@ public final class QueryParams {
         } else {

             params.setFacet(false);

         }

-        params.setFields("*", "score"); // we need the score for post-ranking

-        return params;

-    }

+        params.setFields(buildResultFieldList(this.snippetFetchFullText));
+        return params;
+    }
+
+    private String[] buildResultFieldList(final boolean includeTextField) {
+        final ArrayList<String> fields = new ArrayList<>();
+        for (final CollectionSchema field : CollectionSchema.values()) {
+            if (!this.solrSchema.contains(field) || !field.isStored()) {
+                continue;
+            }
+            if (!includeTextField && field == CollectionSchema.text_t) {
+                continue;
+            }
+            fields.add(field.getSolrFieldName());
+        }
+        fields.add("score");
+        return fields.toArray(new String[fields.size()]);
+    }
     

     long year = 1000L * 60L * 60L * 24L * 365L;

     

@@ -942,7 +961,7 @@ public final class QueryParams {
 

     private volatile String idCacheAnon = null, idCache = null;

     final static private char asterisk = '*';

-    public String id(final boolean anonymized) {

+    public String id(final boolean anonymized) {
         if (anonymized) {

             if (this.idCacheAnon != null) return this.idCacheAnon;

         } else {

@@ -989,7 +1008,9 @@ public final class QueryParams {
             context.append(this.tld).append(asterisk);

             context.append(this.inlink).append(asterisk);

             context.append(this.lat).append(asterisk).append(this.lon).append(asterisk).append(this.radius).append(asterisk);

-            context.append(this.snippetCacheStrategy == null ? "null" : this.snippetCacheStrategy.name());

+            context.append(this.snippetCacheStrategy == null ? "null" : this.snippetCacheStrategy.name()).append(asterisk);
+            context.append(this.snippetFetchFullText).append(asterisk);
+            context.append(this.localSolrOnly);
             

             // Note : this.maxSuggestions search parameter do not need to be part of this id, as it has no impact on results themselves

             

@@ -1001,7 +1022,23 @@ public final class QueryParams {
             }

             return result;

         }

-    }

+    }
+
+    public boolean isSnippetFetchFullText() {
+        return this.snippetFetchFullText;
+    }
+
+    public void setSnippetFetchFullText(final boolean snippetFetchFullText) {
+        this.snippetFetchFullText = snippetFetchFullText;
+    }
+
+    public boolean isLocalSolrOnly() {
+        return this.localSolrOnly;
+    }
+
+    public void setLocalSolrOnly(final boolean localSolrOnly) {
+        this.localSolrOnly = localSolrOnly;
+    }
 

     /**

 	 * Build a search query URL from the given parameters.

diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java
index b7d7e94c2..9ad44a347 100644
--- a/source/net/yacy/search/query/SearchEvent.java
+++ b/source/net/yacy/search/query/SearchEvent.java
@@ -227,6 +227,8 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
 
     /** if this is true, then every entry in result List is polled immediately to prevent a re-ranking in the resultList. This is usefull if there is only one index source. */
     private final boolean pollImmediately;
+    /** Disable post-ranking when operating in explicit local-search mode. */
+    private final boolean disablePostRanking;
     public  final boolean excludeintext_image;
 
     // the following values are filled during the search process as statistics for the search
@@ -430,6 +432,7 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
         this.feedersTerminated = new AtomicInteger(0);
         this.snippetFetchAlive = new AtomicInteger(0);
         this.addRunning = true;
+        this.disablePostRanking = this.query.isLocal();
         this.receivedRemoteReferences = new AtomicInteger(0);
         this.order = new ReferenceOrder(this.query.ranking, this.query.targetlang);
         this.urlhashes = new RowHandleSet(Word.commonHashLength, Word.commonHashOrder, 100);
@@ -496,7 +499,8 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
         } else {
             this.primarySearchThreadsL = null;
             this.nodeSearchThreads = null;
-            this.pollImmediately = !query.getSegment().connectedRWI() || !Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW_SEARCH, false);
+            this.pollImmediately = !query.getSegment().connectedRWI()
+                    || !Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW_SEARCH, false);
             if ( generateAbstracts ) {
                 // we need the results now
                 try {
@@ -1893,9 +1897,11 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
                         this.query.getQueryGoal().getIncludeHashes(),
                         CacheStrategy.CACHEONLY,
                         false,
-                        180,
+                        snippetMaxLength(true),
                         false);
-                final TextSnippet selectedSnippet = solrsnippet.getLineRaw().length() > yacysnippet.getLineRaw().length() ? solrsnippet : yacysnippet;
+                final TextSnippet selectedSnippet = this.query.isSnippetFetchFullText() && node.getText() != null && !node.getText().isEmpty()
+                        ? new TextSnippet(node.url(), node.getText(), false, ResultClass.SOURCE_METADATA, "")
+                        : (solrsnippet.getLineRaw().length() > yacysnippet.getLineRaw().length() ? solrsnippet : yacysnippet);
                 final URIMetadataNode re = node.makeResultEntry(this.query.getSegment(), this.peers, maximizeSnippet(node, selectedSnippet));
                 addResult(re, localEntryElement.getWeight());
                 success = true;
@@ -1937,7 +1943,7 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
      */
     public void addResult(URIMetadataNode resultEntry, final long score) {
         if (resultEntry == null) return;
-        final long ranking = (score * 128) + postRanking(resultEntry, this.ref /*this.getTopicNavigator(MAX_TOPWORDS)*/);
+        final long ranking = this.disablePostRanking ? score : (score * 128) + postRanking(resultEntry, this.ref /*this.getTopicNavigator(MAX_TOPWORDS)*/);
         // TODO: above was originally using (see below), but getTopicNavigator returns this.ref and possibliy alters this.ref on first call (this.ref.size < 2 -> this.ref.clear)
         // TODO: verify and straighten the use of addTopic, getTopic and getTopicNavigator and related score calculation
         // final long ranking = ((long) (score * 128.f)) + postRanking(resultEntry, this.getTopicNavigator(MAX_TOPWORDS));
@@ -2011,6 +2017,14 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
     public URIMetadataNode getSnippet(URIMetadataNode page, final CacheStrategy cacheStrategy) {
         if (page == null) return null;
 
+        if (this.query.isSnippetFetchFullText()) {
+            final String fullText = page.getText();
+            if (fullText != null && !fullText.isEmpty()) {
+                final TextSnippet snippet = new TextSnippet(page.url(), fullText, false, ResultClass.SOURCE_METADATA, "");
+                return page.makeResultEntry(this.query.getSegment(), this.peers, snippet);
+            }
+        }
+
         if (cacheStrategy == null) {
             final TextSnippet snippet = new TextSnippet(
                     null,
@@ -2019,7 +2033,7 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
                     this.snippetFetchWordHashes,
                     null,
                     ((this.query.constraint != null) && (this.query.constraint.get(Tokenizer.flag_cat_indexof))),
-                    SearchEvent.SNIPPET_MAX_LENGTH,
+                    snippetMaxLength(false),
                     !this.query.isLocal());
             return page.makeResultEntry(this.query.getSegment(), this.peers, maximizeSnippet(page, snippet)); // result without snippet
         }
@@ -2036,7 +2050,7 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
                     this.snippetFetchWordHashes,
                     cacheStrategy,
                     ((this.query.constraint != null) && (this.query.constraint.get(Tokenizer.flag_cat_indexof))),
-                    180,
+                    snippetMaxLength(true),
                     !this.query.isLocal());
             SearchEvent.log.info("text snippet load time for " + page.url().toNormalform(true) + ": " + (System.currentTimeMillis() - startTime) + " ms, " + (!snippet.getErrorCode().fail() ? "snippet found" : ("no snippet found (" + snippet.getError() + ")")));
 
@@ -2082,10 +2096,13 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
         return page.makeResultEntry(this.query.getSegment(), this.peers, null); // result without snippet
     }
 
-    private static TextSnippet maximizeSnippet(final URIMetadataNode page, final TextSnippet snippet) {
+    private TextSnippet maximizeSnippet(final URIMetadataNode page, final TextSnippet snippet) {
         if (page == null || snippet == null || !snippet.exists()) {
             return snippet;
         }
+        if (!this.query.isSnippetFetchFullText()) {
+            return snippet;
+        }
         String rawText = page.getText();
         if (rawText == null || rawText.isEmpty()) {
             final ArrayList<String> descriptions = page.getDescription();
@@ -2099,6 +2116,13 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
         return new TextSnippet(page.url(), rawText, snippet.getLineRaw(), snippet.isMarked(), snippet.getErrorCode(), snippet.getError());
     }
 
+    private int snippetMaxLength(final boolean shortSnippetDefault) {
+        if (this.query.isSnippetFetchFullText()) {
+            return -1;
+        }
+        return shortSnippetDefault ? 180 : SearchEvent.SNIPPET_MAX_LENGTH;
+    }
+
     /**
      * This is the access point for the search interface to retrive ranked results.
      * for display.
diff --git a/source/net/yacy/search/snippet/TextSnippet.java b/source/net/yacy/search/snippet/TextSnippet.java
index f6c79bec9..383a8a587 100644
--- a/source/net/yacy/search/snippet/TextSnippet.java
+++ b/source/net/yacy/search/snippet/TextSnippet.java
@@ -197,17 +197,18 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
         init(url, rawLine, displayLine, isMarked, errorCode, errortext, beginTime);
     }
 

-    public TextSnippet(

-            final LoaderDispatcher loader,

-            final URIMetadataNode row,

-            final Set<String> queryTerms,

-            final HandleSet queryhashes,

+    public TextSnippet(
+            final LoaderDispatcher loader,
+            final URIMetadataNode row,
+            final Set<String> queryTerms,
+            final HandleSet queryhashes,
             final CacheStrategy cacheStrategy,

             final boolean pre,

             final int snippetMaxLength,

-            final boolean reindexing) {

-    	long beginTime = System.currentTimeMillis();

-        // heise = "0OQUNU3JSs05"

+            final boolean reindexing) {
+    	long beginTime = System.currentTimeMillis();
+        final boolean fullDocumentSnippet = snippetMaxLength < 0;
+        // heise = "0OQUNU3JSs05"
         

         final DigestURL url = row.url();

         if (queryTerms.isEmpty()) {

@@ -283,19 +284,24 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
                 // not found the snippet

                 init(url, null, null, false, ResultClass.SOURCE_METADATA, null, beginTime);
                 return;

-            }

-

-            if (sentences.iterator().hasNext()) {

-                try {

-                    final SnippetExtractor tsr = new SnippetExtractor(sentences, remainingTerms, snippetMaxLength);

-                    textline = tsr.getSnippet();

-                    remainingTerms = tsr.getRemainingTerms();

-                } catch (final UnsupportedOperationException e) {

-                    init(url, null, null, false, ResultClass.ERROR_NO_MATCH, "snippet extractor failed:" + e.getMessage(), beginTime);
-                    return;

-                }

-            }

-       }

+            }
+
+            if (sentences.iterator().hasNext()) {
+                if (fullDocumentSnippet) {
+                    textline = collectAllSentences(sentences);
+                    remainingTerms.clear();
+                } else {
+                    try {
+                        final SnippetExtractor tsr = new SnippetExtractor(sentences, remainingTerms, snippetMaxLength);
+                        textline = tsr.getSnippet();
+                        remainingTerms = tsr.getRemainingTerms();
+                    } catch (final UnsupportedOperationException e) {
+                        init(url, null, null, false, ResultClass.ERROR_NO_MATCH, "snippet extractor failed:" + e.getMessage(), beginTime);
+                        return;
+                    }
+                }
+            }
+       }
 

        if (remainingTerms.isEmpty()) {

             // we found the snippet or the query is fully included in the headline or url

@@ -330,19 +336,23 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
                                 prevSentence = currentSentence;

                             }

                         }

-                    }

-                    if (textline == null) {

-                    	sentences.reset();

-                        final StringBuilder s = new StringBuilder(snippetMaxLength);

-                        for (final StringBuilder t: sentences) {

-                        	s.append(t).append(' ');

-                        	if (s.length() >= snippetMaxLength / 4 * 3) break;

-                        }

-                        if (s.length() > snippetMaxLength) { s.setLength(snippetMaxLength); s.trimToSize(); }

-                        textline = s.toString();

-                    }

-                }

-            }

+                    }
+                    if (textline == null) {
+                    	sentences.reset();
+                        if (fullDocumentSnippet) {
+                            textline = collectAllSentences(sentences);
+                        } else {
+                            final StringBuilder s = new StringBuilder(snippetMaxLength);
+                            for (final StringBuilder t: sentences) {
+                        	    s.append(t).append(' ');
+                        	    if (s.length() >= snippetMaxLength / 4 * 3) break;
+                            }
+                            if (s.length() > snippetMaxLength) { s.setLength(snippetMaxLength); s.trimToSize(); }
+                            textline = s.toString();
+                        }
+                    }
+                }
+            }
             final String fallbackLine = textline.length() > 0 ? textline : this.line;
             init(url, fallbackLine, fallbackLine, false, ResultClass.SOURCE_METADATA, null, beginTime);
             return;
@@ -397,28 +407,50 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
             return;
         }
 

-        try {

-            final SnippetExtractor tsr = new SnippetExtractor(sentences, remainingTerms, snippetMaxLength);

-            textline = tsr.getSnippet();

-            remainingTerms =  tsr.getRemainingTerms();

-        } catch (final UnsupportedOperationException e) {

-            init(url, null, null, false, ResultClass.ERROR_NO_MATCH, "snippet extractor failed:" + e.getMessage(), beginTime);
-            return;
+        if (fullDocumentSnippet) {
+            textline = collectAllSentences(sentences);
+            remainingTerms.clear();
+        } else {
+            try {
+                final SnippetExtractor tsr = new SnippetExtractor(sentences, remainingTerms, snippetMaxLength);
+                textline = tsr.getSnippet();
+                remainingTerms =  tsr.getRemainingTerms();
+            } catch (final UnsupportedOperationException e) {
+                init(url, null, null, false, ResultClass.ERROR_NO_MATCH, "snippet extractor failed:" + e.getMessage(), beginTime);
+                return;
+            }
         }
-        sentences = null;

+        sentences = null;
 

         if (textline == null || !remainingTerms.isEmpty()) {

             init(url, null, null, false, ResultClass.ERROR_NO_MATCH, "no matching snippet found", beginTime);
             return;
         }
-        if (textline.length() > snippetMaxLength) textline = textline.substring(0, snippetMaxLength);

+        if (!fullDocumentSnippet && textline.length() > snippetMaxLength) textline = textline.substring(0, snippetMaxLength);
 

         // finally store this snippet in our own cache

         if(wordhashes != null) {

         	snippetsCache.put(wordhashes, urlHash, textline);

         }

         init(url, textline, textline, false, source, null, beginTime);
-    }

+    }
+
+    private static String collectAllSentences(final Iterable<StringBuilder> sentences) {
+        if (sentences == null) {
+            return null;
+        }
+        final StringBuilder all = new StringBuilder();
+        for (final StringBuilder sentence : sentences) {
+            if (sentence == null || sentence.length() == 0) {
+                continue;
+            }
+            if (all.length() > 0) {
+                all.append(' ');
+            }
+            all.append(sentence);
+        }
+        return all.toString();
+    }
 

     /**

      * Init a snippet line for urlhash