summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--source/net/yacy/ai/LLM.java5
-rw-r--r--source/net/yacy/ai/RAGAugmentor.java7
-rw-r--r--source/net/yacy/search/query/QueryParams.java67
-rw-r--r--source/net/yacy/search/query/SearchEvent.java38
-rw-r--r--source/net/yacy/search/snippet/TextSnippet.java120
5 files changed, 167 insertions, 70 deletions
diff --git a/source/net/yacy/ai/LLM.java b/source/net/yacy/ai/LLM.java
index df0e82242..8e9530e79 100644
--- a/source/net/yacy/ai/LLM.java
+++ b/source/net/yacy/ai/LLM.java
@@ -272,6 +272,11 @@ public class LLM {
data.put("messages", context);
data.put("stop", new JSONArray(STOPTOKENS));
data.put("stream", false);
+
+ if (model.toLowerCase().contains("qwen3.5")) { // we don't think
+ data.put("reasoning_effort", "none");
+ data.put("enable_thinking", false);
+ }
if (schema != null) {
System.out.println(schema.toString());
diff --git a/source/net/yacy/ai/RAGAugmentor.java b/source/net/yacy/ai/RAGAugmentor.java
index c9b7215a9..1b5723c3f 100644
--- a/source/net/yacy/ai/RAGAugmentor.java
+++ b/source/net/yacy/ai/RAGAugmentor.java
@@ -189,11 +189,9 @@ public final class RAGAugmentor {
String title = node.title();
String text = null;
if (includeSnippet) {
+ TextSnippet snippet = node.textSnippet();
+ if (snippet != null && snippet.exists() && !snippet.getErrorCode().fail()) text = snippet.getLineRaw();
if (text == null || text.isEmpty()) text = node.snippet();
- if (text == null || text.isEmpty()) {
- TextSnippet snippet = node.textSnippet();
- if (snippet != null && snippet.exists() && !snippet.getErrorCode().fail()) text = snippet.getLineRaw();
- }
if (text == null || text.isEmpty()) text = firstFieldString(node.getFieldValue(CollectionSchema.description_txt.getSolrFieldName()));
if (text == null || text.isEmpty()) text = firstFieldString(node.getFieldValue(CollectionSchema.text_t.getSolrFieldName()));
result.put("text", text == null ? "" : text.trim());
@@ -264,6 +262,7 @@ public final class RAGAugmentor {
theQuery.setDateFacetMaxCount(sb.getConfigInt(
SwitchboardConstants.SEARCH_NAVIGATION_DATES_MAXCOUNT,
QueryParams.FACETS_DATE_MAXCOUNT_DEFAULT));
+ theQuery.setSnippetFetchFullText(true);
theQuery.getQueryGoal().filterOut(Switchboard.blueList);
return theQuery;
}
diff --git a/source/net/yacy/search/query/QueryParams.java b/source/net/yacy/search/query/QueryParams.java
index e3bd008bf..4686f7cf5 100644
--- a/source/net/yacy/search/query/QueryParams.java
+++ b/source/net/yacy/search/query/QueryParams.java
@@ -188,7 +188,9 @@ public final class QueryParams {
public final int maxDistance;
public final Bitfield constraint;
public final boolean allofconstraint;
- protected CacheStrategy snippetCacheStrategy;
+ protected CacheStrategy snippetCacheStrategy;
+ private boolean snippetFetchFullText;
+ private boolean localSolrOnly;
public final RankingProfile ranking;
private final Segment indexSegment;
public final String clienthost; // this is the client host that starts the query, not a site operator
@@ -298,9 +300,11 @@ public final class QueryParams {
this.zonecode = domainzone;
this.constraint = constraint;
this.allofconstraint = allofconstraint;
- this.siteexcludes = siteexcludes != null && siteexcludes.isEmpty() ? null: siteexcludes;
- this.snippetCacheStrategy = snippetCacheStrategy;
- this.clienthost = host;
+ this.siteexcludes = siteexcludes != null && siteexcludes.isEmpty() ? null: siteexcludes;
+ this.snippetCacheStrategy = snippetCacheStrategy;
+ this.snippetFetchFullText = false;
+ this.localSolrOnly = false;
+ this.clienthost = host;
this.remotepeer = null;
this.starttime = System.currentTimeMillis();
this.maxtime = 10000;
@@ -658,11 +662,11 @@ public final class QueryParams {
if (bf.length() > 0) params.setParam("boost", bf); // a boost function extension, see http://wiki.apache.org/solr/ExtendedDisMax#bf_.28Boost_Function.2C_additive.29
// set highlighting query attributes
- if (this.contentdom == Classification.ContentDomain.TEXT || this.contentdom == Classification.ContentDomain.ALL) {
- params.setHighlight(true);
- params.setHighlightFragsize(SearchEvent.SNIPPET_MAX_LENGTH);
- //params.setHighlightRequireFieldMatch();
- params.setHighlightSimplePost("</b>");
+ if (!this.snippetFetchFullText && (this.contentdom == Classification.ContentDomain.TEXT || this.contentdom == Classification.ContentDomain.ALL)) {
+ params.setHighlight(true);
+ params.setHighlightFragsize(SearchEvent.SNIPPET_MAX_LENGTH);
+ //params.setHighlightRequireFieldMatch();
+ params.setHighlightSimplePost("</b>");
params.setHighlightSimplePre("<b>");
params.setHighlightSnippets(5);
for (final CollectionSchema field: SOLR_SNIPPET_FIELDS) {
@@ -807,9 +811,24 @@ public final class QueryParams {
} else {
params.setFacet(false);
}
- params.setFields("*", "score"); // we need the score for post-ranking
- return params;
- }
+ params.setFields(buildResultFieldList(this.snippetFetchFullText));
+ return params;
+ }
+
+ private String[] buildResultFieldList(final boolean includeTextField) {
+ final ArrayList<String> fields = new ArrayList<>();
+ for (final CollectionSchema field : CollectionSchema.values()) {
+ if (!this.solrSchema.contains(field) || !field.isStored()) {
+ continue;
+ }
+ if (!includeTextField && field == CollectionSchema.text_t) {
+ continue;
+ }
+ fields.add(field.getSolrFieldName());
+ }
+ fields.add("score");
+ return fields.toArray(new String[fields.size()]);
+ }
long year = 1000L * 60L * 60L * 24L * 365L;
@@ -942,7 +961,7 @@ public final class QueryParams {
private volatile String idCacheAnon = null, idCache = null;
final static private char asterisk = '*';
- public String id(final boolean anonymized) {
+ public String id(final boolean anonymized) {
if (anonymized) {
if (this.idCacheAnon != null) return this.idCacheAnon;
} else {
@@ -989,7 +1008,9 @@ public final class QueryParams {
context.append(this.tld).append(asterisk);
context.append(this.inlink).append(asterisk);
context.append(this.lat).append(asterisk).append(this.lon).append(asterisk).append(this.radius).append(asterisk);
- context.append(this.snippetCacheStrategy == null ? "null" : this.snippetCacheStrategy.name());
+ context.append(this.snippetCacheStrategy == null ? "null" : this.snippetCacheStrategy.name()).append(asterisk);
+ context.append(this.snippetFetchFullText).append(asterisk);
+ context.append(this.localSolrOnly);
// Note : this.maxSuggestions search parameter do not need to be part of this id, as it has no impact on results themselves
@@ -1001,7 +1022,23 @@ public final class QueryParams {
}
return result;
}
- }
+ }
+
+ public boolean isSnippetFetchFullText() {
+ return this.snippetFetchFullText;
+ }
+
+ public void setSnippetFetchFullText(final boolean snippetFetchFullText) {
+ this.snippetFetchFullText = snippetFetchFullText;
+ }
+
+ public boolean isLocalSolrOnly() {
+ return this.localSolrOnly;
+ }
+
+ public void setLocalSolrOnly(final boolean localSolrOnly) {
+ this.localSolrOnly = localSolrOnly;
+ }
/**
* Build a search query URL from the given parameters.
diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java
index b7d7e94c2..9ad44a347 100644
--- a/source/net/yacy/search/query/SearchEvent.java
+++ b/source/net/yacy/search/query/SearchEvent.java
@@ -227,6 +227,8 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
/** if this is true, then every entry in result List is polled immediately to prevent a re-ranking in the resultList. This is usefull if there is only one index source. */
private final boolean pollImmediately;
+ /** Disable post-ranking when operating in explicit local-search mode. */
+ private final boolean disablePostRanking;
public final boolean excludeintext_image;
// the following values are filled during the search process as statistics for the search
@@ -430,6 +432,7 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
this.feedersTerminated = new AtomicInteger(0);
this.snippetFetchAlive = new AtomicInteger(0);
this.addRunning = true;
+ this.disablePostRanking = this.query.isLocal();
this.receivedRemoteReferences = new AtomicInteger(0);
this.order = new ReferenceOrder(this.query.ranking, this.query.targetlang);
this.urlhashes = new RowHandleSet(Word.commonHashLength, Word.commonHashOrder, 100);
@@ -496,7 +499,8 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
} else {
this.primarySearchThreadsL = null;
this.nodeSearchThreads = null;
- this.pollImmediately = !query.getSegment().connectedRWI() || !Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW_SEARCH, false);
+ this.pollImmediately = !query.getSegment().connectedRWI()
+ || !Switchboard.getSwitchboard().getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW_SEARCH, false);
if ( generateAbstracts ) {
// we need the results now
try {
@@ -1893,9 +1897,11 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
this.query.getQueryGoal().getIncludeHashes(),
CacheStrategy.CACHEONLY,
false,
- 180,
+ snippetMaxLength(true),
false);
- final TextSnippet selectedSnippet = solrsnippet.getLineRaw().length() > yacysnippet.getLineRaw().length() ? solrsnippet : yacysnippet;
+ final TextSnippet selectedSnippet = this.query.isSnippetFetchFullText() && node.getText() != null && !node.getText().isEmpty()
+ ? new TextSnippet(node.url(), node.getText(), false, ResultClass.SOURCE_METADATA, "")
+ : (solrsnippet.getLineRaw().length() > yacysnippet.getLineRaw().length() ? solrsnippet : yacysnippet);
final URIMetadataNode re = node.makeResultEntry(this.query.getSegment(), this.peers, maximizeSnippet(node, selectedSnippet));
addResult(re, localEntryElement.getWeight());
success = true;
@@ -1937,7 +1943,7 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
*/
public void addResult(URIMetadataNode resultEntry, final long score) {
if (resultEntry == null) return;
- final long ranking = (score * 128) + postRanking(resultEntry, this.ref /*this.getTopicNavigator(MAX_TOPWORDS)*/);
+ final long ranking = this.disablePostRanking ? score : (score * 128) + postRanking(resultEntry, this.ref /*this.getTopicNavigator(MAX_TOPWORDS)*/);
// TODO: above was originally using (see below), but getTopicNavigator returns this.ref and possibliy alters this.ref on first call (this.ref.size < 2 -> this.ref.clear)
// TODO: verify and straighten the use of addTopic, getTopic and getTopicNavigator and related score calculation
// final long ranking = ((long) (score * 128.f)) + postRanking(resultEntry, this.getTopicNavigator(MAX_TOPWORDS));
@@ -2011,6 +2017,14 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
public URIMetadataNode getSnippet(URIMetadataNode page, final CacheStrategy cacheStrategy) {
if (page == null) return null;
+ if (this.query.isSnippetFetchFullText()) {
+ final String fullText = page.getText();
+ if (fullText != null && !fullText.isEmpty()) {
+ final TextSnippet snippet = new TextSnippet(page.url(), fullText, false, ResultClass.SOURCE_METADATA, "");
+ return page.makeResultEntry(this.query.getSegment(), this.peers, snippet);
+ }
+ }
+
if (cacheStrategy == null) {
final TextSnippet snippet = new TextSnippet(
null,
@@ -2019,7 +2033,7 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
this.snippetFetchWordHashes,
null,
((this.query.constraint != null) && (this.query.constraint.get(Tokenizer.flag_cat_indexof))),
- SearchEvent.SNIPPET_MAX_LENGTH,
+ snippetMaxLength(false),
!this.query.isLocal());
return page.makeResultEntry(this.query.getSegment(), this.peers, maximizeSnippet(page, snippet)); // result without snippet
}
@@ -2036,7 +2050,7 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
this.snippetFetchWordHashes,
cacheStrategy,
((this.query.constraint != null) && (this.query.constraint.get(Tokenizer.flag_cat_indexof))),
- 180,
+ snippetMaxLength(true),
!this.query.isLocal());
SearchEvent.log.info("text snippet load time for " + page.url().toNormalform(true) + ": " + (System.currentTimeMillis() - startTime) + " ms, " + (!snippet.getErrorCode().fail() ? "snippet found" : ("no snippet found (" + snippet.getError() + ")")));
@@ -2082,10 +2096,13 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
return page.makeResultEntry(this.query.getSegment(), this.peers, null); // result without snippet
}
- private static TextSnippet maximizeSnippet(final URIMetadataNode page, final TextSnippet snippet) {
+ private TextSnippet maximizeSnippet(final URIMetadataNode page, final TextSnippet snippet) {
if (page == null || snippet == null || !snippet.exists()) {
return snippet;
}
+ if (!this.query.isSnippetFetchFullText()) {
+ return snippet;
+ }
String rawText = page.getText();
if (rawText == null || rawText.isEmpty()) {
final ArrayList<String> descriptions = page.getDescription();
@@ -2099,6 +2116,13 @@ public final class SearchEvent implements ScoreMapUpdatesListener {
return new TextSnippet(page.url(), rawText, snippet.getLineRaw(), snippet.isMarked(), snippet.getErrorCode(), snippet.getError());
}
+ private int snippetMaxLength(final boolean shortSnippetDefault) {
+ if (this.query.isSnippetFetchFullText()) {
+ return -1;
+ }
+ return shortSnippetDefault ? 180 : SearchEvent.SNIPPET_MAX_LENGTH;
+ }
+
/**
* This is the access point for the search interface to retrive ranked results.
* for display.
diff --git a/source/net/yacy/search/snippet/TextSnippet.java b/source/net/yacy/search/snippet/TextSnippet.java
index f6c79bec9..383a8a587 100644
--- a/source/net/yacy/search/snippet/TextSnippet.java
+++ b/source/net/yacy/search/snippet/TextSnippet.java
@@ -197,17 +197,18 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
init(url, rawLine, displayLine, isMarked, errorCode, errortext, beginTime);
}
- public TextSnippet(
- final LoaderDispatcher loader,
- final URIMetadataNode row,
- final Set<String> queryTerms,
- final HandleSet queryhashes,
+ public TextSnippet(
+ final LoaderDispatcher loader,
+ final URIMetadataNode row,
+ final Set<String> queryTerms,
+ final HandleSet queryhashes,
final CacheStrategy cacheStrategy,
final boolean pre,
final int snippetMaxLength,
- final boolean reindexing) {
- long beginTime = System.currentTimeMillis();
- // heise = "0OQUNU3JSs05"
+ final boolean reindexing) {
+ long beginTime = System.currentTimeMillis();
+ final boolean fullDocumentSnippet = snippetMaxLength < 0;
+ // heise = "0OQUNU3JSs05"
final DigestURL url = row.url();
if (queryTerms.isEmpty()) {
@@ -283,19 +284,24 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
// not found the snippet
init(url, null, null, false, ResultClass.SOURCE_METADATA, null, beginTime);
return;
- }
-
- if (sentences.iterator().hasNext()) {
- try {
- final SnippetExtractor tsr = new SnippetExtractor(sentences, remainingTerms, snippetMaxLength);
- textline = tsr.getSnippet();
- remainingTerms = tsr.getRemainingTerms();
- } catch (final UnsupportedOperationException e) {
- init(url, null, null, false, ResultClass.ERROR_NO_MATCH, "snippet extractor failed:" + e.getMessage(), beginTime);
- return;
- }
- }
- }
+ }
+
+ if (sentences.iterator().hasNext()) {
+ if (fullDocumentSnippet) {
+ textline = collectAllSentences(sentences);
+ remainingTerms.clear();
+ } else {
+ try {
+ final SnippetExtractor tsr = new SnippetExtractor(sentences, remainingTerms, snippetMaxLength);
+ textline = tsr.getSnippet();
+ remainingTerms = tsr.getRemainingTerms();
+ } catch (final UnsupportedOperationException e) {
+ init(url, null, null, false, ResultClass.ERROR_NO_MATCH, "snippet extractor failed:" + e.getMessage(), beginTime);
+ return;
+ }
+ }
+ }
+ }
if (remainingTerms.isEmpty()) {
// we found the snippet or the query is fully included in the headline or url
@@ -330,19 +336,23 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
prevSentence = currentSentence;
}
}
- }
- if (textline == null) {
- sentences.reset();
- final StringBuilder s = new StringBuilder(snippetMaxLength);
- for (final StringBuilder t: sentences) {
- s.append(t).append(' ');
- if (s.length() >= snippetMaxLength / 4 * 3) break;
- }
- if (s.length() > snippetMaxLength) { s.setLength(snippetMaxLength); s.trimToSize(); }
- textline = s.toString();
- }
- }
- }
+ }
+ if (textline == null) {
+ sentences.reset();
+ if (fullDocumentSnippet) {
+ textline = collectAllSentences(sentences);
+ } else {
+ final StringBuilder s = new StringBuilder(snippetMaxLength);
+ for (final StringBuilder t: sentences) {
+ s.append(t).append(' ');
+ if (s.length() >= snippetMaxLength / 4 * 3) break;
+ }
+ if (s.length() > snippetMaxLength) { s.setLength(snippetMaxLength); s.trimToSize(); }
+ textline = s.toString();
+ }
+ }
+ }
+ }
final String fallbackLine = textline.length() > 0 ? textline : this.line;
init(url, fallbackLine, fallbackLine, false, ResultClass.SOURCE_METADATA, null, beginTime);
return;
@@ -397,28 +407,50 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
return;
}
- try {
- final SnippetExtractor tsr = new SnippetExtractor(sentences, remainingTerms, snippetMaxLength);
- textline = tsr.getSnippet();
- remainingTerms = tsr.getRemainingTerms();
- } catch (final UnsupportedOperationException e) {
- init(url, null, null, false, ResultClass.ERROR_NO_MATCH, "snippet extractor failed:" + e.getMessage(), beginTime);
- return;
+ if (fullDocumentSnippet) {
+ textline = collectAllSentences(sentences);
+ remainingTerms.clear();
+ } else {
+ try {
+ final SnippetExtractor tsr = new SnippetExtractor(sentences, remainingTerms, snippetMaxLength);
+ textline = tsr.getSnippet();
+ remainingTerms = tsr.getRemainingTerms();
+ } catch (final UnsupportedOperationException e) {
+ init(url, null, null, false, ResultClass.ERROR_NO_MATCH, "snippet extractor failed:" + e.getMessage(), beginTime);
+ return;
+ }
}
- sentences = null;
+ sentences = null;
if (textline == null || !remainingTerms.isEmpty()) {
init(url, null, null, false, ResultClass.ERROR_NO_MATCH, "no matching snippet found", beginTime);
return;
}
- if (textline.length() > snippetMaxLength) textline = textline.substring(0, snippetMaxLength);
+ if (!fullDocumentSnippet && textline.length() > snippetMaxLength) textline = textline.substring(0, snippetMaxLength);
// finally store this snippet in our own cache
if(wordhashes != null) {
snippetsCache.put(wordhashes, urlHash, textline);
}
init(url, textline, textline, false, source, null, beginTime);
- }
+ }
+
+ private static String collectAllSentences(final Iterable<StringBuilder> sentences) {
+ if (sentences == null) {
+ return null;
+ }
+ final StringBuilder all = new StringBuilder();
+ for (final StringBuilder sentence : sentences) {
+ if (sentence == null || sentence.length() == 0) {
+ continue;
+ }
+ if (all.length() > 0) {
+ all.append(' ');
+ }
+ all.append(sentence);
+ }
+ return all.toString();
+ }
/**
* Init a snippet line for urlhash
generated by cgit v1.2.3 (git 2.46.0) at 2026-08-08 08:24:32 +0000