summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--source/net/yacy/ai/RAGAugmentor.java156
-rw-r--r--source/net/yacy/search/snippet/TextSnippet.java72
2 files changed, 109 insertions, 119 deletions
diff --git a/source/net/yacy/ai/RAGAugmentor.java b/source/net/yacy/ai/RAGAugmentor.java
index d1dbb18db..d28e33c2a 100644
--- a/source/net/yacy/ai/RAGAugmentor.java
+++ b/source/net/yacy/ai/RAGAugmentor.java
@@ -45,7 +45,6 @@ import org.json.JSONObject;
import net.yacy.cora.document.analysis.Classification;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.document.id.MultiProtocolURL;
-import net.yacy.cora.federate.solr.SolrType;
import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector;
import net.yacy.cora.federate.yacy.CacheStrategy;
import net.yacy.cora.lod.vocabulary.Tagging;
@@ -107,47 +106,10 @@ public final class RAGAugmentor {
public static JSONArray searchResults(String query, int count, final boolean includeSnippet, final Set<String> boostTerms) {
final JSONArray results = new JSONArray();
if (query == null || query.length() == 0 || count == 0) return results;
- Switchboard sb = Switchboard.getSwitchboard();
- EmbeddedSolrConnector connector = sb.index.fulltext().getDefaultEmbeddedConnector();
- final SolrQuery params = new SolrQuery();
- // Base query and parser setup.
- params.setQuery(query);
- params.set("defType", "edismax");
- // Static field boosts favor title/headings over body text.
- params.set("qf",
- CollectionSchema.title.getSolrFieldName() + "^3 " +
- CollectionSchema.text_t.getSolrFieldName() + "^1 " +
- CollectionSchema.sku.getSolrFieldName() + "^0.5 " +
- CollectionSchema.h1_txt.getSolrFieldName() + "^2");
- params.set("pf",
- CollectionSchema.title.getSolrFieldName() + "^5 " +
- CollectionSchema.text_t.getSolrFieldName() + "^2");
- final List<String> bqParts = new ArrayList<>();
- if (boostTerms != null && !boostTerms.isEmpty()) {
- // Apply weak boosts to overlap terms, keeping lexical query dominant.
- for (String term : boostTerms) {
- if (term == null || term.isEmpty()) continue;
- bqParts.add(CollectionSchema.title.getSolrFieldName() + ":" + term + "^0.5");
- bqParts.add(CollectionSchema.h1_txt.getSolrFieldName() + ":" + term + "^0.4");
- bqParts.add(CollectionSchema.text_t.getSolrFieldName() + ":" + term + "^0.2");
- }
- }
- // Slightly prefer archive/container file extensions in this use case.
- bqParts.add("(" + CollectionSchema.url_file_ext_s.getSolrFieldName() + ":(zip rar 7z tar gz bz2 xz tgz))^0.1");
- params.set("bq", String.join(" ", bqParts));
- params.setRows(count);
- params.setStart(0);
- params.setFacet(false);
- params.clearSorts();
- // Fetch only fields needed for tool / markdown rendering.
- params.setFields(
- CollectionSchema.sku.getSolrFieldName(), CollectionSchema.title.getSolrFieldName(), CollectionSchema.text_t.getSolrFieldName(),
- CollectionSchema.description_txt.getSolrFieldName(), CollectionSchema.keywords.getSolrFieldName(), CollectionSchema.synonyms_sxt.getSolrFieldName(),
- CollectionSchema.h1_txt.getSolrFieldName(), CollectionSchema.h2_txt.getSolrFieldName(), CollectionSchema.h3_txt.getSolrFieldName(),
- CollectionSchema.h4_txt.getSolrFieldName(), CollectionSchema.h5_txt.getSolrFieldName(), CollectionSchema.h6_txt.getSolrFieldName()
- );
- params.setIncludeScore(true);
- params.set("df", CollectionSchema.text_t.getSolrFieldName());
+ final Switchboard sb = Switchboard.getSwitchboard();
+ final EmbeddedSolrConnector connector = sb.index.fulltext().getDefaultEmbeddedConnector();
+ final QueryParams theQuery = buildTextQueryParams(query, count, QueryParams.Searchdom.LOCAL);
+ final SolrQuery params = theQuery.solrQuery(Classification.ContentDomain.TEXT, false, false, false);
try {
final SolrDocumentList sdl = connector.getDocumentListByParams(params);
@@ -158,11 +120,11 @@ public final class RAGAugmentor {
final JSONObject result = new JSONObject(true);
String url = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName());
result.put("url", url == null ? "" : url.trim());
- String title = getOneString(doc, CollectionSchema.title);
+ String title = firstFieldString(doc.getFieldValue(CollectionSchema.title.getSolrFieldName()));
result.put("title", title == null ? "" : title.trim());
if (includeSnippet) {
// Use indexed text body as quick snippet source.
- String text = (String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName());
+ String text = firstFieldString(doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName()));
result.put("text", limitSnippet(text == null ? "" : text.trim(), 2000));
}
results.put(result);
@@ -250,44 +212,7 @@ public final class RAGAugmentor {
final JSONArray results = new JSONArray();
if (query == null || query.length() == 0 || count == 0) return results;
final Switchboard sb = Switchboard.getSwitchboard();
- final RankingProfile ranking = sb.getRanking();
- final int timezoneOffset = 0;
- final QueryModifier modifier = new QueryModifier(timezoneOffset);
- // Parse modifiers and normalize effective query string.
- String querystring = modifier.parse(query);
- if (querystring.length() == 0) querystring = query == null ? "" : query.trim();
- if (querystring.length() == 0) return results;
- final QueryGoal qg = new QueryGoal(querystring);
- // Construct a standard text-domain global query.
- final QueryParams theQuery = new QueryParams(
- qg,
- modifier,
- 0,
- "",
- Classification.ContentDomain.TEXT,
- "",
- timezoneOffset,
- new HashSet<Tagging.Metatag>(),
- CacheStrategy.IFFRESH,
- count,
- 0,
- ".*",
- null,
- null,
- QueryParams.Searchdom.GLOBAL,
- null,
- true,
- DigestURL.hosthashess(sb.getConfig("search.excludehosth", "")),
- MultiProtocolURL.TLD_any_zone_filter,
- null,
- false,
- sb.index,
- ranking,
- ClientIdentification.yacyIntranetCrawlerAgent.userAgent(),
- 0.0d,
- 0.0d,
- 0.0d,
- sb.getConfigSet("search.navigation"));
+ final QueryParams theQuery = buildTextQueryParams(query, count, QueryParams.Searchdom.GLOBAL);
final SearchEvent theSearch = SearchEventCache.getEvent(
theQuery,
sb.peers,
@@ -332,6 +257,55 @@ public final class RAGAugmentor {
}
/**
+ * Build a standard YaCy text query using the shared query parser and ranking
+ * configuration. This gives local and global RAG retrieval the same Solr query
+ * semantics as the normal search stack.
+ *
+ * @param query raw query string
+ * @param count maximum number of results to retrieve
+ * @param searchdom local or global search scope
+ * @return shared query parameters ready for Solr/event execution
+ */
+ private static QueryParams buildTextQueryParams(final String query, final int count, final QueryParams.Searchdom searchdom) {
+ final Switchboard sb = Switchboard.getSwitchboard();
+ final RankingProfile ranking = sb.getRanking();
+ final int timezoneOffset = 0;
+ final QueryModifier modifier = new QueryModifier(timezoneOffset);
+ String querystring = modifier.parse(query);
+ if (querystring.length() == 0) querystring = query == null ? "" : query.trim();
+ final QueryGoal qg = new QueryGoal(querystring);
+ return new QueryParams(
+ qg,
+ modifier,
+ 0,
+ "",
+ Classification.ContentDomain.TEXT,
+ "",
+ timezoneOffset,
+ new HashSet<Tagging.Metatag>(),
+ CacheStrategy.IFFRESH,
+ count,
+ 0,
+ ".*",
+ null,
+ null,
+ searchdom,
+ null,
+ true,
+ DigestURL.hosthashess(sb.getConfig("search.excludehosth", "")),
+ MultiProtocolURL.TLD_any_zone_filter,
+ null,
+ false,
+ sb.index,
+ ranking,
+ ClientIdentification.yacyIntranetCrawlerAgent.userAgent(),
+ 0.0d,
+ 0.0d,
+ 0.0d,
+ sb.getConfigSet("search.navigation"));
+ }
+
+ /**
* Uses an LLM list schema prompt to compute likely discriminative search
* terms for a user prompt.
*
@@ -490,24 +464,6 @@ public final class RAGAugmentor {
}
/**
- * Reads a single string value from a (possibly multivalued) Solr field.
- *
- * @param doc source Solr document
- * @param field schema field descriptor
- * @return first string value or empty string
- */
- private static String getOneString(SolrDocument doc, CollectionSchema field) {
- assert field.isMultiValued();
- assert field.getType() == SolrType.string || field.getType() == SolrType.text_general;
- Object r = doc.getFieldValue(field.getSolrFieldName());
- if (r == null) return "";
- if (r instanceof ArrayList) {
- return (String) ((ArrayList<?>) r).get(0);
- }
- return r.toString();
- }
-
- /**
* Represents one candidate snippet around a search result and stores its
* relevance score relative to the query.
*/
diff --git a/source/net/yacy/search/snippet/TextSnippet.java b/source/net/yacy/search/snippet/TextSnippet.java
index 7b0271281..948e41c69 100644
--- a/source/net/yacy/search/snippet/TextSnippet.java
+++ b/source/net/yacy/search/snippet/TextSnippet.java
@@ -86,8 +86,16 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
/**
* <code>[^\\p{L}\\p{N}]</code>
*/
- private static final Pattern p4 =
- Pattern.compile("[^\\p{L}\\p{N}]");
+ private static final Pattern p4 =
+ Pattern.compile("[^\\p{L}\\p{N}]");
+ private static final Pattern MARKDOWN_LINK_PATTERN =
+ Pattern.compile("\\[(.+?)\\]\\((.+?)\\)");
+ private static final Pattern MARKDOWN_IMAGE_PATTERN =
+ Pattern.compile("!\\[(.*?)\\]\\((.+?)\\)");
+ private static final Pattern MARKDOWN_EMPHASIS_PATTERN =
+ Pattern.compile("(\\*\\*|__|~~|`|\\*|_)");
+ private static final Pattern MARKDOWN_PREFIX_PATTERN =
+ Pattern.compile("(?m)^\\s{0,3}(#{1,6}\\s+|>\\s+|[-*+]\\s+|\\d+\\.\\s+)");
public static class Cache {
private final ARC<String, String> cache;
@@ -147,8 +155,11 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
/** URL hash of this snippet */
private byte[] urlhash;
- /** The raw (unmodified) line from source ( use getDescriptionLine() to get the html encoded version for display) */
- private String line;
+ /** The raw line from source, kept for RAG and non-display consumers */
+ private String rawline;
+
+ /** Display-oriented snippet line with markdown formatting stripped */
+ private String line;
/** Set to true when query words are already marked in the input text */
private boolean isMarked;
@@ -406,7 +417,8 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
final String errortext,
final long beginTime) {
this.urlhash = url.hash();
- this.line = sanitizeSnippetLine(line);
+ this.rawline = sanitizeSnippetLine(line);
+ this.line = stripMarkdownForDisplay(this.rawline);
this.isMarked = isMarked;
this.resultStatus = errorCode;
this.error = errortext;
@@ -461,6 +473,28 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
return sanitized.toString().replaceAll("\\s{2,}", " ").trim();
}
+ /**
+ * Strip markdown syntax from snippet text intended for browser display while
+ * preserving readable content. The raw snippet remains available through
+ * getLineRaw() for RAG consumers.
+ * @param line raw snippet text
+ * @return display-friendly plain text
+ */
+ static String stripMarkdownForDisplay(final String line) {
+ if (line == null || line.isEmpty()) {
+ return line;
+ }
+ String sanitized = line;
+ sanitized = sanitized.replace("```", " ");
+ sanitized = MARKDOWN_IMAGE_PATTERN.matcher(sanitized).replaceAll("$1");
+ sanitized = MARKDOWN_LINK_PATTERN.matcher(sanitized).replaceAll("$1");
+ sanitized = MARKDOWN_PREFIX_PATTERN.matcher(sanitized).replaceAll("");
+ sanitized = MARKDOWN_EMPHASIS_PATTERN.matcher(sanitized).replaceAll("");
+ sanitized = sanitized.replaceAll("\\\\([\\\\`*_{}\\[\\]()#+\\-.!>~|])", "$1");
+ sanitized = sanitized.replaceAll("\\s{2,}", " ").trim();
+ return sanitized;
+ }
+
private static boolean isLikelyJavaScriptBlock(final String blockContent) {
if (blockContent == null) {
return false;
@@ -487,8 +521,8 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
/**
* @return true when a snippet text is available for the document corresponding to this.urlhash
*/
- public boolean exists() {
- return this.line != null;
+ public boolean exists() {
+ return this.rawline != null;
}
/**
@@ -501,8 +535,8 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
/**
* @return the raw snippet text line, with query words eventually already marked
*/
- public String getLineRaw() {
- return (this.line == null) ? "" : this.line;
+ public String getLineRaw() {
+ return (this.rawline == null) ? "" : this.rawline;
}
public String getError() {
@@ -563,16 +597,16 @@ public class TextSnippet implements Comparable<TextSnippet>, Comparator<TextSnip
* @param queryGoal
* @return html encoded snippet line
*/
- public String descriptionline(QueryGoal queryGoal) {
- if (descriptionline != null) return descriptionline;
- if (this.isMarked) {
- // html encode source, keep <b>..</b>
- descriptionline = CharacterCoding.unicode2html(this.getLineRaw(), false).replaceAll("&lt;b&gt;(.+?)&lt;/b&gt;", "<b>$1</b>");
- } else {
- descriptionline = this.getLineMarked(queryGoal);
- }
- return descriptionline;
- }
+ public String descriptionline(QueryGoal queryGoal) {
+ if (descriptionline != null) return descriptionline;
+ if (this.isMarked) {
+ // html encode source, keep <b>..</b>
+ descriptionline = CharacterCoding.unicode2html(this.line == null ? "" : this.line, false).replaceAll("&lt;b&gt;(.+?)&lt;/b&gt;", "<b>$1</b>");
+ } else {
+ descriptionline = this.getLineMarked(queryGoal);
+ }
+ return descriptionline;
+ }
@Override
public int compareTo(final TextSnippet o) {
generated by cgit v1.2.3 (git 2.46.0) at 2026-08-08 08:24:28 +0000