diff options
| -rw-r--r-- | defaults/web.xml | 14 | ||||
| -rw-r--r-- | source/net/yacy/http/servlets/MCPSearchServlet.java | 323 | ||||
| -rw-r--r-- | source/net/yacy/http/servlets/RAGProxyServlet.java | 59 |
3 files changed, 377 insertions, 19 deletions
diff --git a/defaults/web.xml b/defaults/web.xml index 43a500cc1..d0e850709 100644 --- a/defaults/web.xml +++ b/defaults/web.xml @@ -53,6 +53,11 @@ <servlet> <servlet-name>RAGProxyServlet</servlet-name> <servlet-class>net.yacy.http.servlets.RAGProxyServlet</servlet-class> + </servlet> + + <servlet> + <servlet-name>MCPSearchServlet</servlet-name> + <servlet-class>net.yacy.http.servlets.MCPSearchServlet</servlet-class> </servlet> <!-- servlet to provide searchresults via proxy --> @@ -92,6 +97,15 @@ <servlet-name>RAGProxyServlet</servlet-name> <url-pattern>/v1/chat/completions</url-pattern> </servlet-mapping> + + <servlet-mapping> + <servlet-name>MCPSearchServlet</servlet-name> + <url-pattern>/tools</url-pattern> + <url-pattern>/tools/list</url-pattern> + <url-pattern>/tools/list_changed</url-pattern> + <url-pattern>/tools/call</url-pattern> + </servlet-mapping> + <!-- eof hardcoded mappings --> <!-- additional (optional) mappings --> diff --git a/source/net/yacy/http/servlets/MCPSearchServlet.java b/source/net/yacy/http/servlets/MCPSearchServlet.java new file mode 100644 index 000000000..93a2db1b9 --- /dev/null +++ b/source/net/yacy/http/servlets/MCPSearchServlet.java @@ -0,0 +1,323 @@ +/** + * MCPSearchServlet + * Copyright 2025 by Michael Peter Christen + * First released 11.10.2025 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.http.servlets; + +import java.io.BufferedReader; +import java.io.IOException; + +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.solr.servlet.cache.Method; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONTokener; + +/** + * This servlet exposes a minimal Model Context Protocol (MCP) server that + * offers a single tool, `search`, backed by the YaCy search index. + * Clients can initialize a JSON-RPC session, list available tools and invoke + * the web search tool to retrieve results from the embedded Solr instance. + * The response payload follows the MCP conventions (JSON-RPC 2.0 with tool + * results wrapped inside a `content` array). + */ +public class MCPSearchServlet extends HttpServlet { + + private static final long serialVersionUID = 433609077273989355L; + + private static final String JSONRPC_VERSION = "2.0"; + private static final String MCP_PROTOCOL_VERSION = "2024-11-05"; + private static final String TOOL_NAME = "search"; + private static final int DEFAULT_RESULT_COUNT = 10; + private static final int MAX_RESULT_COUNT = 100; + + @Override + public void service(final ServletRequest request, final ServletResponse response) throws ServletException, IOException { + final HttpServletRequest hrequest = (HttpServletRequest) request; + final HttpServletResponse hresponse = (HttpServletResponse) response; + + hresponse.setContentType("application/json;charset=utf-8"); + hresponse.setHeader("Access-Control-Allow-Origin", "*"); + hresponse.setHeader("Access-Control-Allow-Methods", "POST, GET, OPTIONS, DELETE"); + hresponse.setHeader("Access-Control-Allow-Headers", "Content-Type, Authorization"); + + final Method reqMethod = Method.getMethod(hrequest.getMethod()); + if (reqMethod == Method.OTHER) { + hresponse.setStatus(HttpServletResponse.SC_OK); + return; + } + + final String body = readBody(request); + Object parsed = null; + if (body.length() == 0) { + parsed = new JSONObject(); + } else try { + final JSONTokener tokener = new JSONTokener(body); + parsed = tokener.nextValue(); + } catch (JSONException e) { + writeJsonResponse(hresponse, errorResponse(JSONObject.NULL, -32700, e.getMessage())); + } + + try { + if (parsed instanceof JSONObject) { + if (((JSONObject) parsed).optString("method", "").length() == 0) { + String uri = hrequest.getRequestURI(); + ((JSONObject) parsed).put("method", uri.substring(1)); + } + final JSONObject responseObject = handleRequest((JSONObject) parsed); + if (responseObject != null) { + writeJsonResponse(hresponse, responseObject); + } else { + hresponse.setStatus(HttpServletResponse.SC_NO_CONTENT); + } + } else if (parsed instanceof JSONArray) { + final JSONArray requestArray = (JSONArray) parsed; + final JSONArray responseArray = new JSONArray(); + for (int i = 0; i < requestArray.length(); i++) { + final Object entry = requestArray.get(i); + if (entry instanceof JSONObject) { + final JSONObject responseObject = handleRequest((JSONObject) entry); + if (responseObject != null) { + responseArray.put(responseObject); + } + } + } + if (responseArray.length() > 0) { + writeJsonResponse(hresponse, responseArray); + } else { + hresponse.setStatus(HttpServletResponse.SC_NO_CONTENT); + } + } else { + writeJsonResponse(hresponse, errorResponse(JSONObject.NULL, -32600, "Invalid JSON-RPC payload")); + } + } catch (JSONException e) { + writeJsonResponse(hresponse, errorResponse(JSONObject.NULL, -32700, e.getMessage())); + } + } + + private JSONObject handleRequest(final JSONObject requestObject) { + final Object id = requestObject.optString("id", "0"); + final String jsonrpc = requestObject.optString("jsonrpc", JSONRPC_VERSION); + final String method = requestObject.optString("method", ""); + + if (!JSONRPC_VERSION.equals(jsonrpc)) { + return errorResponse(id, -32600, "Unsupported JSON-RPC version"); + } + + if (method == null || method.isEmpty()) { + return errorResponse(id, -32600, "Missing method"); + } + + if (id == JSONObject.NULL || id == null) { + // Notification: acknowledge silently + if ("notifications/ping".equals(method)) { + return null; + } + // No response for other notifications either + return null; + } + + switch (method) { + case "initialize": + return handleInitialize(id, requestObject.optJSONObject("params")); + case "tools/list": + return handleToolsList(id); + case "tools/call": + return handleToolsCall(id, requestObject.optJSONObject("params")); + default: + return errorResponse(id, -32601, "Unknown method: " + method); + } + } + + private JSONObject handleInitialize(final Object id, final JSONObject params) { + final JSONObject result = new JSONObject(true); + try { + final JSONObject serverInfo = new JSONObject(true); + serverInfo.put("name", "YaCy MCP Web Search"); + serverInfo.put("version", "1.0"); + + final JSONObject capabilities = new JSONObject(true); + final JSONObject toolsCapability = new JSONObject(true); + toolsCapability.put("list", true); + toolsCapability.put("call", true); + capabilities.put("tools", toolsCapability); + + result.put("protocolVersion", MCP_PROTOCOL_VERSION); + result.put("serverInfo", serverInfo); + result.put("capabilities", capabilities); + } catch (JSONException e) { + return errorResponse(id, -32603, e.getMessage()); + } + return successResponse(id, result); + } + + private JSONObject handleToolsList(final Object id) { + try { + final JSONObject tool = new JSONObject(true); + tool.put("name", TOOL_NAME); + tool.put("description", "Search the YaCy index and return the most relevant web results."); + + final JSONObject inputProperties = new JSONObject(true); + final JSONObject querySchema = new JSONObject(true); + querySchema.put("type", "string"); + querySchema.put("description", "Search query to execute against the YaCy index."); + inputProperties.put("query", querySchema); + + final JSONObject limitSchema = new JSONObject(true); + limitSchema.put("type", "integer"); + limitSchema.put("minimum", 1); + limitSchema.put("maximum", MAX_RESULT_COUNT); + limitSchema.put("description", "Maximum number of results to return."); + inputProperties.put("limit", limitSchema); + + final JSONObject includeSnippetSchema = new JSONObject(true); + includeSnippetSchema.put("type", "boolean"); + includeSnippetSchema.put("description", "Include text snippets extracted from the indexed document content."); + inputProperties.put("include_snippet", includeSnippetSchema); + + final JSONObject inputSchema = new JSONObject(true); + inputSchema.put("type", "object"); + inputSchema.put("properties", inputProperties); + final JSONArray required = new JSONArray(); + required.put("query"); + inputSchema.put("required", required); + + tool.put("inputSchema", inputSchema); + + final JSONObject outputSchema = new JSONObject(true); + outputSchema.put("type", "object"); + final JSONObject outputProperties = new JSONObject(true); + final JSONObject resultsSchema = new JSONObject(true); + resultsSchema.put("type", "array"); + resultsSchema.put("description", "Ordered list of search results."); + outputProperties.put("results", resultsSchema); + outputSchema.put("properties", outputProperties); + tool.put("outputSchema", outputSchema); + + final JSONArray tools = new JSONArray(); + tools.put(tool); + + final JSONObject result = new JSONObject(true); + result.put("tools", tools); + return successResponse(id, result); + } catch (JSONException e) { + return errorResponse(id, -32603, e.getMessage()); + } + } + + private JSONObject handleToolsCall(final Object id, final JSONObject params) { + if (params == null) { + return errorResponse(id, -32602, "Missing params"); + } + final String name = params.optString("name", ""); + if (!TOOL_NAME.equals(name)) { + return errorResponse(id, -32601, "Unknown tool: " + name); + } + + final JSONObject arguments = params.optJSONObject("arguments"); + if (arguments == null) { + return errorResponse(id, -32602, "Missing tool arguments"); + } + + final String query = arguments.optString("query", "").trim(); + if (query.isEmpty()) { + return errorResponse(id, -32602, "Tool argument 'query' must be a non-empty string"); + } + + int limit = arguments.optInt("limit", DEFAULT_RESULT_COUNT); + if (limit <= 0) { + limit = DEFAULT_RESULT_COUNT; + } + limit = Math.min(limit, MAX_RESULT_COUNT); + final boolean includeSnippet = arguments.optBoolean("include_snippet", true); + + JSONArray results; + results = RAGProxyServlet.searchResults(query, limit, includeSnippet); + + try { + final JSONObject payload = new JSONObject(true); + payload.put("results", results); + + final JSONObject contentItem = new JSONObject(true); + contentItem.put("type", "json"); + contentItem.put("json", payload); + + final JSONArray content = new JSONArray(); + content.put(contentItem); + + final JSONObject result = new JSONObject(true); + result.put("content", content); + return successResponse(id, result); + } catch (JSONException e) { + return errorResponse(id, -32603, e.getMessage()); + } + } + + private static String readBody(final ServletRequest request) throws IOException { + final StringBuilder builder = new StringBuilder(); + try (BufferedReader reader = request.getReader()) { + String line; + while ((line = reader.readLine()) != null) { + builder.append(line); + } + } + return builder.toString(); + } + + private static JSONObject successResponse(final Object id, final JSONObject result) { + final JSONObject response = new JSONObject(true); + try { + response.put("jsonrpc", JSONRPC_VERSION); + response.put("id", id); + response.put("result", result); + } catch (JSONException e) { + // As this method is only called with valid JSON objects we rethrow as unchecked + throw new IllegalStateException("Failed to construct success response", e); + } + return response; + } + + private static JSONObject errorResponse(final Object id, final int code, final String message) { + final JSONObject response = new JSONObject(true); + try { + response.put("jsonrpc", JSONRPC_VERSION); + response.put("id", id); + final JSONObject error = new JSONObject(true); + error.put("code", code); + error.put("message", message); + response.put("error", error); + } catch (JSONException e) { + throw new IllegalStateException("Failed to construct error response", e); + } + return response; + } + + private static void writeJsonResponse(final HttpServletResponse response, final Object payload) throws IOException { + final String serialized = payload instanceof JSONObject ? ((JSONObject) payload).toString() + : payload instanceof JSONArray ? ((JSONArray) payload).toString() : payload.toString(); + response.getWriter().write(serialized); + } +} diff --git a/source/net/yacy/http/servlets/RAGProxyServlet.java b/source/net/yacy/http/servlets/RAGProxyServlet.java index 638bd7372..bbc891c75 100644 --- a/source/net/yacy/http/servlets/RAGProxyServlet.java +++ b/source/net/yacy/http/servlets/RAGProxyServlet.java @@ -28,8 +28,8 @@ import java.net.HttpURLConnection; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.util.ArrayList; import java.util.Iterator; -import java.util.LinkedHashMap; import javax.servlet.ServletException; import javax.servlet.ServletOutputStream; @@ -49,6 +49,7 @@ import org.json.JSONException; import org.json.JSONObject; import net.yacy.ai.OpenAIClient; +import net.yacy.cora.federate.solr.SolrType; import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector; import net.yacy.search.Switchboard; import net.yacy.search.schema.CollectionSchema; @@ -141,18 +142,16 @@ public class RAGProxyServlet extends HttpServlet { String query = this.searchWordsForPrompt(LLM_QUERY_MODEL, user); out.print(responseLine("Searching for '" + query + "'\n\n").toString() + "\n"); out.flush(); - LinkedHashMap<String, String> searchResults = searchResults(query, 4); - out.print(responseLine("Using the following sources for RAG:\n\n").toString() + "\n"); - out.flush(); - for (String s : searchResults.keySet()) { - out.print(responseLine("- `" + s + "`\n").toString() + "\n"); - out.flush(); - } + JSONArray searchResults = searchResults(query, 4, true); out.print(responseLine("\n").toString()); out.flush(); system += LLM_SYSTEM_PREFIX; user += LLM_USER_PREFIX; - for (String s : searchResults.values()) user += s + "\n\n"; + for (int i = 0; i < searchResults.length(); i++) { + JSONObject r = searchResults.getJSONObject(i); + String snippet = r.optString("snippet", ""); + user += snippet + "\n\n"; + } systemObject.put("content", system); userObject.put("content", user); @@ -200,9 +199,9 @@ public class RAGProxyServlet extends HttpServlet { } } - public static LinkedHashMap<String, String> searchResults(String query, int count) { - LinkedHashMap<String, String> a = new LinkedHashMap<>(); - if (query == null || query.length() == 0 || count == 0) return a; + public static JSONArray searchResults(String query, int count, final boolean includeSnippet) { + final JSONArray results = new JSONArray(); + if (query == null || query.length() == 0 || count == 0) return results; Switchboard sb = Switchboard.getSwitchboard(); EmbeddedSolrConnector connector = sb.index.fulltext().getDefaultEmbeddedConnector(); // construct query @@ -213,7 +212,7 @@ public class RAGProxyServlet extends HttpServlet { params.setFacet(false); params.clearSorts(); params.setFields(CollectionSchema.sku.getSolrFieldName(), CollectionSchema.text_t.getSolrFieldName()); - params.setIncludeScore(false); + params.setIncludeScore(true); params.set("df", CollectionSchema.text_t.getSolrFieldName()); // query the server @@ -221,15 +220,37 @@ public class RAGProxyServlet extends HttpServlet { final SolrDocumentList sdl = connector.getDocumentListByParams(params); Iterator<SolrDocument> i = sdl.iterator(); while (i.hasNext()) { - SolrDocument doc = i.next(); - String url = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName()); - String text = (String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName()); - a.put(url, text); + try { + SolrDocument doc = i.next(); + final JSONObject result = new JSONObject(true); + String url = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName()); + result.put("url", url); + String title = getOneString(doc, CollectionSchema.title); + result.put("title", title == null ? url : title); + if (includeSnippet) { + String text = (String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName()); + result.put("snippet", text == null ? "" : text); + } + results.put(result); + } catch (JSONException e) { + // skip this result + } } - return a; + return results; } catch (SolrException | IOException e) { - return new LinkedHashMap<>(); + return results; + } + } + + private static String getOneString(SolrDocument doc, CollectionSchema field) { + assert field.isMultiValued(); + assert field.getType() == SolrType.string || field.getType() == SolrType.text_general; + Object r = doc.getFieldValue(field.getSolrFieldName()); + if (r == null) return ""; + if (r instanceof ArrayList) { + return ((ArrayList<String>) r).get(0); } + return r.toString(); } private String searchWordsForPrompt(String model, String prompt) { |