refactoring of llm classes

5 files changed, 357 insertions, 388 deletions

diff --git a/source/net/yacy/ai/LLM.java b/source/net/yacy/ai/LLM.java
new file mode 100644
index 000000000..760b975e2
--- /dev/null
+++ b/source/net/yacy/ai/LLM.java
@@ -0,0 +1,339 @@
+/**
+ *  LLMEndpoint
+ *  Copyright 2024 by Michael Peter Christen
+ *  First released 17.05.2024 at https://yacy.net
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public License
+ *  along with this program in the file lgpl21.txt
+ *  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.ai;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.AbstractMap;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+public class LLM {
+
+    private static String[] STOPTOKENS = new String[]{"[/INST]", "<|im_end|>", "<|end_of_turn|>", "<|eot_id|>", "<|end_header_id|>", "<EOS_TOKEN>", "</s>", "<|end|>"};
+
+    public static enum LLMType {
+        OPENAI("https://api.openai.com"),
+        OLLAMA("http://localhost:11434"),
+        LMSTUDIO("http://localhost:1234"),
+        OPENROUTER("https://openrouter.ai/api"),
+        OTHER(null);
+        public String hoststub;
+        private LLMType(String hoststub) {
+            this.hoststub = hoststub;
+        }
+    }
+    
+    public final String hoststub;
+    public final String key; // the apikey
+    public final int max_tokens; // the max_tokens as configured by the endpoint for all models
+    public final LLMType type;
+    
+    public LLM(final String hoststub, String key, final int max_tokens, final LLMType type) {
+        this.hoststub = hoststub.endsWith("/") ? hoststub.substring(0, hoststub.length() - 1) : hoststub;
+        this.key = key == null ? "" : key;
+        this.max_tokens = max_tokens <= 0 ? 4096 : max_tokens;
+        this.type = type;
+    }
+    
+    public String getHoststub() {
+		return this.hoststub;
+	}
+
+
+    // API Helper Methods
+
+    public static String sendPostRequest(final String urls, final JSONObject data) throws IOException, URISyntaxException {
+        final URL url = new URI(urls).toURL();
+        final HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+        conn.setRequestMethod("POST");
+        conn.setRequestProperty("Content-Type", "application/json");
+        conn.setDoOutput(true);
+
+        try (OutputStream os = conn.getOutputStream()) {
+            final byte[] input = data.toString().getBytes("utf-8");
+            os.write(input, 0, input.length);
+        }
+
+        final int responseCode = conn.getResponseCode();
+        if (responseCode == HttpURLConnection.HTTP_OK) {
+            try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8"))) {
+                final StringBuilder response = new StringBuilder();
+                String responseLine;
+                while ((responseLine = br.readLine()) != null) {
+                    response.append(responseLine.trim());
+                }
+                return response.toString();
+            }
+        } else {
+            throw new IOException("Request failed with response code " + responseCode);
+        }
+    }
+
+    public static String sendGetRequest(final String urls) throws IOException, URISyntaxException {
+        final URL url = new URI(urls).toURL();
+        final HttpURLConnection conn = (HttpURLConnection) url.openConnection();
+        conn.setRequestMethod("GET");
+
+        final int responseCode = conn.getResponseCode();
+        if (responseCode == HttpURLConnection.HTTP_OK) {
+            try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8"))) {
+                final StringBuilder response = new StringBuilder();
+                String responseLine;
+                while ((responseLine = br.readLine()) != null) {
+                    response.append(responseLine.trim());
+                }
+                return response.toString();
+            }
+        } else {
+            throw new IOException("Request failed with response code " + responseCode);
+        }
+    }
+    
+    
+    public LinkedHashMap<String, Long> listOllamaModels() {
+        final LinkedHashMap<String, Long> sortedMap = new LinkedHashMap<>();
+        try {
+            final String response = sendGetRequest(this.hoststub + "/api/tags");
+            final JSONObject responseObject = new JSONObject(response);
+            final JSONArray models = responseObject.getJSONArray("models");
+
+            final List<Map.Entry<String, Long>> list = new ArrayList<>();
+            for (int i = 0; i < models.length(); i++) {
+                final JSONObject model = models.getJSONObject(i);
+                final String name = model.optString("name", "");
+                final long size = model.optLong("size", 0);
+                list.add(new AbstractMap.SimpleEntry<>(name, size));
+            }
+
+            // Sort the list in descending order based on the values
+            list.sort((o1, o2) -> o2.getValue().compareTo(o1.getValue()));
+
+            // Create a new LinkedHashMap and add the sorted entries
+            for (final Map.Entry<String, Long> entry : list) {
+                sortedMap.put(entry.getKey(), entry.getValue());
+            }
+        } catch (JSONException | URISyntaxException | IOException e) {
+            e.printStackTrace();
+        }
+        return sortedMap;
+    }
+
+    public boolean ollamaModelExists(final String name) {
+        final JSONObject data = new JSONObject();
+        try {
+            data.put("name", name);
+            sendPostRequest(this.hoststub + "/api/show", data);
+            return true;
+        } catch (JSONException | URISyntaxException | IOException e) {
+            return false;
+        }
+    }
+
+    public boolean pullOllamaModel(final String name) {
+        final JSONObject data = new JSONObject();
+        try {
+            data.put("name", name);
+            data.put("stream", false);
+            final String response = sendPostRequest(this.hoststub + "/api/pull", data);
+            // this sends {"status": "success"} in case of success
+            final JSONObject responseObject = new JSONObject(response);
+            final String status = responseObject.optString("status", "");
+            return status.equals("success");
+        } catch (JSONException | URISyntaxException | IOException e) {
+            return false;
+        }
+    }
+    
+    // chat endpoints
+    
+    public static class Context extends JSONArray {
+        public Context(String systemPrompt) throws JSONException {
+            super();
+            final JSONObject systemPromptObject = new JSONObject(true);
+            systemPromptObject.put("role", "system");
+            systemPromptObject.put("content", systemPrompt);
+            this.put(systemPromptObject);
+        }
+        public void addDialog(String user, String assistant) throws JSONException {
+            final JSONObject userPromptObject = new JSONObject(true);
+            userPromptObject.put("role", "user");
+            userPromptObject.put("content", user);
+            this.put(userPromptObject);
+            final JSONObject assistantPromptObject = new JSONObject(true);
+            assistantPromptObject.put("role", "assistant");
+            assistantPromptObject.put("content", assistant);
+            this.put(assistantPromptObject);
+        }
+        public void addPrompt(String userPrompt) throws JSONException {
+            final JSONObject userPromptObject = new JSONObject(true);
+            userPromptObject.put("role", "user");
+            userPromptObject.put("content", userPrompt);
+            this.put(userPromptObject);
+        }
+    }
+
+    // OpenAI chat client, works with llama.cpp and Ollama
+    public String chat(final String model, final Context context, JSONObject schema, final int max_tokens) throws IOException {
+        final JSONObject data = new JSONObject();
+        
+        try {
+            data.put("model", model);
+            data.put("temperature", 0.1);
+            data.put("max_tokens", max_tokens);
+            data.put("messages", context);
+            data.put("stop", new JSONArray(STOPTOKENS));
+            data.put("stream", false);
+
+            if (schema != null) {
+                System.out.println(schema.toString());
+                JSONObject json_schema = new JSONObject(true);
+                json_schema.put("strict", true);
+                json_schema.put("schema", schema);
+                JSONObject response_format = new JSONObject();
+                response_format.put("type", "json_schema");
+                response_format.put("json_schema", json_schema);            
+                data.put("response_format", response_format);
+            }
+            
+            final String response = sendPostRequest(this.hoststub + "/v1/chat/completions", data);
+            final JSONObject responseObject = new JSONObject(response);
+            final JSONArray choices = responseObject.getJSONArray("choices");
+            final JSONObject choice = choices.getJSONObject(0);
+            final JSONObject message = choice.getJSONObject("message");
+            final String content = message.optString("content", "");
+            return content;
+        } catch (JSONException | URISyntaxException e) {
+            throw new IOException(e.getMessage());
+        }
+    }
+    
+    public String chat(final String model, final String systemPrompt, final String userPrompt, final int max_tokens) throws IOException {
+        try {
+            Context context = new Context(systemPrompt);
+            context.addPrompt(userPrompt);
+            return chat(model, context, null, max_tokens);
+        } catch (JSONException e) {
+            throw new IOException(e.getMessage());
+        }
+    }
+    
+    public static String[] stringsFromChat(String chatanswer) throws JSONException {
+        JSONArray ja = new JSONArray(chatanswer);
+        List<String> list = new ArrayList<>();
+        // parse the JSON array and extract strings
+        for (int i = 0; i < ja.length(); i++) {
+            Object item = ja.get(i);
+            if (item instanceof String) {
+                list.add((String) item);
+            } else if (item instanceof JSONObject) {
+                JSONObject jo = (JSONObject) item;
+                String answer = jo.optString("answer", null);
+                if (answer != null) {
+                    list.add(answer);
+                } else {
+                    // take any string value from the object
+                    for (String key : jo.keySet()) {
+                        Object value = jo.optString(key, null);
+                        if (value != null && value instanceof String) {
+                            list.add((String) value);
+                            break; // take the first string found
+                        }
+                    }
+                }
+            }
+        }
+        // convert the list to an array
+        String[] result = new String[list.size()];
+        return list.toArray(result);        
+    }
+    
+    public final static JSONObject listSchema = new JSONObject(Map.of(
+        "title", "Answer List",
+        "type", "array",
+        "properties", Map.of(
+            "answer", Map.of("type", "string")
+        ),
+        "required", List.of("answer")
+    ));
+    
+    public static void main(final String[] args) {
+        final LLM llm = new LLM(LLMType.OLLAMA.hoststub, null, 4069, LLMType.OLLAMA);
+
+        final LinkedHashMap<String, Long> models = llm.listOllamaModels();
+        System.out.println(models.toString());
+
+        // check if model exists
+        final String model = "qwen2.5:0.5b";
+        if (llm.ollamaModelExists(model))
+            System.out.println("model " + model + " exists");
+        else
+            System.out.println("model " + model + " does not exist");
+
+        // pull a model
+        final boolean success = llm.pullOllamaModel(model);
+        System.out.println("pulled model: " + model + ": " + success);
+        
+        String response;
+		try {
+			response = llm.chat(model, "You are a helpful assistant.", "What is the capital of France?", 1000);
+	        System.out.println("Chat response: " + response);
+		} catch (IOException e) {
+	
+			e.printStackTrace();
+		}
+
+        // make chat completion with model
+        String question = "Who invented the wheel?";
+        try {
+            final String answer = llm.chat(model, "Make short answers.", question, 200);
+            System.out.println(answer);
+        } catch (final IOException e) {
+            e.printStackTrace();
+        }
+
+        // try the json parser from chat results
+        question = "Make a list of four names from Star Wars movies. Use a JSON Array.";
+        try {
+            Context context = new Context("Make short answers");
+            context.addPrompt(question);
+            final String[] a = stringsFromChat(llm.chat(model, context, listSchema, 1000));
+            for (String s : a) {
+                System.out.println(s);
+            }
+        } catch (final IOException | JSONException e) {
+            e.printStackTrace();
+        }
+    }
+    
+}
diff --git a/source/net/yacy/ai/OllamaSwarm.java b/source/net/yacy/ai/LLMSwarm.java
index 9206f13dd..9aa3286b2 100644
--- a/source/net/yacy/ai/OllamaSwarm.java
+++ b/source/net/yacy/ai/LLMSwarm.java
@@ -28,26 +28,22 @@ import java.util.Map;
 import java.util.Random;
 
 
-public class OllamaSwarm {
+public class LLMSwarm {
 	
 	Random random = new Random();
-	List<OllamaClient> swarm;;
+	List<LLM> swarm;;
 
-	public OllamaSwarm() {
+	public LLMSwarm() {
         this.swarm = new ArrayList<>();
 	}
 
-	public void addOllamaClient(final OllamaClient client) {
+	public void addOllamaClient(final LLM client) {
 		this.swarm.add(client);
 	}
 	
-	public void addOllamaClient(final String hoststub) {
-		this.swarm.add(new OllamaClient(hoststub));
-	}
-	
 	public List<String> getSwarm() {
 		List<String> hoststubs = new ArrayList<>();
-		for (OllamaClient client : this.swarm) {
+		for (LLM client : this.swarm) {
 			hoststubs.add(client.getHoststub());
 		}
 		return hoststubs;
@@ -59,7 +55,7 @@ public class OllamaSwarm {
 	 */
     public LinkedHashMap<String, Integer> listOllamaModels() {
     	final LinkedHashMap<String, Integer> modelCountMap = new LinkedHashMap<>();
-		for (OllamaClient client: this.swarm) {
+		for (LLM client: this.swarm) {
 			LinkedHashMap<String, Long> models = client.listOllamaModels();
 			for (Map.Entry<String, Long> entry : models.entrySet()) {
 				modelCountMap.merge(entry.getKey(), 1, Integer::sum);
@@ -74,7 +70,7 @@ public class OllamaSwarm {
      * @return true if at least one client has the model, false otherwise
      */
     public boolean ollamaModelExists(final String name) {
-    	for (OllamaClient client : this.swarm) {
+    	for (LLM client : this.swarm) {
 			if (client.ollamaModelExists(name)) {
 				return true;
 			}
@@ -91,7 +87,7 @@ public class OllamaSwarm {
     public boolean pullOllamaModel(final String name, boolean all) {
     	boolean pulled = false;
     	if (all) {
-			for (OllamaClient client: this.swarm) {
+			for (LLM client: this.swarm) {
 				if (client.pullOllamaModel(name)) {
 					pulled = true;
 				}
@@ -100,7 +96,7 @@ public class OllamaSwarm {
 			// we try several times in case one client is not available
 			for (int i = 0; i < this.swarm.size(); i++) {
 				int index = random.nextInt(this.swarm.size());
-				OllamaClient client = this.swarm.get(index);
+				LLM client = this.swarm.get(index);
 				pulled = client.pullOllamaModel(name);
 				if (pulled) return true;
 			}
@@ -113,9 +109,9 @@ public class OllamaSwarm {
      * @param name
      * @return a List of OllamaClient objects that have the specified model
      */
-    public List<OllamaClient> getSwarm(String name) {
-		List<OllamaClient> clientsWithModel = new ArrayList<>();
-		for (OllamaClient client : this.swarm) {
+    public List<LLM> getSwarm(String name) {
+		List<LLM> clientsWithModel = new ArrayList<>();
+		for (LLM client : this.swarm) {
 			if (client.ollamaModelExists(name)) {
 				clientsWithModel.add(client);
 			}
@@ -132,11 +128,11 @@ public class OllamaSwarm {
      * @throws IOException
      */
     public String getAnswer(String model, String systemPrompt, String userPrompt) throws IOException {
-		List<OllamaClient> clientsWithModel = getSwarm(model);
+		List<LLM> clientsWithModel = getSwarm(model);
 		if (clientsWithModel.isEmpty()) {
 			return "No client with model " + model + " found.";
 		}
-		OllamaClient client = clientsWithModel.get(random.nextInt(clientsWithModel.size()));
+		LLM client = clientsWithModel.get(random.nextInt(clientsWithModel.size()));
 		return client.chat(model, systemPrompt, userPrompt, 4096);
 	}
     
diff --git a/source/net/yacy/ai/OllamaClient.java b/source/net/yacy/ai/OllamaClient.java
deleted file mode 100644
index 53aa66742..000000000
--- a/source/net/yacy/ai/OllamaClient.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- *  OllamaClient
- *  Copyright 2024 by Michael Peter Christen
- *  First released 17.05.2024 at https://yacy.net
- *
- *  This library is free software; you can redistribute it and/or
- *  modify it under the terms of the GNU Lesser General Public
- *  License as published by the Free Software Foundation; either
- *  version 2.1 of the License, or (at your option) any later version.
- *
- *  This library is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  Lesser General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public License
- *  along with this program in the file lgpl21.txt
- *  If not, see <http://www.gnu.org/licenses/>.
- */
-
-package net.yacy.ai;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.util.AbstractMap;
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
-
-public class OllamaClient extends OpenAIClient {
-
-    public static String OLLAMA_API_HOST  = "http://localhost:11434";
-
-    public OllamaClient(final String hoststub) {
-        super(hoststub);
-    }
-    
-    public String getHoststub() {
-		return this.hoststub;
-	}
-
-    public LinkedHashMap<String, Long> listOllamaModels() {
-        final LinkedHashMap<String, Long> sortedMap = new LinkedHashMap<>();
-        try {
-            final String response = OpenAIClient.sendGetRequest(this.hoststub + "/api/tags");
-            final JSONObject responseObject = new JSONObject(response);
-            final JSONArray models = responseObject.getJSONArray("models");
-
-            final List<Map.Entry<String, Long>> list = new ArrayList<>();
-            for (int i = 0; i < models.length(); i++) {
-                final JSONObject model = models.getJSONObject(i);
-                final String name = model.optString("name", "");
-                final long size = model.optLong("size", 0);
-                list.add(new AbstractMap.SimpleEntry<>(name, size));
-            }
-
-            // Sort the list in descending order based on the values
-            list.sort((o1, o2) -> o2.getValue().compareTo(o1.getValue()));
-
-            // Create a new LinkedHashMap and add the sorted entries
-            for (final Map.Entry<String, Long> entry : list) {
-                sortedMap.put(entry.getKey(), entry.getValue());
-            }
-        } catch (JSONException | URISyntaxException | IOException e) {
-            e.printStackTrace();
-        }
-        return sortedMap;
-    }
-
-    public boolean ollamaModelExists(final String name) {
-        final JSONObject data = new JSONObject();
-        try {
-            data.put("name", name);
-            OpenAIClient.sendPostRequest(this.hoststub + "/api/show", data);
-            return true;
-        } catch (JSONException | URISyntaxException | IOException e) {
-            return false;
-        }
-    }
-
-    public boolean pullOllamaModel(final String name) {
-        final JSONObject data = new JSONObject();
-        try {
-            data.put("name", name);
-            data.put("stream", false);
-            final String response = OpenAIClient.sendPostRequest(this.hoststub + "/api/pull", data);
-            // this sends {"status": "success"} in case of success
-            final JSONObject responseObject = new JSONObject(response);
-            final String status = responseObject.optString("status", "");
-            return status.equals("success");
-        } catch (JSONException | URISyntaxException | IOException e) {
-            return false;
-        }
-    }
-    
-    public static void main(final String[] args) {
-        final OllamaClient oc = new OllamaClient(OLLAMA_API_HOST);
-
-        final LinkedHashMap<String, Long> models = oc.listOllamaModels();
-        System.out.println(models.toString());
-
-        // check if model exists
-        final String model = "qwen2.5:0.5b";
-        if (oc.ollamaModelExists(model))
-            System.out.println("model " + model + " exists");
-        else
-            System.out.println("model " + model + " does not exist");
-
-        // pull a model
-        final boolean success = oc.pullOllamaModel(model);
-        System.out.println("pulled model: " + model + ": " + success);
-        
-        String response;
-		try {
-			response = oc.chat(model, "You are a helpful assistant.", "What is the capital of France?", 1000);
-	        System.out.println("Chat response: " + response);
-		} catch (IOException e) {
-	
-			e.printStackTrace();
-		}
-
-    }
-    
-}
diff --git a/source/net/yacy/ai/OpenAIClient.java b/source/net/yacy/ai/OpenAIClient.java
deleted file mode 100644
index cee112495..000000000
--- a/source/net/yacy/ai/OpenAIClient.java
+++ /dev/null
@@ -1,237 +0,0 @@
-/**
- *  OpenAIClient
- *  Copyright 2024 by Michael Peter Christen
- *  First released 17.05.2024 at https://yacy.net
- *
- *  This library is free software; you can redistribute it and/or
- *  modify it under the terms of the GNU Lesser General Public
- *  License as published by the Free Software Foundation; either
- *  version 2.1 of the License, or (at your option) any later version.
- *
- *  This library is distributed in the hope that it will be useful,
- *  but WITHOUT ANY WARRANTY; without even the implied warranty of
- *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- *  Lesser General Public License for more details.
- *
- *  You should have received a copy of the GNU Lesser General Public License
- *  along with this program in the file lgpl21.txt
- *  If not, see <http://www.gnu.org/licenses/>.
- */
-
-package net.yacy.ai;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.net.HttpURLConnection;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
-
-
-public class OpenAIClient {
-
-    private static String[] STOPTOKENS = new String[]{"[/INST]", "<|im_end|>", "<|end_of_turn|>", "<|eot_id|>", "<|end_header_id|>", "<EOS_TOKEN>", "</s>", "<|end|>"};
-
-    protected final String hoststub;
-
-    public OpenAIClient(final String hoststub) {
-        this.hoststub = hoststub;
-    }
-
-
-    // API Helper Methods
-
-    public static String sendPostRequest(final String endpoint, final JSONObject data) throws IOException, URISyntaxException {
-        final URL url = new URI(endpoint).toURL();
-        final HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-        conn.setRequestMethod("POST");
-        conn.setRequestProperty("Content-Type", "application/json");
-        conn.setDoOutput(true);
-
-        try (OutputStream os = conn.getOutputStream()) {
-            final byte[] input = data.toString().getBytes("utf-8");
-            os.write(input, 0, input.length);
-        }
-
-        final int responseCode = conn.getResponseCode();
-        if (responseCode == HttpURLConnection.HTTP_OK) {
-            try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8"))) {
-                final StringBuilder response = new StringBuilder();
-                String responseLine;
-                while ((responseLine = br.readLine()) != null) {
-                    response.append(responseLine.trim());
-                }
-                return response.toString();
-            }
-        } else {
-            throw new IOException("Request failed with response code " + responseCode);
-        }
-    }
-
-    public static String sendGetRequest(final String endpoint) throws IOException, URISyntaxException {
-        final URL url = new URI(endpoint).toURL();
-        final HttpURLConnection conn = (HttpURLConnection) url.openConnection();
-        conn.setRequestMethod("GET");
-
-        final int responseCode = conn.getResponseCode();
-        if (responseCode == HttpURLConnection.HTTP_OK) {
-            try (BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream(), "utf-8"))) {
-                final StringBuilder response = new StringBuilder();
-                String responseLine;
-                while ((responseLine = br.readLine()) != null) {
-                    response.append(responseLine.trim());
-                }
-                return response.toString();
-            }
-        } else {
-            throw new IOException("Request failed with response code " + responseCode);
-        }
-    }
-    
-    public static class Context extends JSONArray {
-    	public Context(String systemPrompt) throws JSONException {
-    		super();
-    		final JSONObject systemPromptObject = new JSONObject(true);
-    		systemPromptObject.put("role", "system");
-        	systemPromptObject.put("content", systemPrompt);
-        	this.put(systemPromptObject);
-    	}
-    	public void addDialog(String user, String assistant) throws JSONException {
-            final JSONObject userPromptObject = new JSONObject(true);
-        	userPromptObject.put("role", "user");
-        	userPromptObject.put("content", user);
-            this.put(userPromptObject);
-            final JSONObject assistantPromptObject = new JSONObject(true);
-            assistantPromptObject.put("role", "assistant");
-            assistantPromptObject.put("content", assistant);
-            this.put(assistantPromptObject);
-    	}
-    	public void addPrompt(String userPrompt) throws JSONException {
-            final JSONObject userPromptObject = new JSONObject(true);
-        	userPromptObject.put("role", "user");
-        	userPromptObject.put("content", userPrompt);
-            this.put(userPromptObject);
-    	}
-    }
-
-    // OpenAI chat client, works with llama.cpp and Ollama
-
-    public String chat(final String model, final Context context, JSONObject schema, final int max_tokens) throws IOException {
-        final JSONObject data = new JSONObject();
-        
-        try {
-            data.put("model", model);
-            data.put("temperature", 0.1);
-            data.put("max_tokens", max_tokens);
-            data.put("messages", context);
-            data.put("stop", new JSONArray(STOPTOKENS));
-            data.put("stream", false);
-
-            if (schema != null) {
-            	System.out.println(schema.toString());
-	            JSONObject json_schema = new JSONObject(true);
-	            json_schema.put("strict", true);
-	            json_schema.put("schema", schema);
-	            JSONObject response_format = new JSONObject();
-	            response_format.put("type", "json_schema");
-	            response_format.put("json_schema", json_schema);            
-	            data.put("response_format", response_format);
-            }
-            
-            final String response = sendPostRequest(this.hoststub + "/v1/chat/completions", data);
-            final JSONObject responseObject = new JSONObject(response);
-            final JSONArray choices = responseObject.getJSONArray("choices");
-            final JSONObject choice = choices.getJSONObject(0);
-            final JSONObject message = choice.getJSONObject("message");
-            final String content = message.optString("content", "");
-            return content;
-        } catch (JSONException | URISyntaxException e) {
-            throw new IOException(e.getMessage());
-        }
-    }
-    
-    public String chat(final String model, final String systemPrompt, final String userPrompt, final int max_tokens) throws IOException {
-        try {
-        	Context context = new Context(systemPrompt);
-        	context.addPrompt(userPrompt);
-        	return chat(model, context, null, max_tokens);
-        } catch (JSONException e) {
-            throw new IOException(e.getMessage());
-        }
-    }
-    
-    public static String[] stringsFromChat(String chatanswer) throws JSONException {
-    	JSONArray ja = new JSONArray(chatanswer);
-    	List<String> list = new ArrayList<>();
-    	// parse the JSON array and extract strings
-    	for (int i = 0; i < ja.length(); i++) {
-			Object item = ja.get(i);
-			if (item instanceof String) {
-				list.add((String) item);
-			} else if (item instanceof JSONObject) {
-				JSONObject jo = (JSONObject) item;
-				String answer = jo.optString("answer", null);
-				if (answer != null) {
-					list.add(answer);
-				} else {
-					// take any string value from the object
-					for (String key : jo.keySet()) {
-						Object value = jo.optString(key, null);
-						if (value != null && value instanceof String) {
-							list.add((String) value);
-							break; // take the first string found
-						}
-					}
-				}
-			}
-		}
-		// convert the list to an array
-		String[] result = new String[list.size()];
-		return list.toArray(result);    	
-    }
-    
-    public final static JSONObject listSchema = new JSONObject(Map.of(
-        "title", "Answer List",
-        "type", "array",
-        "properties", Map.of(
-            "answer", Map.of("type", "string")
-        ),
-        "required", List.of("answer")
-    ));
-
-    public static void main(final String[] args) {
-        final String model = "qwen2.5:0.5b";
-        final OpenAIClient oaic = new OpenAIClient(OllamaClient.OLLAMA_API_HOST);
-        // make chat completion with model
-        String question = "Who invented the wheel?";
-        try {
-            final String answer = oaic.chat(model, "Make short answers.", question, 200);
-            System.out.println(answer);
-        } catch (final IOException e) {
-            e.printStackTrace();
-        }
-
-        // try the json parser from chat results
-        question = "Make a list of four names from Star Wars movies. Use a JSON Array.";
-        try {
-        	Context context = new Context("Make short answers");
-        	context.addPrompt(question);
-            final String[] a = stringsFromChat(oaic.chat(model, context, listSchema, 1000));
-            for (String s : a) {
-				System.out.println(s);
-			}
-        } catch (final IOException | JSONException e) {
-            e.printStackTrace();
-        }
-    }
-
-}
diff --git a/source/net/yacy/http/servlets/RAGProxyServlet.java b/source/net/yacy/http/servlets/RAGProxyServlet.java
index bbc891c75..63c28c284 100644
--- a/source/net/yacy/http/servlets/RAGProxyServlet.java
+++ b/source/net/yacy/http/servlets/RAGProxyServlet.java
@@ -48,7 +48,7 @@ import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
 
-import net.yacy.ai.OpenAIClient;
+import net.yacy.ai.LLM;
 import net.yacy.cora.federate.solr.SolrType;
 import net.yacy.cora.federate.solr.connector.EmbeddedSolrConnector;
 import net.yacy.search.Switchboard;
@@ -257,10 +257,10 @@ public class RAGProxyServlet extends HttpServlet {
         StringBuilder query = new StringBuilder();
         String question = "Make a list of a maximum of four search words for the following question; use a JSON Array: " + prompt;
         try {
-            OpenAIClient oaic = new OpenAIClient(LLM_API_HOST);
-            OpenAIClient.Context context = new OpenAIClient.Context(LLM_SYSTEM_PREFIX);
+            LLM llm = new LLM(LLM_API_HOST, null, 4096, LLM.LLMType.OLLAMA);
+            LLM.Context context = new LLM.Context(LLM_SYSTEM_PREFIX);
             context.addPrompt(question);
-            String[] a = OpenAIClient.stringsFromChat(oaic.chat(model, context, OpenAIClient.listSchema, 80));
+            String[] a = LLM.stringsFromChat(llm.chat(model, context, LLM.listSchema, 80));
             for (String s : a)
                 query.append(s).append(' ');
             return query.toString().trim();