Merge branch 'master' of https://github.com/yacy/yacy_search_server

3 files changed, 128 insertions, 35 deletions

diff --git a/source/net/yacy/ai/llama3/Tensor/AbstractFloatTensor.java b/source/net/yacy/ai/llama3/Tensor/AbstractFloatTensor.java
index afe9fef73..c4d3749ce 100644
--- a/source/net/yacy/ai/llama3/Tensor/AbstractFloatTensor.java
+++ b/source/net/yacy/ai/llama3/Tensor/AbstractFloatTensor.java
@@ -138,6 +138,7 @@ public abstract class AbstractFloatTensor implements FloatTensor {
         LongStream.range(startInclusive, endExclusive).parallel().forEach(action);
     }
     
+    /*
     public float dot(final int thisOffset, final FloatTensor that, final int thatOffset, final int size) {
         float result = 0f;
         for (int j = 0; j < size; j++) {
@@ -145,6 +146,39 @@ public abstract class AbstractFloatTensor implements FloatTensor {
         }
         return result;
     }
+    */
+
+    public float dot(final int thisOffset,
+            final FloatTensor that,
+            final int thatOffset,
+            final int size) {
+
+        float sum0 = 0f, sum1 = 0f, sum2 = 0f, sum3 = 0f;
+
+        int i = thisOffset;
+        int k = thatOffset;
+
+        // Loop-Unrolling
+        final int limit = size & ~3;
+        for (int j = 0; j < limit; j += 4) {
+            sum0 += this.getFloat(i    ) * that.getFloat(k    );
+            sum1 += this.getFloat(i + 1) * that.getFloat(k + 1);
+            sum2 += this.getFloat(i + 2) * that.getFloat(k + 2);
+            sum3 += this.getFloat(i + 3) * that.getFloat(k + 3);
+            i += 4;
+            k += 4;
+        }
+
+        float result = sum0 + sum1 + sum2 + sum3;
+
+        // remaining values
+        for (int j = limit; j < size; j++) {
+            result += this.getFloat(j) * that.getFloat(j);
+        }
+
+        return result;
+    }
+
 
     public void matmul(final FloatTensor that, final FloatTensor out, final int dim0, final int dim1) {
         parallelFor(0, dim0, i -> out.setFloat(i, dot(i * dim1, that, 0, dim1)));
diff --git a/source/net/yacy/ai/llama3/Tensor/DirectBufferFloatTensor.java b/source/net/yacy/ai/llama3/Tensor/DirectBufferFloatTensor.java
index c123b5efc..4a3db4d03 100644
--- a/source/net/yacy/ai/llama3/Tensor/DirectBufferFloatTensor.java
+++ b/source/net/yacy/ai/llama3/Tensor/DirectBufferFloatTensor.java
@@ -31,7 +31,8 @@ import net.yacy.ai.llama3.Model.GGMLType;
 public class DirectBufferFloatTensor extends AbstractFloatTensor implements FloatTensor {
 
     final ByteBuffer byteBuffer; // must be direct
-
+    //private static final VarHandle VH = MethodHandles.byteBufferViewVarHandle(int[].class, ByteOrder.nativeOrder());
+    
     public DirectBufferFloatTensor(ByteBuffer bb) {
         if (bb.isDirect()) {
             this.byteBuffer = bb.slice().order(bb.order());
@@ -66,29 +67,77 @@ public class DirectBufferFloatTensor extends AbstractFloatTensor implements Floa
 
     @Override
     public final float getFloat(final int index) {
-        final int i = this.byteBuffer.getInt(index << 2);
-        //final int base = index << 2;
-        //int i = (this.byteBuffer.get(base) & 0xFF) | ((this.byteBuffer.get(base + 1) & 0xFF) << 8) | ((this.byteBuffer.get(base + 2) & 0xFF) << 16) | ((this.byteBuffer.get(base + 3) & 0xFF) << 24);
-        return Float.intBitsToFloat(i);
+        return this.byteBuffer.getFloat(index << 2);
+        //return Float.intBitsToFloat(this.byteBuffer.getInt(index << 2));
+        //return Float.intBitsToFloat((int) VH.get(this.byteBuffer, index << 2));
     }
 
     @Override
     public final void setFloat(final int index, final float value) {
-        final int i = Float.floatToRawIntBits(value);
-        this.byteBuffer.putInt(index << 2, i);
-        //int base = index << 2;
-        //this.byteBuffer.put(base++, (byte) ( i        & 0xFF)); // Little-endian:
-        //this.byteBuffer.put(base++, (byte) ((i >> 8)  & 0xFF));
-        //this.byteBuffer.put(base++, (byte) ((i >> 16) & 0xFF));
-        //this.byteBuffer.put(base, (byte) ((i >> 24) & 0xFF));
+        this.byteBuffer.putFloat(index << 2, value);
+        //VH.set(this.byteBuffer, index << 2, Float.floatToRawIntBits(value));
+        //this.byteBuffer.putInt(index << 2, Float.floatToRawIntBits(value));
     }
-    
+
     @Override
     public final GGMLType type() {
         return GGMLType.F32;
     }
+    
+    public float dot(final int thisOffset,
+            final FloatTensor that,
+            final int thatOffset,
+            final int size) {
+
+        float sum0 = 0f, sum1 = 0f, sum2 = 0f, sum3 = 0f;
+        final int limit = size & ~3;
+
+        if (that instanceof DirectBufferFloatTensor) {
+
+            DirectBufferFloatTensor thatb = (DirectBufferFloatTensor) that;
+            
+            int i = thisOffset << 2;
+            int k = thatOffset << 2;
+    
+            // Loop-Unrolling
+            for (int j = 0; j < limit; j += 4) {
+                sum0 += this.byteBuffer.getFloat(i     ) * thatb.byteBuffer.getFloat(k     );
+                sum1 += this.byteBuffer.getFloat(i +  4) * thatb.byteBuffer.getFloat(k +  4);
+                sum2 += this.byteBuffer.getFloat(i +  8) * thatb.byteBuffer.getFloat(k +  8);
+                sum3 += this.byteBuffer.getFloat(i + 12) * thatb.byteBuffer.getFloat(k + 12);
+                i += 16;
+                k += 16;
+            }
+            
+        } else {
+        
+            int i = thisOffset << 2;
+            int k = thatOffset;
+    
+            // Loop-Unrolling
+            for (int j = 0; j < limit; j += 4) {
+                sum0 += this.byteBuffer.getFloat(i     ) * that.getFloat(k    );
+                sum1 += this.byteBuffer.getFloat(i +  4) * that.getFloat(k + 1);
+                sum2 += this.byteBuffer.getFloat(i +  8) * that.getFloat(k + 2);
+                sum3 += this.byteBuffer.getFloat(i + 12) * that.getFloat(k + 3);
+                i += 16;
+                k += 4;
+            }
+    
+        }
+
+        float result = sum0 + sum1 + sum2 + sum3;
+
+        // remaining values
+        for (int j = limit; j < size; j++) {
+            result += this.byteBuffer.getFloat(j << 2) * that.getFloat(j);
+        }
+        
+        return result;
+    }
 
     /*
+    
     @Override
     public final FloatTensor fillInPlace(final int thisOffset, final int size, final float value) {
         int end = thisOffset + size;
diff --git a/source/net/yacy/http/servlets/RAGProxyServlet.java b/source/net/yacy/http/servlets/RAGProxyServlet.java
index cc1c14514..0545b3d40 100644
--- a/source/net/yacy/http/servlets/RAGProxyServlet.java
+++ b/source/net/yacy/http/servlets/RAGProxyServlet.java
@@ -141,8 +141,8 @@ public class RAGProxyServlet extends HttpServlet {
             
             // get messages
             JSONArray messages = bodyObject.optJSONArray("messages");
-            JSONObject systemObject = messages.getJSONObject(0);
-            String system = systemObject.optString("content", ""); // the system prompt
+            //JSONObject systemObject = messages.getJSONObject(0);
+            //String system = systemObject.optString("content", ""); // the system prompt
             JSONObject userObject = messages.getJSONObject(messages.length() - 1);
             String user = userObject.optString("content", ""); // this is the latest prompt
 
@@ -150,19 +150,9 @@ public class RAGProxyServlet extends HttpServlet {
             if (rag) {
                 // modify system and user prompt here in bodyObject to enable RAG
                 String query = this.searchWordsForPrompt(llm4tldr.llm, llm4tldr.model, user);
-                out.print(responseLine("Searching for '" + query + "'\n\n").toString() + "\n");
-                out.flush();
-                JSONArray searchResults = searchResults(query, 4, true);
-                out.print(responseLine("\n").toString());
-                out.flush();
-                system += LLM_SYSTEM_PREFIX;
+                String searchResultMarkdown = searchResultsAsMarkdown(query, 4);
                 user += LLM_USER_PREFIX;
-                for (int i  = 0; i < searchResults.length(); i++) {
-                    JSONObject r = searchResults.getJSONObject(i);
-                    String snippet = r.optString("snippet", "");
-                    user += snippet + "\n\n";
-                }
-                systemObject.put("content", system);
+                user += searchResultMarkdown;
                 userObject.put("content", user);
     
             }
@@ -184,21 +174,20 @@ public class RAGProxyServlet extends HttpServlet {
             try (OutputStream os = conn.getOutputStream()) {
                 os.write(body.getBytes());
                 os.flush();
-            }
+            } // here we wait for the response from upstream
 
             // write back response of the back-end service to the client; use status of
             // backend-response
-            int status = conn.getResponseCode();
+            final int status = conn.getResponseCode();
             // String rmessage = conn.getResponseMessage();
             hresponse.setStatus(status);
 
             if (status == 200) {
                 // read the response of the back-end line-by-line and write it to the client line-by-line
-                BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream()));
+                final BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream()));
                 String inputLine;
                 while ((inputLine = in.readLine()) != null) {
-                    out.print(inputLine); // i.e. data:
-                                          // {"id":"chatcmpl-69","object":"chat.completion.chunk","created":1715908287,"model":"llama3:8b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"ߘŠ"},"finish_reason":null}]}
+                    out.print(inputLine); // i.e. data: {"id":"chatcmpl-69","object":"chat.completion.chunk","created":1715908287,"model":"llama3:8b","system_fingerprint":"fp_ollama","choices":[{"index":0,"delta":{"role":"assistant","content":"ߘŠ"},"finish_reason":null}]}
                     out.flush();
                 }
                 in.close();
@@ -234,12 +223,12 @@ public class RAGProxyServlet extends HttpServlet {
                     SolrDocument doc = i.next();
                     final JSONObject result = new JSONObject(true);
                     String url = (String) doc.getFieldValue(CollectionSchema.sku.getSolrFieldName());
-                    result.put("url", url);
+                    result.put("url", url == null ? "" : url.trim());
                     String title = getOneString(doc, CollectionSchema.title);
-                    result.put("title", title == null ? url : title);
+                    result.put("title", title == null ? "" : title.trim());
                     if (includeSnippet) {
                         String text = (String) doc.getFieldValue(CollectionSchema.text_t.getSolrFieldName());
-                        result.put("snippet", text == null ? "" : text);
+                        result.put("snippet", text == null ? "" : text.trim());
                     }
                 results.put(result);
                 } catch (JSONException e) {
@@ -252,6 +241,27 @@ public class RAGProxyServlet extends HttpServlet {
         }
     }
     
+    public static String searchResultsAsMarkdown(String query, int count) {
+        JSONArray searchResults = searchResults(query, count, true);
+        StringBuilder sb = new StringBuilder();
+        
+        for (int i  = 0; i < searchResults.length(); i++) {
+            try {
+                JSONObject r = searchResults.getJSONObject(i);
+                String title = r.optString("title", "");
+                String url = r.optString("url", "");
+                String snippet = r.optString("snippet", "");
+                if (title.length() > 0 && snippet.length() > 0) {
+                    sb.append("## ").append(title).append("\n");
+                    sb.append(snippet).append("\n");
+                    if (url.length() > 0) sb.append("Source: ").append(url).append("\n");
+                    sb.append("\n\n");
+                }
+            } catch (JSONException e) {}
+        }
+        return sb.toString();
+    }
+    
     private static String getOneString(SolrDocument doc, CollectionSchema field) {
         assert field.isMultiValued();
         assert field.getType() == SolrType.string || field.getType() == SolrType.text_general;