diff options
35 files changed, 3981 insertions, 62 deletions
diff --git a/.github/workflows/ant-build.yaml b/.github/workflows/ant-build.yaml new file mode 100644 index 000000000..35cb574c9 --- /dev/null +++ b/.github/workflows/ant-build.yaml @@ -0,0 +1,63 @@ +name: YaCy CI/CD Pipeline + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + workflow_dispatch: + +jobs: + build-and-release: + runs-on: ubuntu-latest + env: + RELEASE_DIR: RELEASE + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Required for git version info in build + + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + java-version: '11' + distribution: 'temurin' + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -yq ant git wkhtmltopdf imagemagick xvfb ghostscript + + - name: Configure runtime settings + run: | + sed -i "/adminAccountBase64MD5=/c\adminAccountBase64MD5=MD5:8cffbc0d66567a0987a4aba1ec46d63c" defaults/yacy.init + sed -i "/adminAccountForLocalhost=/c\adminAccountForLocalhost=false" defaults/yacy.init + sed -i "/server.https=false/c\server.https=true" defaults/yacy.init + + - name: Build with Ant + run: ant clean all dist + + - name: Verify tarball creation + run: | + ls -la ${{ env.RELEASE_DIR }}/ + tar -ztvf ${{ env.RELEASE_DIR }}/yacy_*.tar.gz | head + + - name: Upload build artifact + uses: actions/upload-artifact@v4 + with: + name: yacy-release + path: ${{ env.RELEASE_DIR }}/yacy_*.tar.gz + retention-days: 5 + + - name: Create GitHub Release (on tag) + if: startsWith(github.ref, 'refs/tags/v') + uses: softprops/action-gh-release@v1 + with: + files: ${{ env.RELEASE_DIR }}/yacy_*.tar.gz + body: | + YaCy Search Server Release + Built from ${{ github.sha }} + Version: ${{ github.ref_name }} + draft: false + prerelease: false diff --git a/docker/Dockerfile b/docker/Dockerfile index ca201b1f7..3967b0728 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,17 +6,14 @@ # run with # docker run -d --name yacy -p 8090:8090 -p 8443:8443 -v yacy_data:/opt/yacy_search_server/DATA --log-opt max-size=200m --log-opt max-file=2 yacy/yacy_search_server:latest + ## builder image -FROM eclipse-temurin:21-jdk AS builder -RUN apt-get update && apt-get install -y --no-install-recommends \ - ant \ - git \ - curl \ - && rm -rf /var/lib/apt/lists/* +FROM eclipse-temurin:24-jdk-noble AS builder +RUN apt-get update && apt-get install -y --no-install-recommends ant git curl && rm -rf /var/lib/apt/lists/* +# compile YaCy WORKDIR /opt COPY . /opt/yacy_search_server/ - RUN ant compile -f /opt/yacy_search_server/build.xml # Set initial admin password: "yacy" (encoded with custom yacy md5 function net.yacy.cora.order.Digest.encodeMD5Hex()) @@ -26,16 +23,11 @@ RUN sed -i "/adminAccountBase64MD5=/c\adminAccountBase64MD5=MD5:8cffbc0d66567a09 sed -i "/server.https=false/c\server.https=true" /opt/yacy_search_server/defaults/yacy.init ## build final image -FROM eclipse-temurin:21-jre AS app - -RUN apt-get update && apt-get install -y --no-install-recommends \ - wkhtmltopdf \ - xvfb \ - ghostscript \ - && rm -rf /var/lib/apt/lists/* - +FROM eclipse-temurin:24-jdk-noble AS app +RUN apt-get update && apt-get install -y --no-install-recommends wkhtmltopdf xvfb ghostscript && rm -rf /var/lib/apt/lists/* LABEL maintainer="Michael Peter Christen <mc@yacy.net>" +# copy YaCy to app image RUN adduser --system --group --no-create-home --disabled-password yacy WORKDIR /opt COPY --chown=yacy:yacy --from=builder /opt/yacy_search_server /opt/yacy_search_server diff --git a/htroot/IndexImportZim_p.html b/htroot/IndexImportZim_p.html new file mode 100644 index 000000000..68d74762e --- /dev/null +++ b/htroot/IndexImportZim_p.html @@ -0,0 +1,53 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml"> + <head> + <title>YaCy '#[clientname]#': ZIM File Import</title> + #%env/templates/metas.template%# + #(import)#::<meta http-equiv="REFRESH" content="10;url=IndexImportZim_p.html" /> + <!-- the url= removes http get parameters on refresh, preventing restart of import --> + #(/import)# + </head> + <body id="IndexImportZim"> + #%env/templates/header.template%# + #%env/templates/submenuIndexImport.template%# + <h2>ZIM File Import</h2> + + #(import)# + <p>No import thread is running, you can start a new thread here</p> + <form action="IndexImportZim_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8"> + <!-- no post method here, we don't want to transmit the whole file, only the path--> + <fieldset> + <legend>Zim File Selection: select a '.zim' file</legend> + <p> + You can download ZIM files for example here + <a href="https://download.kiwix.org/zim/" target="_blank">Kiwix ZIM Archive</a>. + </p> + <dl> + <dt class="TableCellDark"><label for="file">File:</label></dt> + <dd><input name="file" id="file" type="file" value="" size="75" /></dd> + <dt></dt> + <dd><input name="submit" class="btn btn-primary" type="submit" value="Import ZIM File" /></dd> + </dl> + </fieldset> + </form> + + <br /> + :: + <form> + <fieldset><legend>Import Process</legend> + <dl> + <dt>Thread:</dt><dd>#[thread]#</dd> + <dt>ZIM File:</dt><dd>#[zimfile]#</dd> + <dt>Processed:</dt><dd>#[count]# Entries</dd> + <dt>Speed:</dt><dd>#[speed]# pages per second</dd> + <dt>Running Time:</dt><dd>#[runningHours]# hours, #[runningMinutes]# minutes</dd> + <dt>Remaining Time:</dt><dd>#[remainingHours]# hours, #[remainingMinutes]# minutes</dd> + </dl> + </fieldset> + <input name="abort" type="submit" class="btn btn-danger" value="Stop"/> + </form> + #(/import)# + + #%env/templates/footer.template%# + </body> +</html>
\ No newline at end of file diff --git a/htroot/env/templates/submenuIndexImport.template b/htroot/env/templates/submenuIndexImport.template index 39a1029a3..ccbc49412 100644 --- a/htroot/env/templates/submenuIndexImport.template +++ b/htroot/env/templates/submenuIndexImport.template @@ -14,6 +14,7 @@ <li><a href="Load_RSS_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">RSS Feed Importer</a></li> <li><a href="IndexImportOAIPMH_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">OAI-PMH Importer</a></li> <li><a href="IndexImportWarc_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Warc Importer</a></li> + <li><a href="IndexImportZim_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Zim Importer</a></li> <li><a href="IndexImportJsonList_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">JsonList Importer</a></li> </ul> </div> @@ -15,7 +15,7 @@ <dependency org="com.drewnoakes" name="metadata-extractor" rev="2.19.0" /> <dependency org="com.fasterxml.jackson.core" name="jackson-databind" rev="2.18.2"/> <dependency org="com.github.ben-manes.caffeine" name="caffeine" rev="3.1.8"/> - <dependency org="com.google.guava" name="guava" rev="33.3.1-jre" conf="compile->master"/> + <dependency org="com.google.guava" name="guava" rev="33.4.8-jre" conf="compile->master"/> <dependency org="com.google.guava" name="failureaccess" rev="1.0.2" /> <dependency org="com.ibm.icu" name="icu4j" rev="76.1"/> <dependency org="com.github.mwiede" name="jsch" rev="0.2.21" /> diff --git a/source/net/yacy/ai/OllamaClient.java b/source/net/yacy/ai/OllamaClient.java index 9eaba920b..53aa66742 100644 --- a/source/net/yacy/ai/OllamaClient.java +++ b/source/net/yacy/ai/OllamaClient.java @@ -32,15 +32,17 @@ import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; -public class OllamaClient { +public class OllamaClient extends OpenAIClient { public static String OLLAMA_API_HOST = "http://localhost:11434"; - private final String hoststub; - public OllamaClient(final String hoststub) { - this.hoststub = hoststub; + super(hoststub); } + + public String getHoststub() { + return this.hoststub; + } public LinkedHashMap<String, Long> listOllamaModels() { final LinkedHashMap<String, Long> sortedMap = new LinkedHashMap<>(); @@ -95,7 +97,7 @@ public class OllamaClient { return false; } } - + public static void main(final String[] args) { final OllamaClient oc = new OllamaClient(OLLAMA_API_HOST); @@ -103,7 +105,7 @@ public class OllamaClient { System.out.println(models.toString()); // check if model exists - final String model = "phi3:3.8b"; + final String model = "qwen2.5:0.5b"; if (oc.ollamaModelExists(model)) System.out.println("model " + model + " exists"); else @@ -111,7 +113,17 @@ public class OllamaClient { // pull a model final boolean success = oc.pullOllamaModel(model); - System.out.println("pulled model + " + model + ": " + success); + System.out.println("pulled model: " + model + ": " + success); + + String response; + try { + response = oc.chat(model, "You are a helpful assistant.", "What is the capital of France?", 1000); + System.out.println("Chat response: " + response); + } catch (IOException e) { + + e.printStackTrace(); + } } + } diff --git a/source/net/yacy/ai/OpenAIClient.java b/source/net/yacy/ai/OpenAIClient.java index f068eadd2..cee112495 100644 --- a/source/net/yacy/ai/OpenAIClient.java +++ b/source/net/yacy/ai/OpenAIClient.java @@ -28,6 +28,9 @@ import java.net.HttpURLConnection; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import org.json.JSONArray; import org.json.JSONException; @@ -38,7 +41,7 @@ public class OpenAIClient { private static String[] STOPTOKENS = new String[]{"[/INST]", "<|im_end|>", "<|end_of_turn|>", "<|eot_id|>", "<|end_header_id|>", "<EOS_TOKEN>", "</s>", "<|end|>"}; - private final String hoststub; + protected final String hoststub; public OpenAIClient(final String hoststub) { this.hoststub = hoststub; @@ -93,27 +96,57 @@ public class OpenAIClient { throw new IOException("Request failed with response code " + responseCode); } } + + public static class Context extends JSONArray { + public Context(String systemPrompt) throws JSONException { + super(); + final JSONObject systemPromptObject = new JSONObject(true); + systemPromptObject.put("role", "system"); + systemPromptObject.put("content", systemPrompt); + this.put(systemPromptObject); + } + public void addDialog(String user, String assistant) throws JSONException { + final JSONObject userPromptObject = new JSONObject(true); + userPromptObject.put("role", "user"); + userPromptObject.put("content", user); + this.put(userPromptObject); + final JSONObject assistantPromptObject = new JSONObject(true); + assistantPromptObject.put("role", "assistant"); + assistantPromptObject.put("content", assistant); + this.put(assistantPromptObject); + } + public void addPrompt(String userPrompt) throws JSONException { + final JSONObject userPromptObject = new JSONObject(true); + userPromptObject.put("role", "user"); + userPromptObject.put("content", userPrompt); + this.put(userPromptObject); + } + } // OpenAI chat client, works with llama.cpp and Ollama - public String chat(final String model, final String prompt, final int max_tokens) throws IOException { + public String chat(final String model, final Context context, JSONObject schema, final int max_tokens) throws IOException { final JSONObject data = new JSONObject(); - final JSONArray messages = new JSONArray(); - final JSONObject systemPrompt = new JSONObject(true); - final JSONObject userPrompt = new JSONObject(true); - messages.put(systemPrompt); - messages.put(userPrompt); + try { - systemPrompt.put("role", "system"); - systemPrompt.put("content", "Make short answers."); - userPrompt.put("role", "user"); - userPrompt.put("content", prompt); data.put("model", model); data.put("temperature", 0.1); data.put("max_tokens", max_tokens); - data.put("messages", messages); + data.put("messages", context); data.put("stop", new JSONArray(STOPTOKENS)); data.put("stream", false); + + if (schema != null) { + System.out.println(schema.toString()); + JSONObject json_schema = new JSONObject(true); + json_schema.put("strict", true); + json_schema.put("schema", schema); + JSONObject response_format = new JSONObject(); + response_format.put("type", "json_schema"); + response_format.put("json_schema", json_schema); + data.put("response_format", response_format); + } + final String response = sendPostRequest(this.hoststub + "/v1/chat/completions", data); final JSONObject responseObject = new JSONObject(response); final JSONArray choices = responseObject.getJSONArray("choices"); @@ -125,28 +158,63 @@ public class OpenAIClient { throw new IOException(e.getMessage()); } } - - public static String[] stringsFromChat(final String answer) { - final int p = answer.indexOf('['); - final int q = answer.indexOf(']'); - if (p < 0 || q < 0 || q < p) return new String[0]; + + public String chat(final String model, final String systemPrompt, final String userPrompt, final int max_tokens) throws IOException { try { - final JSONArray a = new JSONArray(answer.substring(p, q + 1)); - final String[] arr = new String[a.length()]; - for (int i = 0; i < a.length(); i++) arr[i] = a.getString(i); - return arr; - } catch (final JSONException e) { - return new String[0]; + Context context = new Context(systemPrompt); + context.addPrompt(userPrompt); + return chat(model, context, null, max_tokens); + } catch (JSONException e) { + throw new IOException(e.getMessage()); } } + + public static String[] stringsFromChat(String chatanswer) throws JSONException { + JSONArray ja = new JSONArray(chatanswer); + List<String> list = new ArrayList<>(); + // parse the JSON array and extract strings + for (int i = 0; i < ja.length(); i++) { + Object item = ja.get(i); + if (item instanceof String) { + list.add((String) item); + } else if (item instanceof JSONObject) { + JSONObject jo = (JSONObject) item; + String answer = jo.optString("answer", null); + if (answer != null) { + list.add(answer); + } else { + // take any string value from the object + for (String key : jo.keySet()) { + Object value = jo.optString(key, null); + if (value != null && value instanceof String) { + list.add((String) value); + break; // take the first string found + } + } + } + } + } + // convert the list to an array + String[] result = new String[list.size()]; + return list.toArray(result); + } + + public final static JSONObject listSchema = new JSONObject(Map.of( + "title", "Answer List", + "type", "array", + "properties", Map.of( + "answer", Map.of("type", "string") + ), + "required", List.of("answer") + )); public static void main(final String[] args) { - final String model = "phi3:3.8b"; + final String model = "qwen2.5:0.5b"; final OpenAIClient oaic = new OpenAIClient(OllamaClient.OLLAMA_API_HOST); // make chat completion with model String question = "Who invented the wheel?"; try { - final String answer = oaic.chat(model, question, 80); + final String answer = oaic.chat(model, "Make short answers.", question, 200); System.out.println(answer); } catch (final IOException e) { e.printStackTrace(); @@ -155,9 +223,13 @@ public class OpenAIClient { // try the json parser from chat results question = "Make a list of four names from Star Wars movies. Use a JSON Array."; try { - final String[] a = stringsFromChat(oaic.chat(model, question, 80)); - for (final String s: a) System.out.println(s); - } catch (final IOException e) { + Context context = new Context("Make short answers"); + context.addPrompt(question); + final String[] a = stringsFromChat(oaic.chat(model, context, listSchema, 1000)); + for (String s : a) { + System.out.println(s); + } + } catch (final IOException | JSONException e) { e.printStackTrace(); } } diff --git a/source/net/yacy/ai/llama3/ChatFormat.java b/source/net/yacy/ai/llama3/ChatFormat.java new file mode 100644 index 000000000..871b83d05 --- /dev/null +++ b/source/net/yacy/ai/llama3/ChatFormat.java @@ -0,0 +1,130 @@ +/** + * ChatFormat.java + + * This file was extracted from the llama3/qwen2 projects + * https://github.com/mukel/llama3.java + * https://github.com/mukel/qwen2.svm.java + * + * License: MIT License + * + * Copyright (c) 2024 Andrej Karpathy (for llama2.c) + * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2) + * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp) + * Copyright (c) 2025 Michael Peter Christen for modifications: + * The code was modified to fit the YaCy AI project: + * - back-port to Java 11 (removal of Vector API operations and record types) + * - removal of interactive mode and system.out printing + * - separation of the classes in the single java and refactoring + * - run-time performance optimizations for dot product computation of quantized values + * - joining of llama3/qwen2 into one code base; multi-arch options + * - alignment with code from https://github.com/ggml-org/llama.cpp/ + */ + +package net.yacy.ai.llama3; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import net.yacy.ai.llama3.Model.Tokenizer; + +/** + * Utility tailored for Llama 3 instruct prompt format. + */ +public class ChatFormat { + + final Tokenizer tokenizer; + final int beginOfText; + final int endHeader; + final int startHeader; + final int endOfTurn; + final int endOfText; + final int endOfMessage; + final Set<Integer> stopTokens; + + public ChatFormat(Tokenizer tokenizer) { + this.tokenizer = tokenizer; + Map<String, Integer> specialTokens = this.tokenizer.getSpecialTokens(); + this.beginOfText = specialTokens.get("<|begin_of_text|>"); + this.startHeader = specialTokens.get("<|start_header_id|>"); + this.endHeader = specialTokens.get("<|end_header_id|>"); + this.endOfTurn = specialTokens.get("<|eot_id|>"); + this.endOfText = specialTokens.get("<|end_of_text|>"); + this.endOfMessage = specialTokens.getOrDefault("<|eom_id|>", -1); // only in 3.1 + this.stopTokens = Set.of(endOfText, endOfTurn); + } + + public Tokenizer getTokenizer() { + return tokenizer; + } + + public Set<Integer> getStopTokens() { + return stopTokens; + } + + public List<Integer> encodeHeader(ChatFormat.Message message) { + List<Integer> tokens = new ArrayList<>(); + tokens.add(startHeader); + tokens.addAll(this.tokenizer.encodeAsList(message.role().name())); + tokens.add(endHeader); + tokens.addAll(this.tokenizer.encodeAsList("\n")); + return tokens; + } + + public List<Integer> encodeMessage(ChatFormat.Message message) { + List<Integer> tokens = this.encodeHeader(message); + tokens.addAll(this.tokenizer.encodeAsList(message.content().strip())); + tokens.add(endOfTurn); + return tokens; + } + + public List<Integer> encodeDialogPrompt(boolean appendAssistantTurn, List<ChatFormat.Message> dialog) { + List<Integer> tokens = new ArrayList<>(); + tokens.add(beginOfText); + for (ChatFormat.Message message : dialog) { + tokens.addAll(this.encodeMessage(message)); + } + if (appendAssistantTurn) { + // Add the start of an assistant message for the model to complete. + tokens.addAll(this.encodeHeader(new ChatFormat.Message(ChatFormat.Role.ASSISTANT, ""))); + } + return tokens; + } + + public static final class Message { + private final ChatFormat.Role role; + private final String content; + + public Message(ChatFormat.Role role, String content) { + this.role = role; + this.content = content; + } + + public ChatFormat.Role role() { + return role; + } + + public String content() { + return content; + } + + } + + public static final class Role { + private final String name; + + public static final Role SYSTEM = new Role("system"); + public static final Role USER = new Role("user"); + public static final Role ASSISTANT = new Role("assistant"); + + public Role(String name) { + this.name = name; + } + + public String name() { + return name; + } + + } +}
\ No newline at end of file diff --git a/source/net/yacy/ai/llama3/Context.java b/source/net/yacy/ai/llama3/Context.java new file mode 100644 index 000000000..b4cfbe1b2 --- /dev/null +++ b/source/net/yacy/ai/llama3/Context.java @@ -0,0 +1,61 @@ +/** + * Context + * Copyright 2025 by Michael Peter Christen + * First released 25.05.2025 at https://yacy.net + * + ** This class was not part of the original llama3 implementation, + ** but added later by the author to support different architectures. + ** It therefore does not inherit the llama3 copyright. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.ai.llama3; + + +public final class Context { + + public final String prompt; + public final String systemPrompt; + public final float temp; + public final float topp; + public final long seed; + public final int maxTokens; + + public static final int DEFAULT_MAX_TOKENS = 512; + + /** + * Create a context. + * + * @param prompt the prompt to use for the model + * @param systemPrompt a system prompt to use for the model + * @param temp temperature for sampling, must be >= 0; 0 means greedy sampling (deterministic) + * @param topp if 0 <= topp <= 1, use top-p (nucleus) sampling; otherwise, use categorical sampling + * @param seed random seed for sampling; use 0 for a random seed + * @param maxTokens maximum number of tokens to generate; use 0 for no limit (default is 512) + */ + public Context(String prompt, String systemPrompt, + float temp, float topp, long seed, int maxTokens) { + assert(0 <= temp): "temperature must be positive"; + assert(0 <= topp && topp <= 1): "top-p must be between 0 and 1"; + + this.prompt = prompt; + this.systemPrompt = systemPrompt; + this.temp = temp; + this.topp = topp; + this.seed = seed; + this.maxTokens = maxTokens; + } +}
\ No newline at end of file diff --git a/source/net/yacy/ai/llama3/Llama.java b/source/net/yacy/ai/llama3/Llama.java new file mode 100644 index 000000000..e6d04bffd --- /dev/null +++ b/source/net/yacy/ai/llama3/Llama.java @@ -0,0 +1,427 @@ +/** + * Llama.java + + * This file was extracted from the llama3/qwen2 projects + * https://github.com/mukel/llama3.java + * https://github.com/mukel/qwen2.svm.java + * + * License: MIT License + * + * Copyright (c) 2024 Andrej Karpathy (for llama2.c) + * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2) + * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp) + * Copyright (c) 2025 Michael Peter Christen for modifications: + * The code was modified to fit the YaCy AI project: + * - back-port to Java 11 (removal of Vector API operations and record types) + * - removal of interactive mode and system.out printing + * - separation of the classes in the single java and refactoring + * - run-time performance optimizations for dot product computation of quantized values + * - joining of llama3/qwen2 into one code base; multi-arch options + * - alignment with code from https://github.com/ggml-org/llama.cpp/ + */ + +package net.yacy.ai.llama3; + +import java.nio.FloatBuffer; +import java.util.*; +import java.util.function.IntConsumer; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import net.yacy.ai.llama3.Model.Arch; +import net.yacy.ai.llama3.Model.Tokenizer; +import net.yacy.ai.llama3.Tensor.ArrayFloatTensor; +import net.yacy.ai.llama3.Tensor.FloatTensor; + +public final class Llama { + + private final Configuration configuration; + private final Tokenizer tokenizer; + private final Weights weights; + + public Llama(Configuration configuration, Tokenizer tokenizer, Weights weights) { + this.configuration = configuration; + this.tokenizer = tokenizer; + this.weights = weights; + } + + public Configuration configuration() { + return configuration; + } + + public Tokenizer tokenizer() { + return tokenizer; + } + + public Weights weights() { + return weights; + } + + public State createNewState(int batchsize) { + State state = new State(configuration(), batchsize); + state.latestToken = tokenizer().getSpecialTokens().get("<|begin_of_text|>"); + return state; + } + + public static final class Configuration { + public final Arch arch; // architecture + public final int dim; // transformer dimension + public final int hiddenDim; // for ffn layers + public final int numberOfLayers; // number of layers + public final int numberOfHeads; // number of query heads + public final int numberOfKeyValueHeads; // number of key/value heads (can be < query heads because of multiquery) + public final int vocabularySize; // vocabulary size, usually 256 (byte-level) + public final int contextLength; // max sequence length + public final boolean sharedWeights; + public final float rmsNormEps; + public final float ropeTheta; + public final int headSize; + + public Configuration(Arch arch, int dim, int hiddenDim, int numberOfLayers, int numberOfHeads, int numberOfKeyValueHeads, + int vocabularySize, int contextLength, boolean sharedWeights, float rmsNormEps, float ropeTheta) { + this.arch = arch; + this.dim = dim; + this.hiddenDim = hiddenDim; + this.numberOfLayers = numberOfLayers; + this.numberOfHeads = numberOfHeads; + this.numberOfKeyValueHeads = numberOfKeyValueHeads; + this.vocabularySize = vocabularySize; + this.contextLength = contextLength; + this.sharedWeights = sharedWeights; + this.rmsNormEps = rmsNormEps; + this.ropeTheta = ropeTheta; + this.headSize = dim / numberOfHeads; + } + + public Configuration withContextLength(int newContextLength) { + if (newContextLength < 0) { + return this; // no change + } + return new Configuration(this.arch, this.dim, this.hiddenDim, this.numberOfLayers, this.numberOfHeads, + this.numberOfKeyValueHeads, this.vocabularySize, newContextLength, + this.sharedWeights, this.rmsNormEps, this.ropeTheta); + } + + } + + public static final class Weights { + // token embedding table + public final FloatTensor token_embedding_table; // (vocab_size, dim) + // weights for rmsnorms + public final FloatBuffer[] rms_att_weight; // (layer, dim) rmsnorm weights + // weights for matmuls + public final FloatTensor[] wq; // (layer, n_heads * head_size) + public final FloatTensor[] wk; // (layer, n_kv_heads, head_size) + public final FloatTensor[] wv; // (layer, n_kv_heads * head_size) + public final FloatTensor[] wo; // (layer, n_heads * head_size, dim) + public final FloatTensor[] q_bias; // (layer, dim) + public final FloatTensor[] k_bias; // (layer, kv_dim) + public final FloatTensor[] v_bias; // (layer, kv_dim) + public final FloatBuffer[] rms_ffn_weight; // (layer, dim) + // weights for ffn + public final FloatTensor[] w1; // (layer, hidden_dim, dim) + public final FloatTensor[] w2; // (layer, dim, hidden_dim) + public final FloatTensor[] w3; // (layer, hidden_dim, dim) + // public final rmsnorm + public final FloatBuffer rms_final_weight; // (dim,) + // freq_cis for RoPE relatively positional embeddings + public final FloatBuffer freq_cis_real; // (seq_len, head_size/2) + public final FloatBuffer freq_cis_imag; // (seq_len, head_size/2) + // (optional) classifier weights for the logits, on the last layer + public final FloatTensor wcls; // (vocab_size, dim) + + public Weights(FloatTensor token_embedding_table, FloatBuffer[] rms_att_weight, FloatTensor[] wq, + FloatTensor[] wk, FloatTensor[] wv, + FloatTensor[] q_bias, FloatTensor[] k_bias, FloatTensor[] v_bias, + FloatTensor[] wo, FloatBuffer[] rms_ffn_weight, + FloatTensor[] w1, FloatTensor[] w2, FloatTensor[] w3, FloatBuffer rms_final_weight, + FloatBuffer freq_cis_real, FloatBuffer freq_cis_imag, FloatTensor wcls) { + this.token_embedding_table = token_embedding_table; + this.rms_att_weight = rms_att_weight; + this.wq = wq; + this.wk = wk; + this.wv = wv; + this.q_bias = q_bias; + this.k_bias = k_bias; + this.v_bias = v_bias; + this.wo = wo; + this.rms_ffn_weight = rms_ffn_weight; + this.w1 = w1; + this.w2 = w2; + this.w3 = w3; + this.rms_final_weight = rms_final_weight; + this.freq_cis_real = freq_cis_real; + this.freq_cis_imag = freq_cis_imag; + this.wcls = wcls; + } + + } + + public static final class State { + + // current wave of activations + public final int batchsize; + public final FloatTensor[] x; // activation at current time stamp (dim,) + public final FloatTensor[] xb; // same, but inside a residual branch (dim,) + public final FloatTensor[] xb2; // an additional buffer just for convenience (dim,) + public final FloatTensor[] hb; // buffer for hidden dimension in the ffn (hidden_dim,) + public final FloatTensor[] hb2; // buffer for hidden dimension in the ffn (hidden_dim,) + public final FloatTensor[] q; // query (dim,) + public final FloatTensor[] k; // key (dim,) + public final FloatTensor[] v; // value (dim,) + public final FloatTensor[] att; // buffer for scores/attention values (n_heads, seq_len) + public final FloatTensor logits; // output logits + + // kv cache + public final FloatTensor[] keyCache; // (n_layer, seq_len, kv_dim) + public final FloatTensor[] valueCache; // (n_layer, seq_len, kv_dim) + + /** last index in previous block */ + int idxPrevBlock; + + public int latestToken; + + State(Configuration config, int batchsize) { + this.batchsize = batchsize; + this.x = allocate(batchsize, config.dim); + this.xb = allocate(batchsize, config.dim); + this.xb2 = allocate(batchsize, config.dim); + this.hb = allocate(batchsize, config.hiddenDim); + this.hb2 = allocate(batchsize, config.hiddenDim); + this.q = allocate(batchsize, config.dim); + this.k = allocate(batchsize, config.dim); + this.v = allocate(batchsize, config.dim); + this.att = allocate(batchsize, config.numberOfHeads, config.contextLength); + idxPrevBlock = -1; + + this.logits = ArrayFloatTensor.allocate(config.vocabularySize); + int kvDim = (config.dim * config.numberOfKeyValueHeads) / config.numberOfHeads; + this.keyCache = Stream.generate(() -> ArrayFloatTensor.allocate(config.contextLength, kvDim)).limit(config.numberOfLayers).toArray(FloatTensor[]::new); + this.valueCache = Stream.generate(() -> ArrayFloatTensor.allocate(config.contextLength, kvDim)).limit(config.numberOfLayers).toArray(FloatTensor[]::new); + } + + private static FloatTensor[] allocate(int numTokens, int... dims) { + return IntStream.range(0, numTokens) + .mapToObj(i -> ArrayFloatTensor.allocate(dims)) + .toArray(FloatTensor[]::new); + } + + } + + static void rmsnorm(FloatTensor out, FloatTensor x, FloatBuffer weight, int size, float rmsNormEps) { + // calculate sum of squares + float ss = x.reduce(0, size, 0f, (acc, xi) -> acc + xi * xi); + ss /= size; + ss += rmsNormEps; + ss = (float) (1.0 / Math.sqrt(ss)); + // normalize and scale + for (int i = 0; i < size; ++i) { + out.setFloat(i, weight.get(i) * (ss * x.getFloat(i))); + } + } + + static FloatTensor forward(Llama model, State state, int[] tokens, int position, boolean computeLogits) { + // a few convenience variables + Configuration config = model.configuration(); + Weights weights = model.weights(); + int dim = config.dim; + int headSize = config.headSize; + int kvDim = (config.dim * config.numberOfKeyValueHeads) / config.numberOfHeads; + int kvMul = config.numberOfHeads / config.numberOfKeyValueHeads; // integer multiplier of the kv sharing in multiquery + float sqrtHeadSize = (float) Math.sqrt(headSize); + final int nTokens = tokens.length; + + // copy the token embedding into x + FloatTensor.parallelFor(0, nTokens, t -> + weights.token_embedding_table.copyTo(tokens[t] * dim, state.x[t], 0, dim) + ); + + // forward all the layers + for (int l = 0; l < config.numberOfLayers; l++) { + + // attention rmsnorm + final int curLayer = l; + FloatTensor.parallelFor(0, nTokens, t -> + rmsnorm(state.xb[t], state.x[t], weights.rms_att_weight[curLayer], dim, config.rmsNormEps) + ); + + // qkv matmuls for this position + weights.wq[l].matmul(nTokens, state.xb, state.q, dim, dim); + weights.wk[l].matmul(nTokens, state.xb, state.k, kvDim, dim); + weights.wv[l].matmul(nTokens, state.xb, state.v, kvDim, dim); + + // RoPE relative positional encoding: complex-valued rotate q and k in each head + FloatTensor.parallelFor(0, nTokens, t -> { + for (int i = 0; i < dim; i += 2) { + int head_dim = i % headSize; + float fcr = weights.freq_cis_real.get((position + t) * (headSize / 2) + (head_dim / 2)); + float fci = weights.freq_cis_imag.get((position + t) * (headSize / 2) + (head_dim / 2)); + int rotn = i < kvDim ? 2 : 1; // how many vectors? 2 = q & k, 1 = q only + for (int vi = 0; vi < rotn; vi++) { + FloatTensor vec = vi == 0 ? state.q[t] : state.k[t]; // the vector to rotate (query or key) + float v0 = vec.getFloat(i); + float v1 = vec.getFloat(i + 1); + vec.setFloat(i, v0 * fcr - v1 * fci); + vec.setFloat(i + 1, v0 * fci + v1 * fcr); + } + } + }); + + // save key,value at this time step (position) to our kv cache + FloatTensor.parallelFor(0, nTokens, t -> { + state.k[t].copyTo(0, state.keyCache[curLayer], (position + t) * kvDim, kvDim); + state.v[t].copyTo(0, state.valueCache[curLayer], (position + t) * kvDim, kvDim); + }); + + // If the logits are not required, the attention and FFN of the last layer can be skipped entirely. + if (!computeLogits && curLayer == config.numberOfLayers - 1) { + state.idxPrevBlock = nTokens - 1; + return null; + } + + // multihead attention. iterate over all heads + FloatTensor.parallelForLong(0, (long) nTokens * (long) config.numberOfHeads, ht -> { + int token = (int) (ht / config.numberOfHeads); + int h = (int) (ht % config.numberOfHeads); + int qOffset = h * headSize; + int attOffset = h * config.contextLength; + + for (int t = 0; t <= position + token; t++) { + int keyCacheOffset = t * kvDim + (h / kvMul) * headSize; + float score = state.q[token].dot(qOffset, state.keyCache[curLayer], keyCacheOffset, headSize); + score /= sqrtHeadSize; + state.att[token].setFloat(attOffset + t, score); + } + + state.att[token].softmaxInPlace(attOffset, position + token + 1); + + int xbOffset = h * headSize; + state.xb[token].fillInPlace(xbOffset, headSize, 0f); + + for (int t = 0; t <= position + token; t++) { + int vOffset = t * kvDim + (h / kvMul) * headSize; + float a = state.att[token].getFloat(attOffset + t); + state.xb[token].saxpyInPlace(xbOffset, state.valueCache[curLayer], vOffset, headSize, a); + } + }); + + // final matmul to get the output of the attention + weights.wo[l].matmul(nTokens, state.xb, state.xb2, dim, dim); + + // residual connection back into x + FloatTensor.parallelFor(0, nTokens, t -> { + state.x[t].addInPlace(state.xb2[t]); + }); + + // ffn rmsnorm + FloatTensor.parallelFor(0, nTokens, t -> { + rmsnorm(state.xb[t], state.x[t], weights.rms_ffn_weight[curLayer], dim, config.rmsNormEps); + }); + + // Now for FFN in PyTorch we have: self.w2(F.silu(self.w1(x)) * self.w3(x)) + weights.w1[l].matmul(nTokens, state.xb, state.hb, config.hiddenDim, dim); + weights.w3[l].matmul(nTokens, state.xb, state.hb2, config.hiddenDim, dim); + + // SwiGLU non-linearity + FloatTensor.parallelFor(0, nTokens, t -> { + state.hb[t].mapInPlace(value -> value / (float) (1.0 + Math.exp(-value))); + }); + + // elementwise multiply with w3(x) + FloatTensor.parallelFor(0, nTokens, t -> { + state.hb[t].multiplyInPlace(state.hb2[t]); + }); + + // final matmul to get the output of the ffn + weights.w2[l].matmul(nTokens, state.hb, state.xb, dim, config.hiddenDim); + + // residual connection + FloatTensor.parallelFor(0, nTokens, t -> { + state.x[t].addInPlace(state.xb[t]); + }); + } + + // final rmsnorm + FloatTensor.parallelFor(0, nTokens, t -> { + rmsnorm(state.x[t], state.x[t], weights.rms_final_weight, dim, config.rmsNormEps); + }); + + // classifier into logits + weights.wcls.matmul(state.x[nTokens - 1], state.logits, config.vocabularySize, dim); + state.idxPrevBlock = nTokens - 1; + + return state.logits; + } + + /** + * LLM generation entry point, ingest prompt tokens and generates new tokens. + * + * <p> + * All prompt tokens are ingested first, then inference starts, until a stop token is found. + * The returned tokens only include generated/inferred tokens. + * + * @param model model to run inference (including weights, configuration, tokenizer ...) + * @param state state of the model e.g. key/value caches ... this is mutated by this call + * @param startPosition start prompt ingestion + inference at this position in the context e.g. useful if state was kept across calls (chained generation). 0 implies run with no previous context. + * @param promptTokens prompt tokens to ingest, all the prompt tokens will be ingested, given there's enough capacity left in the context + * @param stopTokens set of tokens that abort generation during inference, stop tokens do not affect prompt ingestion + * @param maxTokens maximum number of tokens (can go up to {@link Configuration#contextLength context length} + * if this value is negative or greater than {@link Configuration#contextLength context length} + * @param sampler {@link Sampler strategy} used to select tokens + * @param echo debugging flag, prints ALL, prompt and inferred tokens, to {@link System#err stderr} + * @param onTokenGenerated callback, if non-null, it's called every time a token is inferred e.g. it's not called when ingesting prompt tokens + * @return list of generated/inferred tokens, including the stop token, if any e.g. does not include any token from the prompt + */ + public static List<Integer> generateTokens(Llama model, State state, int startPosition, List<Integer> promptTokens, + Set<Integer> stopTokens, int maxTokens, Sampler sampler, + IntConsumer onTokenGenerated) { + //long startNanos = System.nanoTime(); + //long startGen = 0; + if (maxTokens < 0 || model.configuration().contextLength < maxTokens) { + maxTokens = model.configuration().contextLength; + } + List<Integer> generatedTokens = new ArrayList<>(maxTokens); + int token = state.latestToken; // BOS? + int nextToken; + int promptIndex = 0; + for (int position = startPosition; position < maxTokens; ++position) { + if (promptIndex < promptTokens.size()) { + final int nTokens = Math.min(maxTokens - position, Math.min(promptTokens.size() - promptIndex, state.batchsize)); + final int[] tokens = new int[nTokens]; + for (int i = 0; i < nTokens; i++) { + tokens[i] = promptTokens.get(promptIndex + i); + //System.out.print(Tokenizer.replaceControlCharacters(model.tokenizer().decode(List.of(tokens[i])))); + + } + //System.out.format("position=%d, promptIdx=%d, promptSize=%d, tokens=%s%n", position, promptIndex, promptTokens.size(), Arrays.toString(tokens)); + + boolean computeLogits = promptIndex + nTokens >= promptTokens.size(); + forward(model, state, tokens, position, computeLogits); + position += nTokens - 1; + promptIndex += nTokens; + if (promptIndex < promptTokens.size()) { + continue; + } + //startGen = System.nanoTime(); + } else { + forward(model, state, new int[]{token}, position, true); + } + nextToken = sampler.sampleToken(state.logits); + // System.out.print(Tokenizer.replaceControlCharacters(model.tokenizer().decode(List.of(nextToken)))); + generatedTokens.add(nextToken); + if (onTokenGenerated != null) { + onTokenGenerated.accept(nextToken); + } + if (stopTokens.contains(nextToken)) { + break; + } + state.latestToken = token = nextToken; + } + + //long elapsedNanos = System.nanoTime() - startNanos; + //long promptNanos = startGen - startNanos; + //long genNanos = elapsedNanos - startGen + startNanos; + // System.out.printf(startPosition + promptIndex + generatedTokens.size(), model.configuration().contextLength, promptTokens.size() / (promptNanos / 1_000_000_000.0), promptTokens.size(), generatedTokens.size() / (genNanos / 1_000_000_000.0), generatedTokens.size()); + return generatedTokens; + } +}
\ No newline at end of file diff --git a/source/net/yacy/ai/llama3/Llama3.java b/source/net/yacy/ai/llama3/Llama3.java new file mode 100644 index 000000000..8d1ab166f --- /dev/null +++ b/source/net/yacy/ai/llama3/Llama3.java @@ -0,0 +1,155 @@ +/** + * Llama3.java + + * This file was extracted from the llama3/qwen2 projects + * https://github.com/mukel/llama3.java + * https://github.com/mukel/qwen2.svm.java + * + * License: MIT License + * + * Copyright (c) 2024 Andrej Karpathy (for llama2.c) + * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2) + * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp) + * Copyright (c) 2025 Michael Peter Christen for modifications: + * The code was modified to fit the YaCy AI project: + * - back-port to Java 11 (removal of Vector API operations and record types) + * - removal of interactive mode and system.out printing + * - separation of the classes in the single java and refactoring + * - run-time performance optimizations for dot product computation of quantized values + * - joining of llama3/qwen2 into one code base; multi-arch options + * - alignment with code from https://github.com/ggml-org/llama.cpp/ + */ + +package net.yacy.ai.llama3; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Scanner; +import java.util.Set; +import java.util.function.IntConsumer; + +import net.yacy.ai.llama3.Model.ModelLoader; + +public class Llama3 { + + // Batch-size used in prompt evaluation. + private static final int BATCH_SIZE = Integer.getInteger("llama.BatchSize", 16); + + + static void runInteractive(Llama model, Sampler sampler, Context options) { + Llama.State state = null; + List<Integer> conversationTokens = new ArrayList<>(); + ChatFormat chatFormat = new ChatFormat(model.tokenizer()); + conversationTokens.add(chatFormat.beginOfText); + if (options.systemPrompt != null) { + conversationTokens.addAll(chatFormat.encodeMessage(new ChatFormat.Message(ChatFormat.Role.SYSTEM, options.systemPrompt))); + } + int startPosition = 0; + @SuppressWarnings("resource") + Scanner in = new Scanner(System.in); + while (true) { + System.out.print("> "); + System.out.flush(); + String userText = in.nextLine(); + if (state == null) { + state = model.createNewState(BATCH_SIZE); + } + conversationTokens.addAll(chatFormat.encodeMessage(new ChatFormat.Message(ChatFormat.Role.USER, userText))); + conversationTokens.addAll(chatFormat.encodeHeader(new ChatFormat.Message(ChatFormat.Role.ASSISTANT, ""))); + Set<Integer> stopTokens = chatFormat.getStopTokens(); + List<Integer> responseTokens = Llama.generateTokens(model, state, startPosition, conversationTokens.subList(startPosition, conversationTokens.size()), stopTokens, options.maxTokens, sampler, token -> { + if (!model.tokenizer().isSpecialToken(token)) { + System.out.print(model.tokenizer().decode(List.of(token))); + } + }); + // Include stop token in the prompt history, but not in the response displayed to the user. + conversationTokens.addAll(responseTokens); + startPosition = conversationTokens.size(); + Integer stopToken = null; + if (!responseTokens.isEmpty() && stopTokens.contains(responseTokens.get(responseTokens.size()-1))) { + stopToken = responseTokens.get(responseTokens.size()-1); + responseTokens.remove(responseTokens.size()-1); + } + //System.out.println(model.tokenizer().decode(responseTokens)); + if (stopToken == null) { + System.out.println("Ran out of context length..."); + break; + } + } + } + + public static List<Integer> runInstructOnce(Llama model, Sampler sampler, Context options, IntConsumer onTokenGenerated) { + Llama.State state = model.createNewState(BATCH_SIZE); + ChatFormat chatFormat = new ChatFormat(model.tokenizer()); + + List<Integer> promptTokens = new ArrayList<>(); + promptTokens.add(chatFormat.beginOfText); + if (options.systemPrompt != null) { + promptTokens.addAll(chatFormat.encodeMessage(new ChatFormat.Message(ChatFormat.Role.SYSTEM, options.systemPrompt))); + } + //System.out.println("Context after System Prompt: " + toString(model, promptTokens)); + promptTokens.addAll(chatFormat.encodeMessage(new ChatFormat.Message(ChatFormat.Role.USER, options.prompt))); + //System.out.println("Context after User Prompt: " + toString(model, promptTokens)); + promptTokens.addAll(chatFormat.encodeHeader(new ChatFormat.Message(ChatFormat.Role.ASSISTANT, ""))); + //System.out.println("Context after Assitant Prompt: " + toString(model, promptTokens)); + + Set<Integer> stopTokens = chatFormat.getStopTokens(); + List<Integer> responseTokens = Llama.generateTokens(model, state, 0, promptTokens, stopTokens, options.maxTokens, sampler, onTokenGenerated); + + // remove stop token at the end of the response, if present + if (!responseTokens.isEmpty() && stopTokens.contains(responseTokens.get(responseTokens.size()-1))) { + responseTokens.remove(responseTokens.size()-1); + } + //System.out.println(model.tokenizer().decode(responseTokens)); + return responseTokens; + } + + public static String toString(Llama model, List<Integer> tokens) { + return model.tokenizer().decode(tokens); + } + + public static void main(String[] args) throws IOException { + // model download paths: + // https://huggingface.co/mukel/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf + // https://huggingface.co/mukel/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q8_0.gguf + // https://huggingface.co/mukel/Llama-3.2-3B-Instruct-GGUF/resolve/main/Llama-3.2-3B-Instruct-Q4_0.gguf + + // performance on iMac x86: + // semeru 11 : 1.40 T/s + // semeru 21 : 1.13 T/s + // GraalVM 21: 2.27 T/s + // openjdk 21: 3.02 T/s; 3.2 with VarHandle + + Path modelPath = Path.of("/Users/admin/git/yacy_search_server", "DATA", "LLMS", "Llama-3.2-1B-Instruct-Q4_0.gguf"); // 26.7 T/s/M4 orig jdk 21; 24.9 T/s/M4 Temurin 24; 25.8 T/s/M4 jdk 21; 22.2 T/s/M4 GraalVM21; 9.2 T/s/M4 Semeru 11; 9.2 T/s/M1 Ultra jdk 11 + //Path modelPath = Path.of("/Users/admin/git/yacy_search_server", "DATA", "LLMS", "Llama-3.2-1B-Instruct-Q8_0.gguf"); // 10.7 T/s/M4 orig jdk 21; 21.2 T/s/M4 Temurin 24; 22 T/s/M4 jdk 21; 18 T/s/M4 GraalVM21; 5.8 T/s/M4 Semeru 11; 6.7 T/s/M1 Ultra jdk 11 + //Path modelPath = Path.of("/Users/admin/git/yacy_search_server", "DATA", "LLMS", "Llama-3.2-3B-Instruct-Q4_0.gguf"); // 9.8 T/s/M4 orig jdk 21; 9.6 T/s/M4 Temurin 24; 9.3 T/s/M4 jdk 21; 8.0 T/s/M4 GraalVM21; 3.2 T/s/M4 Semeru 11; 3.8 T/s/M1 Ultra jdk 11 + //Path modelPath = Path.of("/Users/admin/git/yacy_search_server", "DATA", "LLMS", "Llama-3.2-3B-Instruct-Q8_0.gguf"); // 7.2 T/s/M4 jdk 24 + //Path modelPath = Path.of("/Users/admin/git/yacy_search_server", "DATA", "LLMS", "Meta-Llama-3-8B-Instruct-Q4_0.gguf"); // 3.6 T/s/M4 jdk 24; + //Path modelPath = Path.of("/Users/admin/git/yacy_search_server", "DATA", "LLMS", "OLMo-2-0425-1B-Instruct-Q4_0.gguf"); + Context options = new Context("Write a Java program which computes the first 42 prime numbers.", "Be a very good programmer.", 0.0f, 0.95f, 0, 1024); + Llama model = ModelLoader.loadModel(modelPath, 1024, true); + // get time + long startTime = System.currentTimeMillis(); + Sampler sampler = Sampler.selectSampler(model.configuration().vocabularySize, options.temp, options.topp, options.seed); + List<Integer> resultToken = runInstructOnce(model, sampler, options, token -> { + if (!model.tokenizer().isSpecialToken(token)) { + System.out.print(model.tokenizer().decode(List.of(token))); + } + }); + long endTime = System.currentTimeMillis(); + System.out.println("\nToken: " + resultToken.size() + ", " + ((double) resultToken.size()) * 1000.0d / ((double) (endTime - startTime)) + " Tokens per second"); + } +} + + + + + + + + + + + diff --git a/source/net/yacy/ai/llama3/Model/Arch.java b/source/net/yacy/ai/llama3/Model/Arch.java new file mode 100644 index 000000000..816980c63 --- /dev/null +++ b/source/net/yacy/ai/llama3/Model/Arch.java @@ -0,0 +1,31 @@ +/** + * Arch + * Copyright 2025 by Michael Peter Christen + * First released 25.05.2025 at https://yacy.net + * + ** This class was not part of the original llama3 implementation, + ** but added later by the author to support different architectures. + ** It therefore does not inherit the llama3 copyright. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.ai.llama3.Model; + + +// architecture of the model, see https://github.com/ggml-org/llama.cpp/blob/master/src/llama-arch.cpp +public enum Arch { + LLM_ARCH_LLAMA, LLM_ARCH_QWEN2, LLM_ARCH_QWEN3, LLM_ARCH_OLMO2 +} diff --git a/source/net/yacy/ai/llama3/Model/GGMLTensorEntry.java b/source/net/yacy/ai/llama3/Model/GGMLTensorEntry.java new file mode 100644 index 000000000..3b2f19e7b --- /dev/null +++ b/source/net/yacy/ai/llama3/Model/GGMLTensorEntry.java @@ -0,0 +1,97 @@ +/** + * GGMLTensorEntry.java + + * This file was extracted from the llama3/qwen2 projects + * https://github.com/mukel/llama3.java + * https://github.com/mukel/qwen2.svm.java + * + * License: MIT License + * + * Copyright (c) 2024 Andrej Karpathy (for llama2.c) + * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2) + * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp) + * Copyright (c) 2025 Michael Peter Christen for modifications: + * The code was modified to fit the YaCy AI project: + * - back-port to Java 11 (removal of Vector API operations and record types) + * - removal of interactive mode and system.out printing + * - separation of the classes in the single java and refactoring + * - run-time performance optimizations for dot product computation of quantized values + * - joining of llama3/qwen2 into one code base; multi-arch options + * - alignment with code from https://github.com/ggml-org/llama.cpp/ + */ + +package net.yacy.ai.llama3.Model; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.FloatBuffer; +import java.util.Arrays; +import java.util.Objects; + +import net.yacy.ai.llama3.Tensor.FloatTensor; +import net.yacy.ai.llama3.Tensor.Q4_0FloatTensor; +import net.yacy.ai.llama3.Tensor.Q8_0FloatTensor; + +public final class GGMLTensorEntry { + + private final ByteBuffer buffer; + private final String name; + private final GGMLType ggmlType; + private final int[] shape; + + + public GGMLTensorEntry(String name, GGMLType ggmlType, int[] shape, ByteBuffer buffer) throws IOException { + assert buffer.isDirect() : "Buffer must be a direct ByteBuffer"; + this.buffer = buffer; + buffer.order(ByteOrder.nativeOrder()); // Set correct byte order + this.name = Objects.requireNonNull(name); + this.ggmlType = Objects.requireNonNull(ggmlType); + this.shape = Arrays.copyOf(Objects.requireNonNull(shape), shape.length); + } + + public float getFloat(int index) { + return buffer.getFloat(index * Float.BYTES); + } + + public String name() { + return name; + } + + public GGMLType ggmlType() { + return ggmlType; + } + + public int[] shape() { + return Arrays.copyOf(shape, shape.length); + } + + public FloatBuffer toFloatBuffer() { + switch (ggmlType) { + case F32: return buffer.order(ByteOrder.LITTLE_ENDIAN).asFloatBuffer(); + default: throw new UnsupportedOperationException("Conversion to " + ggmlType); + } + } + + public FloatTensor loadQuantized() { + FloatTensor tensor = null; + switch (ggmlType) { + //case F32: return new F32FloatTensor(FloatTensor.numberOfElements(entry.shape()), entry.memorySegment()); + case Q8_0: tensor = new Q8_0FloatTensor(FloatTensor.numberOfElements(this.shape()), this.buffer); break; + case Q4_0: tensor = new Q4_0FloatTensor(FloatTensor.numberOfElements(this.shape()), this.buffer); break; + default: throw new UnsupportedOperationException("Quantization format " + ggmlType); + } + return tensor; + + // create a new ArrayFloatTensor(final float[] values) + //float[] values = new float[tensor.size()]; + // copy the values from the tensor to the float array + //for (int i = 0; i < values.length; i++) { + /// values[i] = tensor.getFloat(i); + //} + //return new DirectBufferFloatTensor(values); + //return new F16FloatTensor(values); + //return new BF16FloatTensor(values); + //return new ArrayFloatTensor(values); + } +}
\ No newline at end of file diff --git a/source/net/yacy/ai/llama3/Model/GGMLType.java b/source/net/yacy/ai/llama3/Model/GGMLType.java new file mode 100644 index 000000000..d0fe6c427 --- /dev/null +++ b/source/net/yacy/ai/llama3/Model/GGMLType.java @@ -0,0 +1,99 @@ +/** + * GGMLType.java + + * This file was extracted from the llama3/qwen2 projects + * https://github.com/mukel/llama3.java + * https://github.com/mukel/qwen2.svm.java + * + * License: MIT License + * + * Copyright (c) 2024 Andrej Karpathy (for llama2.c) + * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2) + * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp) + * Copyright (c) 2025 Michael Peter Christen for modifications: + * The code was modified to fit the YaCy AI project: + * - back-port to Java 11 (removal of Vector API operations and record types) + * - removal of interactive mode and system.out printing + * - separation of the classes in the single java and refactoring + * - run-time performance optimizations for dot product computation of quantized values + * - joining of llama3/qwen2 into one code base; multi-arch options + * - alignment with code from https://github.com/ggml-org/llama.cpp/ + */ + +package net.yacy.ai.llama3.Model; + +public enum GGMLType { + F32(Float.BYTES), + F16(GGMLType.FLOAT16_BYTES), + Q4_0(GGMLType.FLOAT16_BYTES + 16 * Byte.BYTES, 32), + Q4_1(2 * GGMLType.FLOAT16_BYTES + 16 * Byte.BYTES, 32), + UNSUPPORTED_Q4_2(Integer.MAX_VALUE), // support has been removed + UNSUPPORTED_Q4_3(Integer.MAX_VALUE), // support has been removed + Q5_0(Integer.MAX_VALUE), + Q5_1(Integer.MAX_VALUE), + Q8_0(GGMLType.FLOAT16_BYTES + 32 * Byte.BYTES, 32), + Q8_1(32 * Byte.BYTES + 2 * Float.BYTES, 32), + // k-quantizations + Q2_K(Integer.MAX_VALUE), + Q3_K(Integer.MAX_VALUE), + Q4_K(2 * GGMLType.FLOAT16_BYTES + ((GGMLType.QK_K / 16) / 8 * 6) + GGMLType.QK_K / 2, GGMLType.QK_K), + Q5_K(2 * GGMLType.FLOAT16_BYTES + ((GGMLType.QK_K / 16) / 8 * 6) + GGMLType.QK_K / 8 + GGMLType.QK_K / 2, GGMLType.QK_K), + Q6_K(GGMLType.QK_K / 2 + GGMLType.QK_K / 4 + GGMLType.QK_K / 16 + GGMLType.FLOAT16_BYTES, GGMLType.QK_K), + Q8_K(Integer.MAX_VALUE), + + IQ2_XXS(Integer.MAX_VALUE), + IQ2_XS(Integer.MAX_VALUE), + IQ3_XXS(Integer.MAX_VALUE), + IQ1_S(Integer.MAX_VALUE), + IQ4_NL(Integer.MAX_VALUE), + IQ3_S(Integer.MAX_VALUE), + IQ2_S(Integer.MAX_VALUE), + IQ4_XS(Integer.MAX_VALUE), + + I8(Byte.BYTES), + I16(Short.BYTES), + I32(Integer.BYTES), + I64(Long.BYTES), + F64(Double.BYTES), + IQ1_M(Integer.MAX_VALUE), + BF16(GGMLType.BFLOAT16_BYTES), + Q4_0_4_4(GGMLType.FLOAT16_BYTES + 16 * Byte.BYTES, 32), + Q4_0_4_8(GGMLType.FLOAT16_BYTES + 16 * Byte.BYTES, 32), + Q4_0_8_8(GGMLType.FLOAT16_BYTES + 16 * Byte.BYTES, 32), + TQ1_0(Integer.MAX_VALUE), + TQ2_0(Integer.MAX_VALUE); + + public static final int BFLOAT16_BYTES = 2; + public static final int FLOAT16_BYTES = 2; + private static final GGMLType[] VALUES = values(); + public final int typeSize; + public final int blockSize; + + public final static GGMLType fromId(int id) { + return VALUES[id]; + } + + GGMLType(int typeSize) { + this(typeSize, 1); + } + + public long byteSizeFor(int numberOfElements) { + long t = numberOfElements * (long) this.typeSize; + assert t % this.blockSize == 0; + return Math.toIntExact(t / this.blockSize); + } + + public static final int QK_K = 256; // or 64? + + GGMLType(int typeSize, int blockSize) { + assert blockSize > 0; + assert typeSize > 0; + assert isPowerOf2(blockSize); + this.typeSize = typeSize; + this.blockSize = blockSize; + } + + private final static boolean isPowerOf2(int n) { + return n > 0 && (n & (n - 1)) == 0; + } +} diff --git a/source/net/yacy/ai/llama3/Model/GGUF.java b/source/net/yacy/ai/llama3/Model/GGUF.java new file mode 100644 index 000000000..6a4fd7209 --- /dev/null +++ b/source/net/yacy/ai/llama3/Model/GGUF.java @@ -0,0 +1,645 @@ +/** + * GGUF.java + + * This file was extracted from the llama3/qwen2 projects + * https://github.com/mukel/llama3.java + * https://github.com/mukel/qwen2.svm.java + * + * License: MIT License + * + * Copyright (c) 2024 Andrej Karpathy (for llama2.c) + * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2) + * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp) + * Copyright (c) 2025 Michael Peter Christen for modifications: + * The code was modified to fit the YaCy AI project: + * - back-port to Java 11 (removal of Vector API operations and record types) + * - removal of interactive mode and system.out printing + * - separation of the classes in the single java and refactoring + * - run-time performance optimizations for dot product computation of quantized values + * - joining of llama3/qwen2 into one code base; multi-arch options + * - alignment with code from https://github.com/ggml-org/llama.cpp/ + */ + +package net.yacy.ai.llama3.Model; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.MappedByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Objects; +import java.util.stream.Collectors; + +import net.yacy.ai.llama3.Tensor.FloatTensor; + +/* + * GGUF File Reader. For specification see https://github.com/ggml-org/ggml/blob/master/docs/gguf.md + */ +public final class GGUF { + private static final int GGUF_MAGIC = 0x46554747; + private static final int DEFAULT_ALIGNMENT = 32; // must be a power of 2 + private static final List<Integer> SUPPORTED_GGUF_VERSIONS = List.of(2, 3); + private int magic; + private int version; + private int tensorCount; // uint64_t + private int alignment; + private int metadata_kv_count; // uint64_t + private Map<String, Object> metadata; + private Map<String, GGUFTensorInfo> tensorInfos; + private long tensorDataOffset; + private Map<String, GGMLTensorEntry> tensorEntries; + + public Map<String, GGUFTensorInfo> getTensorInfos() { + return tensorInfos; + } + + public long getTensorDataOffset() { + return tensorDataOffset; + } + + public Map<String, Object> getMetadata() { + return metadata; + } + + private final ByteBuffer BB_1 = ByteBuffer.allocate(Byte.BYTES).order(ByteOrder.LITTLE_ENDIAN); + private final ByteBuffer BB_2 = ByteBuffer.allocate(Short.BYTES).order(ByteOrder.LITTLE_ENDIAN); + private final ByteBuffer BB_4 = ByteBuffer.allocate(Integer.BYTES).order(ByteOrder.LITTLE_ENDIAN); + private final ByteBuffer BB_8 = ByteBuffer.allocate(Long.BYTES).order(ByteOrder.LITTLE_ENDIAN); + + public Map<String, GGMLTensorEntry> getTensorEntries() { + return tensorEntries; + } + + public static GGUF loadModel(Path modelPath) throws IOException { + try (FileChannel fileChannel = FileChannel.open(modelPath)) { + GGUF gguf = new GGUF(); + gguf.loadModelImpl(fileChannel); + return gguf; + } + } + + enum MetadataValueType { + // The value is a 8-bit unsigned integer. + UINT8(1), + // The value is a 8-bit signed integer. + INT8(1), + // The value is a 16-bit unsigned little-endian integer. + UINT16(2), + // The value is a 16-bit signed little-endian integer. + INT16(2), + // The value is a 32-bit unsigned little-endian integer. + UINT32(4), + // The value is a 32-bit signed little-endian integer. + INT32(4), + // The value is a 32-bit IEEE754 floating point number. + FLOAT32(4), + // The value is a boolean. + // 1-byte value where 0 is false and 1 is true. + // Anything else is invalid, and should be treated as either the model being invalid or the reader being buggy. + BOOL(1), + // The value is a UTF-8 non-null-terminated string, with length prepended. + STRING(-8), + // The value is an array of other values, with the length and type prepended. + // Arrays can be nested, and the length of the array is the number of elements in the array, not the number of bytes. + ARRAY(-8), + // The value is a 64-bit unsigned little-endian integer. + UINT64(8), + // The value is a 64-bit signed little-endian integer. + INT64(8), + // The value is a 64-bit IEEE754 floating point number. + FLOAT64(8); + private final int byteSize; + + MetadataValueType(int byteSize) { + this.byteSize = byteSize; + } + + private static final MetadataValueType[] VALUES = values(); + + public static MetadataValueType fromIndex(int index) { + return VALUES[index]; + } + + public int byteSize() { + return byteSize; + } + } + + private void loadModelImpl(FileChannel fileChannel) throws IOException { + // The header of the file. + readHeader(fileChannel); // gguf_header_t header; + // Tensor infos, which can be used to locate the tensor data. + // gguf_tensor_info_t tensor_infos[header.tensor_count]; + this.tensorInfos = new HashMap<>(tensorCount); + for (int i = 0; i < tensorCount; ++i) { + GGUF.GGUFTensorInfo ti = readTensorInfo(fileChannel); + assert !tensorInfos.containsKey(ti.name); + tensorInfos.put(ti.name, ti); + } + // Padding to the nearest multiple of `ALIGNMENT`. + // uint8_t _padding[ALIGNMENT - (sizeof(header + tensor_infos) % ALIGNMENT)]; + //long _padding = -fileChannel.position() & (ALIGNMENT - 1); + long _padding = getAlignment() - (fileChannel.position() % getAlignment()); + fileChannel.position(fileChannel.position() + _padding); + this.tensorDataOffset = fileChannel.position(); + } + /* + public static Map<String, GGMLTensorEntry> loadTensors(FileChannel fileChannel, long tensorDataOffset, Map<String, GGUFTensorInfo> tensorInfos) throws IOException { + // Map the whole remaining file region into memory + MappedByteBuffer mappedBuffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, tensorDataOffset, fileChannel.size() - tensorDataOffset); + mappedBuffer.order(ByteOrder.nativeOrder()); + + Map<String, GGMLTensorEntry> tensorEntries = new HashMap<>(tensorInfos.size()); + + for (Map.Entry<String, GGUFTensorInfo> entry : tensorInfos.entrySet()) { + GGUFTensorInfo ti = entry.getValue(); + int numberOfElements = FloatTensor.numberOfElements(ti.dimensions()); + int sizeInBytes = Math.toIntExact(ti.ggmlType().byteSizeFor(numberOfElements)); + int offset = Math.toIntExact(ti.offset()); // assumes offset is within Integer range + + // Create a slice of the mapped buffer for this tensor + MappedByteBuffer buffer = (MappedByteBuffer) mappedBuffer.duplicate(); + buffer.position(offset); + buffer.limit(offset + sizeInBytes); + + // copy this into a DirectByteBuffer (tests show that this is not faster than using the MappedByteBuffer directly) + //ByteBuffer directBuffer = ByteBuffer.allocateDirect(sizeInBytes).order(ByteOrder.nativeOrder()); + //directBuffer.put(buffer); + //directBuffer.flip(); + //tensorEntries.put(ti.name(), new GGMLTensorEntry(ti.name(), ti.ggmlType(), ti.dimensions(), directBuffer)); + + // Create a slice of the mapped buffer for this tensor + MappedByteBuffer tensorBuffer = (MappedByteBuffer) buffer.slice().order(ByteOrder.nativeOrder()); + tensorEntries.put(ti.name(), new GGMLTensorEntry(ti.name(), ti.ggmlType(), ti.dimensions(), tensorBuffer)); + } + + return tensorEntries; + } +*/ + + static Map<String, GGMLTensorEntry> loadTensors(FileChannel fileChannel, long tensorDataOffset, + Map<String, GGUFTensorInfo> tensorInfos) throws IOException { + + final long totalDataSize = fileChannel.size() - tensorDataOffset; + Map<String, GGMLTensorEntry> tensorEntries = new HashMap<>(tensorInfos.size()); + + // Fast path: if the entire tensor data fits in one mapped buffer + if (totalDataSize <= Integer.MAX_VALUE) { + MappedByteBuffer fullBuffer = (MappedByteBuffer) fileChannel.map( + FileChannel.MapMode.READ_ONLY, + tensorDataOffset, + totalDataSize + ).order(ByteOrder.nativeOrder()); + + for (Map.Entry<String, GGUFTensorInfo> entry : tensorInfos.entrySet()) { + GGUFTensorInfo ti = entry.getValue(); + long offset = ti.offset(); + int sizeInBytes = Math.toIntExact(ti.ggmlType().byteSizeFor( + FloatTensor.numberOfElements(ti.dimensions()))); + + MappedByteBuffer tensorBuffer = (MappedByteBuffer) fullBuffer.duplicate(); + tensorBuffer.position((int)offset); + tensorBuffer.limit((int)offset + sizeInBytes); + tensorBuffer = (MappedByteBuffer) tensorBuffer.slice().order(ByteOrder.nativeOrder()); + + tensorEntries.put(ti.name(), new GGMLTensorEntry( + ti.name(), ti.ggmlType(), ti.dimensions(), tensorBuffer)); + } + return tensorEntries; + } + + // Slow path: only for large files > 2GB: + // Models can be very large, so we cannot map the entire tensor data at once. + // Instead, we will map segments of the tensor data as needed. + + // First pass: collect all tensor boundaries and sort them + List<Long> boundaries = new ArrayList<>(); + for (GGUFTensorInfo ti : tensorInfos.values()) { + long start = ti.offset(); + long end = start + ti.ggmlType().byteSizeFor(FloatTensor.numberOfElements(ti.dimensions())); + boundaries.add(start); + boundaries.add(end); + } + boundaries = boundaries.stream() + .distinct() + .sorted() + .collect(Collectors.toList()); + + // Second pass: create memory mappings for each segment between boundaries + List<MappedSegment> mappings = new ArrayList<>(); + long currentPos = tensorDataOffset; + + for (long boundary : boundaries) { + if (boundary <= currentPos) continue; + + long mappingSize = boundary - currentPos; + // Split into chunks no larger than Integer.MAX_VALUE + while (mappingSize > 0) { + long chunkSize = Math.min(mappingSize, Integer.MAX_VALUE); + MappedByteBuffer buffer = (MappedByteBuffer) fileChannel.map( + FileChannel.MapMode.READ_ONLY, + currentPos, + chunkSize + ).order(ByteOrder.nativeOrder()); + mappings.add(new MappedSegment(currentPos, buffer)); + currentPos += chunkSize; + mappingSize -= chunkSize; + } + } + + // Handle any remaining data after last boundary + if (currentPos < fileChannel.size()) { + long remaining = fileChannel.size() - currentPos; + while (remaining > 0) { + long chunkSize = Math.min(remaining, Integer.MAX_VALUE); + MappedByteBuffer buffer = (MappedByteBuffer) fileChannel.map( + FileChannel.MapMode.READ_ONLY, + currentPos, + chunkSize + ).order(ByteOrder.nativeOrder()); + mappings.add(new MappedSegment(currentPos, buffer)); + currentPos += chunkSize; + remaining -= chunkSize; + } + } + + // Third pass: create tensor views + for (Map.Entry<String, GGUFTensorInfo> entry : tensorInfos.entrySet()) { + GGUFTensorInfo ti = entry.getValue(); + String name = ti.name(); + long tensorOffset = ti.offset() + tensorDataOffset; + long tensorSize = ti.ggmlType().byteSizeFor(FloatTensor.numberOfElements(ti.dimensions())); + long tensorEnd = tensorOffset + tensorSize; + + // Find all segments that overlap with this tensor + List<MappedSegment> overlappingSegments = mappings.stream() + .filter(seg -> seg.startOffset <= tensorEnd && + seg.startOffset + seg.buffer.capacity() > tensorOffset) + .sorted(Comparator.comparingLong(seg -> seg.startOffset)) + .collect(Collectors.toList()); + + if (overlappingSegments.isEmpty()) { + throw new IOException("Tensor " + name + " not found in any mapped segment"); + } + + if (overlappingSegments.size() == 1) { + // Simple case - tensor fits entirely in one segment + MappedSegment segment = overlappingSegments.get(0); + int bufferOffset = (int)(tensorOffset - segment.startOffset); + int sizeInBytes = (int)tensorSize; + + MappedByteBuffer tensorBuffer = (MappedByteBuffer) segment.buffer.duplicate(); + tensorBuffer.position(bufferOffset); + tensorBuffer.limit(bufferOffset + sizeInBytes); + tensorBuffer = (MappedByteBuffer) tensorBuffer.slice().order(ByteOrder.nativeOrder()); + + tensorEntries.put(name, new GGMLTensorEntry( + name, ti.ggmlType(), ti.dimensions(), tensorBuffer)); + } else { + // Complex case - tensor spans multiple segments + ByteBuffer combinedBuffer = ByteBuffer.allocateDirect((int)tensorSize) + .order(ByteOrder.nativeOrder()); + + long remainingBytes = tensorSize; + long currentTensorPos = tensorOffset; + + for (MappedSegment segment : overlappingSegments) { + long segmentEnd = segment.startOffset + segment.buffer.capacity(); + long copyStart = Math.max(currentTensorPos, segment.startOffset); + long copyEnd = Math.min(tensorEnd, segmentEnd); + int bytesToCopy = (int)(copyEnd - copyStart); + + int srcPos = (int)(copyStart - segment.startOffset); + segment.buffer.position(srcPos); + + byte[] temp = new byte[bytesToCopy]; + segment.buffer.get(temp); + combinedBuffer.put(temp); + + remainingBytes -= bytesToCopy; + currentTensorPos += bytesToCopy; + + if (remainingBytes <= 0) break; + } + + combinedBuffer.flip(); + tensorEntries.put(name, new GGMLTensorEntry( + name, ti.ggmlType(), ti.dimensions(), combinedBuffer)); + } + } + + return tensorEntries; + } + + // Helper class to track mapped segments + private static class MappedSegment { + final long startOffset; + final MappedByteBuffer buffer; + + MappedSegment(long startOffset, MappedByteBuffer buffer) { + this.startOffset = startOffset; + this.buffer = buffer; + } + } + + public static final class GGUFTensorInfo { + private final String name; + private final int[] dimensions; + private final GGMLType ggmlType; + private final long offset; + + public GGUFTensorInfo(String name, int[] dimensions, GGMLType ggmlType, long offset) { + this.name = name; + this.dimensions = dimensions != null ? dimensions.clone() : null; + this.ggmlType = ggmlType; + this.offset = offset; + } + + public String name() { + return name; + } + + public int[] dimensions() { + return dimensions != null ? dimensions.clone() : null; + } + + public GGMLType ggmlType() { + return ggmlType; + } + + public long offset() { + return offset; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + GGUFTensorInfo that = (GGUFTensorInfo) o; + return offset == that.offset && + Objects.equals(name, that.name) && + Arrays.equals(dimensions, that.dimensions) && + Objects.equals(ggmlType, that.ggmlType); + } + + @Override + public int hashCode() { + int result = Objects.hash(name, ggmlType, offset); + result = 31 * result + Arrays.hashCode(dimensions); + return result; + } + + @Override + public String toString() { + return "GGUFTensorInfo[" + + "name=" + name + + ", dimensions=" + Arrays.toString(dimensions) + + ", ggmlType=" + ggmlType + + ", offset=" + offset + + ']'; + } + } + + private GGMLType readGGMLType(FileChannel fileChannel) throws IOException { + int ggmlTypeId = readInt(fileChannel); // ggml_type type; + return GGMLType.fromId(ggmlTypeId); + } + + private GGUF.GGUFTensorInfo readTensorInfo(FileChannel fileChannel) throws IOException { + // The name of the tensor. It is a standard GGUF string, with the caveat that + // it must be at most 64 bytes long. + String name = readString(fileChannel); // gguf_string_t name; + assert name.length() <= 64; + // The number of dimensions in the tensor. + // Currently at most 4, but this may change in the future. + int n_dimensions = readInt(fileChannel); // uint32_t n_dimensions; + assert n_dimensions <= 4; + // The dimensions of the tensor. + int[] dimensions = new int[n_dimensions]; // uint64_t dimensions[n_dimensions]; + for (int i = 0; i < n_dimensions; ++i) { + dimensions[i] = Math.toIntExact(readLong(fileChannel)); + } + // The type of the tensor. + GGMLType ggmlType = readGGMLType(fileChannel); // ggml_type type; + // The offset of the tensor's data in this file in bytes. + // This offset is relative to `tensor_data`, not to the start + // of the file, to make it easier for writers to write the file. + // Readers should consider exposing this offset relative to the + // file to make it easier to read the data. + // Must be a multiple of `ALIGNMENT`. + long offset = readLong(fileChannel); // uint64_t offset; + assert offset % getAlignment() == 0; + return new GGUF.GGUFTensorInfo(name, dimensions, ggmlType, offset); + } + + private String readString(FileChannel fileChannel) throws IOException { + // A string in GGUF. + // The length of the string, in bytes. + int len = Math.toIntExact(readLong(fileChannel)); // uint64_t len; + // The string as a UTF-8 non-null-terminated string. + byte[] bytes = new byte[len]; // char string[len]; + int bytesRead = fileChannel.read(ByteBuffer.wrap(bytes)); + assert len == bytesRead; + return new String(bytes, StandardCharsets.UTF_8); + } + + private Pair<String, Object> readKeyValuePair(FileChannel fileChannel) throws IOException { + // The key of the metadata. It is a standard GGUF string, with the following caveats: + // - It must be a valid ASCII string. + // - It must be a hierarchical key, where each segment is `lower_snake_case` and separated by a `.`. + // - It must be at most 2^16-1/65535 bytes long. + // Any keys that do not follow these rules are invalid. + String key = readString(fileChannel); // gguf_string_t key; + assert key.length() < (1 << 16); + assert key.codePoints().allMatch(cp -> ('a' <= cp && cp <= 'z') || ('0' <= cp && cp <= '9') || cp == '_' || cp == '.'); + Object value = readMetadataValue(fileChannel); + return new Pair<>(key, value); + } + + private Object readMetadataValue(FileChannel fileChannel) throws IOException { + // The type of the value. + // Must be one of the `gguf_metadata_value_type` values. + MetadataValueType value_type = readMetadataValueType(fileChannel); // gguf_metadata_value_type value_type; + // The value. + return readMetadataValueOfType(value_type, fileChannel); // gguf_metadata_value_t value; + } + + // compare this with https://github.com/ggml-org/llama.cpp/blob/master/gguf-py/gguf/gguf_reader.py#L132 + void readHeader(FileChannel fileChannel) throws IOException { + // Magic number to announce that this is a GGUF file. + // Must be `GGUF` at the byte level: `0x47` `0x47` `0x55` `0x46`. + // Your executor might do little-endian byte order, so it might be + // check for 0x46554747 and letting the endianness cancel out. + // Consider being *very* explicit about the byte order here. + this.magic = readInt(fileChannel); // uint32_t magic; + if (magic != GGUF_MAGIC) { + throw new IllegalArgumentException("unsupported header.magic " + magic); + } + // The version of the format implemented. + // Must be `3` for version described in this spec. + // + // This version should only be increased for structural changes to the format. + // Changes that do not affect the structure of the file should instead update the metadata + // to signify the change. + this.version = readInt(fileChannel); // uint32_t version; + if (!SUPPORTED_GGUF_VERSIONS.contains(version)) { + throw new IllegalArgumentException("unsupported header.version " + version); + } + // The number of tensors in the file. + // This is explicit, instead of being included in the metadata, to ensure it is always present + // for loading the tensors. + // https://github.com/ggml-org/llama.cpp/blob/master/gguf-py/gguf/gguf_reader.py#L165 + this.tensorCount = Math.toIntExact(readLong(fileChannel)); // uint64_t tensor_count; + // The number of metadata key-value pairs. + this.metadata_kv_count = Math.toIntExact(readLong(fileChannel)); // uint64_t metadata_kv_count; + // The metadata key-value pairs. + // gguf_metadata_kv_t metadata_kv[metadata_kv_count]; + this.metadata = new HashMap<>(metadata_kv_count); + for (int i = 0; i < metadata_kv_count; ++i) { + Pair<String, Object> keyValue = readKeyValuePair(fileChannel); + assert !metadata.containsKey(keyValue.first()); + metadata.put(keyValue.first(), keyValue.second()); + } + } + + private Object readArray(FileChannel fileChannel) throws IOException { + // Any value type is valid, including arrays. + MetadataValueType value_type = readMetadataValueType(fileChannel); // gguf_metadata_value_type type; + // Number of elements, not bytes + int len = Math.toIntExact(readLong(fileChannel)); // uint64_t len; + // The array of values. + // gguf_metadata_value_t array[len]; + switch (value_type) { + case UINT8: + case INT8: { + byte[] bytes = new byte[len]; + for (int i = 0; i < len; ++i) { + bytes[i] = readByte(fileChannel); + } + return bytes; + } + case UINT16: + case INT16: { + short[] shorts = new short[len]; + for (int i = 0; i < len; ++i) { + shorts[i] = readShort(fileChannel); + } + return shorts; + } + case UINT32: + case INT32: { + int[] ints = new int[len]; + for (int i = 0; i < len; ++i) { + ints[i] = readInt(fileChannel); + } + return ints; + } + case FLOAT32: { + float[] floats = new float[len]; + for (int i = 0; i < len; ++i) { + floats[i] = readFloat(fileChannel); + } + return floats; + } + case BOOL: { + boolean[] booleans = new boolean[len]; + for (int i = 0; i < len; ++i) { + booleans[i] = readBoolean(fileChannel); + } + return booleans; + } + case STRING: { + String[] strings = new String[len]; + for (int i = 0; i < len; ++i) { + strings[i] = readString(fileChannel); + } + return strings; + } + case ARRAY: { + Object[] arrays = new Object[len]; + for (int i = 0; i < len; ++i) { + arrays[i] = readArray(fileChannel); + } + return arrays; + } + default: throw new UnsupportedOperationException("read array of " + value_type); + } + } + + private Object readMetadataValueOfType(MetadataValueType valueType, FileChannel fileChannel) throws IOException { + switch (valueType) { + case UINT8: + case INT8: return readByte(fileChannel); + case UINT16: + case INT16: return readShort(fileChannel); + case UINT32: + case INT32: return readInt(fileChannel); + case FLOAT32: return readFloat(fileChannel); + case UINT64: + case INT64: return readLong(fileChannel); + case FLOAT64: return readDouble(fileChannel); + case BOOL: return readBoolean(fileChannel); + case STRING: return readString(fileChannel); + case ARRAY: return readArray(fileChannel); + default: throw new AssertionError(); + } + } + + private byte readByte(FileChannel fileChannel) throws IOException { + int bytesRead = fileChannel.read(BB_1); + assert bytesRead == 1; + return BB_1.clear().get(0); + } + + private boolean readBoolean(FileChannel fileChannel) throws IOException { + return readByte(fileChannel) != 0; + } + + private short readShort(FileChannel fileChannel) throws IOException { + int bytesRead = fileChannel.read(BB_2); + assert bytesRead == 2; + return BB_2.clear().getShort(0); + } + + private int readInt(FileChannel fileChannel) throws IOException { + int bytesRead = fileChannel.read(BB_4); + assert bytesRead == 4; + return BB_4.clear().getInt(0); + } + + private long readLong(FileChannel fileChannel) throws IOException { + int bytesRead = fileChannel.read(BB_8); + assert bytesRead == 8; + return BB_8.clear().getLong(0); + } + + private float readFloat(FileChannel fileChannel) throws IOException { + return Float.intBitsToFloat(readInt(fileChannel)); + } + + private double readDouble(FileChannel fileChannel) throws IOException { + return Double.longBitsToDouble(readLong(fileChannel)); + } + + private MetadataValueType readMetadataValueType(FileChannel fileChannel) throws IOException { + int index = readInt(fileChannel); + return MetadataValueType.fromIndex(index); + } + + public int getAlignment() { + if (alignment != 0) { + return alignment; + } + alignment = (int) metadata.getOrDefault("general.alignment", DEFAULT_ALIGNMENT); + assert Integer.bitCount(alignment) == 1 : "alignment must be a power of two"; + return alignment; + } +}
\ No newline at end of file diff --git a/source/net/yacy/ai/llama3/Model/ModelLoader.java b/source/net/yacy/ai/llama3/Model/ModelLoader.java new file mode 100644 index 000000000..4cd7ce40b --- /dev/null +++ b/source/net/yacy/ai/llama3/Model/ModelLoader.java @@ -0,0 +1,333 @@ +/** + * ModelLoader.java + + * This file was extracted from the llama3/qwen2 projects + * https://github.com/mukel/llama3.java + * https://github.com/mukel/qwen2.svm.java + * + * License: MIT License + * + * Copyright (c) 2024 Andrej Karpathy (for llama2.c) + * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2) + * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp) + * Copyright (c) 2025 Michael Peter Christen for modifications: + * The code was modified to fit the YaCy AI project: + * - back-port to Java 11 (removal of Vector API operations and record types) + * - removal of interactive mode and system.out printing + * - separation of the classes in the single java and refactoring + * - run-time performance optimizations for dot product computation of quantized values + * - joining of llama3/qwen2 into one code base; multi-arch options + * - alignment with code from https://github.com/ggml-org/llama.cpp/ + */ + +package net.yacy.ai.llama3.Model; + +import java.io.IOException; +import java.nio.FloatBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.function.IntFunction; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import net.yacy.ai.llama3.Llama; +import net.yacy.ai.llama3.Tensor.FloatTensor; + +public final class ModelLoader { + private static final String TOKENIZER_LLAMA_3_MODEL = "gpt2"; + //private static final String TOKENIZER_QWEN2_7B_MODEL = "gpt2"; + + private static final String LLAMA_3_PATTERN = "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"; + private final static String QWEN2_PATTERN = "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"; + + private static Vocabulary loadVocabulary(Map<String, Object> metadata) { + String model = (String) metadata.get("tokenizer.ggml.model"); + if (!TOKENIZER_LLAMA_3_MODEL.equals(model)) { + throw new IllegalArgumentException("expected " + TOKENIZER_LLAMA_3_MODEL + " but found " + model); + } + String[] tokens = (String[]) metadata.get("tokenizer.ggml.tokens"); + float[] scores = (float[]) metadata.get("tokenizer.ggml.scores"); + return new Vocabulary(tokens, scores); + } + + public static Llama loadModel(Path ggufPath, int contextLength, boolean loadWeights) throws IOException { + String name = ggufPath.getFileName().toString(); + if (name.contains("Llama")) return loadModelLlama3(ggufPath, contextLength, loadWeights); + //if (name.contains("OLMo")) return loadModelLlama3(ggufPath, contextLength, loadWeights); + if (name.contains("Qwen3")) return loadModelQwen3(ggufPath, contextLength); + //if (name.contains("Qwen2")) return loadModelQwen2(ggufPath, contextLength); + throw new IOException("model type unknown"); + } + + private static Llama loadModelLlama3(Path ggufPath, int contextLength, boolean loadWeights) throws IOException { + GGUF gguf = GGUF.loadModel(ggufPath); + FileChannel fileChannel = FileChannel.open(ggufPath, StandardOpenOption.READ); + Map<String, Object> metadata = gguf.getMetadata(); + Vocabulary vocabulary = loadVocabulary(metadata); + Tokenizer tokenizer = createLlama3Tokenizer(metadata, vocabulary); + + Llama.Configuration config = new Llama.Configuration( + Arch.LLM_ARCH_LLAMA, + (int) metadata.get("llama.embedding_length"), + (int) metadata.get("llama.feed_forward_length"), + (int) metadata.get("llama.block_count"), + (int) metadata.get("llama.attention.head_count"), + + metadata.containsKey("llama.attention.head_count_kv") + ? (int) metadata.get("llama.attention.head_count_kv") + : (int) metadata.get("llama.attention.head_count"), + + vocabulary.size(), + (int) metadata.get("llama.context_length"), + false, + (float) metadata.getOrDefault("llama.attention.layer_norm_rms_epsilon", 1e-5f), + (float) metadata.getOrDefault("llama.rope.freq_base", 10000f) + ).withContextLength(contextLength); + + Llama.Weights weights = null; + if (loadWeights) { + Map<String, GGMLTensorEntry> tensorEntries = GGUF.loadTensors(fileChannel, gguf.getTensorDataOffset(), gguf.getTensorInfos()); + weights = loadWeightsLlama3(tensorEntries, config); + } + return new Llama(config, tokenizer, weights); + } + + private static Llama.Weights loadWeightsLlama3(Map<String, GGMLTensorEntry> tensorEntries, Llama.Configuration config) { + boolean ropeScaling = tensorEntries.containsKey("rope_freqs"); + float scaleFactor = 8; + float loFreqFactor = 1; + float hiFreqFactor = 3; + int oldContextLength = 8192; + Pair<float[], float[]> ropeFreqs = precomputeFreqsCis4Llama3(config.contextLength, config.headSize, config.ropeTheta, + ropeScaling, scaleFactor, loFreqFactor, hiFreqFactor, oldContextLength); + float[] ropeFreqsReal = ropeFreqs.first(); + float[] ropeFreqsImag = ropeFreqs.second(); + + GGMLTensorEntry tokenEmbeddings = tensorEntries.get("token_embd.weight"); + Llama.Weights qw = new Llama.Weights( + tokenEmbeddings.loadQuantized(), + loadArrayOfFloatBuffer(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_norm.weight")), + loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_q.weight")), + loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_k.weight")), + loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_v.weight")), + null, null, null, + loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_output.weight")), + loadArrayOfFloatBuffer(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".ffn_norm.weight")), + loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".ffn_gate.weight")), // w1 + loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".ffn_down.weight")), // w2 + loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".ffn_up.weight")), // w3 + tensorEntries.get("output_norm.weight").toFloatBuffer(), + FloatBuffer.wrap(ropeFreqsReal), + FloatBuffer.wrap(ropeFreqsImag), + // If "output.weight" is not present then the embedding weights are tied/shared with the decoder. + // This is commonly referred as "tie word embeddings". + tensorEntries.getOrDefault("output.weight", tokenEmbeddings).loadQuantized() + ); + + return qw; + } + + private static Llama loadModelQwen3(Path ggufPath, int contextLength) throws IOException { + // see https://github.com/ggml-org/llama.cpp/pull/12501/files + GGUF gguf = GGUF.loadModel(ggufPath); + Map<String, Object> metadata = gguf.getMetadata(); + + Vocabulary vocabulary = loadVocabulary(metadata); + Tokenizer tokenizer = createQwen2Tokenizer(metadata, vocabulary); + + int modelContextLength = (int) metadata.get("qwen2.context_length"); + if (contextLength < 0 || modelContextLength < contextLength) { + contextLength = modelContextLength; + } + + Llama.Configuration config = new Llama.Configuration( + Arch.LLM_ARCH_QWEN3, + (int) metadata.get("qwen2.embedding_length"), + (int) metadata.get("qwen2.feed_forward_length"), + (int) metadata.get("qwen2.block_count"), + (int) metadata.get("qwen2.attention.head_count"), + + metadata.containsKey("qwen2.attention.head_count_kv") + ? (int) metadata.get("qwen2.attention.head_count_kv") + : (int) metadata.get("qwen2.attention.head_count"), + + vocabulary.size(), + contextLength, + false, + (float) metadata.get("qwen2.attention.layer_norm_rms_epsilon"), + (float) metadata.get("qwen2.rope.freq_base") + ); + + Map<String, GGMLTensorEntry> tensorEntries = gguf.getTensorEntries(); + + Pair<float[], float[]> ropeFreqs = precomputeFreqsCis4Qwen2(config.contextLength, config.headSize, config.ropeTheta); + float[] ropeFreqsReal = ropeFreqs.first(); + float[] ropeFreqsImag = ropeFreqs.second(); + + + FloatTensor tokenEmbeddingTable = tensorEntries.get("token_embd.weight").loadQuantized(); + Llama.Weights qw = new Llama.Weights( + tokenEmbeddingTable, + loadArrayOfFloatBuffer(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_norm.weight")), + loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_q.weight")), + loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_k.weight")), + loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_v.weight")), + + loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_q.bias")), + loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_k.bias")), + loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_v.bias")), + + loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".attn_output.weight")), + loadArrayOfFloatBuffer(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".ffn_norm.weight")), + loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".ffn_gate.weight")), // w1 + loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".ffn_down.weight")), // w2 + loadArrayOfQuantized(config.numberOfLayers, i -> tensorEntries.get("blk." + i + ".ffn_up.weight")), // w3 + tensorEntries.get("output_norm.weight").toFloatBuffer(), + FloatBuffer.wrap(ropeFreqsReal), + FloatBuffer.wrap(ropeFreqsImag), + tensorEntries.containsKey("output.weight") + ? tensorEntries.get("output.weight").loadQuantized() + : tokenEmbeddingTable // weights are shared + ); + + return new Llama(config, tokenizer, qw); + } + + private static Tokenizer createLlama3Tokenizer(Map<String, Object> metadata, Vocabulary vocabulary) { + String[] mergeLines = (String[]) metadata.get("tokenizer.ggml.merges"); + List<Pair<Integer, Integer>> merges = Arrays.stream(mergeLines) + .map(line -> line.split(" ")) + .map(parts -> + new Pair<>( + vocabulary.getIndex(parts[0]).orElseThrow(), + vocabulary.getIndex(parts[1]).orElseThrow()) + ).collect(Collectors.toList()); + + int allTokens = vocabulary.size(); + int baseTokens = 128000; // assume all tokens after the base ones are special. + //int reservedSpecialTokens = allTokens - baseTokens; + List<String> specialTokensList = Arrays.stream(vocabulary.tokens(), baseTokens, allTokens).collect(Collectors.toList()); + + assert specialTokensList.stream().allMatch(token -> vocabulary.getIndex(token).isPresent()); + + Map<String, Integer> specialTokens = + IntStream.range(0, specialTokensList.size()) + .boxed() + .collect(Collectors.toMap( + i -> specialTokensList.get(i), + i -> baseTokens + i) + ); + + return new Tokenizer(vocabulary, merges, LLAMA_3_PATTERN, specialTokens, null); + } + + private static Tokenizer createQwen2Tokenizer(Map<String, Object> metadata, Vocabulary vocabulary) { + int[] tokenTypes = (int[]) metadata.get("tokenizer.ggml.token_type"); + String[] mergeLines = (String[]) metadata.get("tokenizer.ggml.merges"); + List<Pair<Integer, Integer>> merges = Arrays.stream(mergeLines) + .map(line -> line.split(" ")) + .map(parts -> + new Pair<>( + vocabulary.getIndex(parts[0]).orElseThrow(), + vocabulary.getIndex(parts[1]).orElseThrow()) + ) + .collect(Collectors.toList()); + + int allTokens = vocabulary.size(); + int baseTokens = vocabulary.getIndex("<|endoftext|>").orElseThrow(); // assume all tokens after the base ones are special. + //int reservedSpecialTokens = allTokens - baseTokens; + List<String> specialTokensList = Arrays.stream(vocabulary.tokens(), baseTokens, allTokens).collect(Collectors.toList()); + + assert specialTokensList.stream().allMatch(token -> vocabulary.getIndex(token).isPresent()); + + Map<String, Integer> specialTokens = + IntStream.range(0, specialTokensList.size()) + .boxed() + .collect(Collectors.toMap( + i -> specialTokensList.get(i), + i -> baseTokens + i) + ); + + return new Tokenizer(vocabulary, merges, QWEN2_PATTERN, specialTokens, tokenTypes); + } + + + private static FloatTensor[] loadArrayOfQuantized(int size, IntFunction<GGMLTensorEntry> getTensorEntry) { + FloatTensor[] array = new FloatTensor[size]; + for (int i = 0; i < size; i++) { + array[i] = getTensorEntry.apply(i).loadQuantized(); + } + return array; + } + + private static FloatBuffer[] loadArrayOfFloatBuffer(int size, IntFunction<GGMLTensorEntry> getTensorEntry) { + FloatBuffer[] array = new FloatBuffer[size]; + for (int i = 0; i < size; i++) { + array[i] = getTensorEntry.apply(i).toFloatBuffer(); + } + return array; + } + + // RoPE + + // for LLama3 + private static Pair<float[], float[]> precomputeFreqsCis4Llama3( + int contextLength, int headSize, double theta, + boolean ropeScaling, float scaleFactor, + float loFreqFactor, float hiFreqFactor, float oldContextLength) { + assert headSize % 2 == 0; + float[] cr = new float[contextLength * (headSize / 2)]; + float[] ci = new float[contextLength * (headSize / 2)]; + int n = 0; + for (int pos = 0; pos < contextLength; ++pos) { + for (int i = 0; i < headSize; i += 2) { + float freq = (float) (1.0 / Math.pow(theta, i / (double) headSize)); + if (ropeScaling) { + // Llama 3.1 scaling + float loFreqWavelen = oldContextLength / loFreqFactor; + float hiFreqWavelen = oldContextLength / hiFreqFactor; + float wavelen = (float) (2.0 * Math.PI / freq); + if (wavelen < hiFreqWavelen) { + //freq = freq; + } else if (wavelen > loFreqWavelen) { + freq = freq / scaleFactor; + } else { + float smooth = (oldContextLength / wavelen - loFreqFactor) / (hiFreqFactor - loFreqFactor); + freq = (1.0f - smooth) * freq / scaleFactor + smooth * freq; + } + } + float val = pos * freq; + cr[n] = (float) Math.cos(val); + ci[n] = (float) Math.sin(val); + n++; + } + } + assert contextLength * (headSize / 2) == n; + return new Pair<>(cr, ci); + } + + // for Qwen3 + private static Pair<float[], float[]> precomputeFreqsCis4Qwen2( + int contextLength, int headSize, double theta) { + assert headSize % 2 == 0; + float[] cr = new float[contextLength * (headSize / 2)]; + float[] ci = new float[contextLength * (headSize / 2)]; + int n = 0; + for (int pos = 0; pos < contextLength; ++pos) { + for (int i = 0; i < headSize; i += 2) { + float freq = (float) (1.0 / Math.pow(theta, i / (double) headSize)); + float val = pos * freq; + cr[n] = (float) Math.cos(val); + ci[n] = (float) Math.sin(val); + n++; + } + } + assert contextLength * (headSize / 2) == n; + return new Pair<>(cr, ci); + } + +}
\ No newline at end of file diff --git a/source/net/yacy/ai/llama3/Model/Pair.java b/source/net/yacy/ai/llama3/Model/Pair.java new file mode 100644 index 000000000..528994cc8 --- /dev/null +++ b/source/net/yacy/ai/llama3/Model/Pair.java @@ -0,0 +1,59 @@ +/** + * Pair.java + + * This file was extracted from the llama3/qwen2 projects + * https://github.com/mukel/llama3.java + * https://github.com/mukel/qwen2.svm.java + * + * License: MIT License + * + * Copyright (c) 2024 Andrej Karpathy (for llama2.c) + * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2) + * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp) + * Copyright (c) 2025 Michael Peter Christen for modifications: + * The code was modified to fit the YaCy AI project: + * - back-port to Java 11 (removal of Vector API operations and record types) + * - removal of interactive mode and system.out printing + * - separation of the classes in the single java and refactoring + * - run-time performance optimizations for dot product computation of quantized values + * - joining of llama3/qwen2 into one code base; multi-arch options + * - alignment with code from https://github.com/ggml-org/llama.cpp/ + */ + +package net.yacy.ai.llama3.Model; + +import java.util.Objects; + + +public final class Pair<F, S> { + public final F first; + public final S second; + + public Pair(F first, S second) { + this.first = first; + this.second = second; + } + + public F first() { return first; } + public S second() { return second; } + + // Optional factory method (avoids 'new' keyword) + public static <F, S> Pair<F, S> of(F first, S second) { + return new Pair<>(first, second); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Pair)) return false; + Pair<?, ?> pair = (Pair<?, ?>) o; + return Objects.equals(first, pair.first) && + Objects.equals(second, pair.second); + } + + @Override + public int hashCode() { + return Objects.hash(first, second); + } + +}
\ No newline at end of file diff --git a/source/net/yacy/ai/llama3/Model/Tokenizer.java b/source/net/yacy/ai/llama3/Model/Tokenizer.java new file mode 100644 index 000000000..8b3a118a1 --- /dev/null +++ b/source/net/yacy/ai/llama3/Model/Tokenizer.java @@ -0,0 +1,295 @@ +/** + * Tokenizer.java + + * This file was extracted from the llama3/qwen2 projects + * https://github.com/mukel/llama3.java + * https://github.com/mukel/qwen2.svm.java + * + * License: MIT License + * + * Copyright (c) 2024 Andrej Karpathy (for llama2.c) + * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2) + * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp) + * Copyright (c) 2025 Michael Peter Christen for modifications: + * The code was modified to fit the YaCy AI project: + * - back-port to Java 11 (removal of Vector API operations and record types) + * - removal of interactive mode and system.out printing + * - separation of the classes in the single java and refactoring + * - run-time performance optimizations for dot product computation of quantized values + * - joining of llama3/qwen2 into one code base; multi-arch options + * - alignment with code from https://github.com/ggml-org/llama.cpp/ + */ + +package net.yacy.ai.llama3.Model; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.nio.charset.StandardCharsets; + +/** + * Byte Pair Encoding tokenizer. + * <p> + * Based on <a href="https://github.com/karpathy/minbpe">minbpe</a>, algorithmically follows along the + * <a href="https://github.com/openai/gpt-2/blob/master/src/encoder.py">GPT 2 tokenizer</a> + */ +public class Tokenizer { + private final Pattern compiledPattern; + private final Vocabulary vocabulary; + private final Map<Pair<Integer, Integer>, Integer> merges; + private final Map<String, Integer> specialTokens; + private final int[] tokenTypes; + + public String regexPattern() { + if (compiledPattern == null) { + return null; + } + return compiledPattern.pattern(); + } + + public Map<String, Integer> getSpecialTokens() { + return specialTokens; + } + + public boolean isSpecialToken(int tokenIndex) { + return specialTokens.containsValue(tokenIndex); + } + + public int getTokenType(int tokenIndex) { + return tokenTypes[tokenIndex]; + } + + public Tokenizer(Vocabulary vocabulary, List<Pair<Integer, Integer>> merges, String regexPattern, Map<String, Integer> specialTokens, int[] tokenTypes) { + this.vocabulary = vocabulary; + this.compiledPattern = regexPattern != null ? Pattern.compile(regexPattern) : null; + this.specialTokens = new HashMap<>(specialTokens); + this.merges = new HashMap<>(); + this.tokenTypes = tokenTypes; + for (Pair<Integer, Integer> pair : merges) { + int firstIndex = pair.first(); + int secondIndex = pair.second(); + int mergeIndex = vocabulary.getIndex(vocabulary.get(firstIndex) + vocabulary.get(secondIndex)).orElseThrow(); + this.merges.put(pair, mergeIndex); + } + } + + private int[] encodeImpl(String text) { + return encode(text, Set.of()).stream().mapToInt(i -> i).toArray(); + } + + /** + * Unlike {@link #encodeOrdinary(String)}, this function handles special tokens. + * allowed_special: can be "all"|"none"|"none_raise" or a custom set of special tokens + * if none_raise, then an error is raised if any special token is encountered in text + * this is the default tiktoken behavior right now as well + * any other behavior is either annoying, or a major footgun. + */ + List<Integer> encode(String text, Set<String> allowedSpecial) { + // decode the user desire w.r.t. handling of special tokens + Set<String> special = allowedSpecial; + assert getSpecialTokens().keySet().containsAll(special); + if (special.isEmpty()) { + // shortcut: if no special tokens, just use the ordinary encoding + return encodeOrdinary(text); + } + + // otherwise, we have to be careful with potential special tokens in text + // we handle special tokens by splitting the text + // based on the occurrence of any exact match with any of the special tokens + // we can use re.split for this. note that surrounding the pattern with () + // makes it into a capturing group, so the special tokens will be included + String specialPattern = special + .stream() + .map(Pattern::quote) + .collect(Collectors.joining("|", "(", ")")); + + String[] specialChunks = text.split(specialPattern); + // now all the special characters are separated from the rest of the text + // all chunks of text are encoded separately, then results are joined + List<Integer> ids = new ArrayList<>(); + for (String part : specialChunks) { + if (special.contains(part)) { + // this is a special token, encode it separately as a special case + ids.add(getSpecialTokens().get(part)); + } else { + // this is an ordinary sequence, encode it normally + ids.addAll(encodeOrdinary(part)); + } + } + return ids; + } + + private static List<String> findAll(Pattern pattern, String text) { + List<String> allMatches = new ArrayList<>(); + Matcher matcher = pattern.matcher(text); + while (matcher.find()) { + allMatches.add(matcher.group()); + } + return allMatches; + } + + /** + * Encoding that ignores any special tokens. + */ + public List<Integer> encodeOrdinary(String text) { + // split text into chunks of text by categories defined in regex pattern + List<String> textChunks = findAll(compiledPattern, text); + // all chunks of text are encoded separately, then results are joined + List<Integer> ids = new ArrayList<>(); + for (String chunk : textChunks) { + List<Integer> chunkIds = encodeChunk(chunk); + ids.addAll(chunkIds); + } + return ids; + } + + private Map<Pair<Integer, Integer>, Integer> getStats(List<Integer> ids) { + Map<Pair<Integer, Integer>, Integer> map = new HashMap<>(); + for (int i = 0; i + 1 < ids.size(); i++) { + Pair<Integer, Integer> key = new Pair<>(ids.get(i), ids.get(i + 1)); + map.put(key, map.getOrDefault(key, 0) + 1); + } + return map; + } + + private List<Integer> encodeChunk(String chunk) { + // return the token ids + // let's begin. first, convert all bytes to integers in range 0..255 + List<Integer> ids = new ArrayList<>(); + for (int b : chunk.toCharArray()) { + int tokenIndex = this.vocabulary.getIndex(String.valueOf((char) b)).orElseThrow(); + ids.add(tokenIndex); + } + + while (ids.size() >= 2) { + // find the pair with the lowest merge index + Map<Pair<Integer, Integer>, Integer> stats = getStats(ids); + Pair<Integer, Integer> pair = stats.keySet().stream().min(Comparator.comparingInt(key -> this.merges.getOrDefault(key, Integer.MAX_VALUE))).orElseThrow(); + // subtle: if there are no more merges available, the key will + // result in an inf for every single pair, and the min will be + // just the first pair in the list, arbitrarily + // we can detect this terminating case by a membership check + if (!this.merges.containsKey(pair)) { + break; // nothing else can be merged anymore + } + // otherwise let's merge the best pair (lowest merge index) + int idx = this.merges.get(pair); + ids = merge(ids, pair, idx); + } + return ids; + } + + private static List<Integer> merge(List<Integer> ids, Pair<Integer, Integer> pair, int idx) { + List<Integer> newids = new ArrayList<>(); + int i = 0; + while (i < ids.size()) { + // if not at the very last position AND the pair matches, replace it + if (ids.get(i).equals(pair.first()) && i < ids.size() - 1 && ids.get(i + 1).equals(pair.second())) { + newids.add(idx); + i += 2; + } else { + newids.add(ids.get(i)); + i += 1; + } + } + return newids; + } + + public String decodeImpl(List<Integer> tokens) { + StringBuilder sb = new StringBuilder(); + for (int token : tokens) { + String tokenString = vocabulary.get(token); + sb.append(tokenString); + } + return sb.toString(); + } + + /** + * Returns list of utf-8 byte and a corresponding list of unicode strings. + * The reversible bpe codes work on unicode strings. + * This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + * When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + * This is a significant percentage of your normal, say, 32K bpe vocab. + * To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + * And avoids mapping to whitespace/control characters the bpe code barfs on. + */ + private static Map<Integer, Integer> bytesToUnicode() { + List<Integer> bs = new ArrayList<>(); + IntStream.rangeClosed('!', '~').forEach(bs::add); + IntStream.rangeClosed('¡', '¬').forEach(bs::add); + IntStream.rangeClosed('®', 'ÿ').forEach(bs::add); + + List<Integer> cs = new ArrayList<>(bs); + int n = 0; + for (int b = 0; b < 256; ++b) { + if (!bs.contains(b)) { + bs.add(b); + cs.add(256 + n); + n += 1; + } + } + + // return dict(zip(bs, cs)) + return IntStream.range(0, bs.size()) + .boxed() + .collect(Collectors.toMap(bs::get, cs::get)); + } + + static final Map<Integer, Integer> BYTE_ENCODER = bytesToUnicode(); + static final Map<Integer, Integer> BYTE_DECODER = BYTE_ENCODER.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getValue, Map.Entry::getKey)); + + public int[] encode(String text) { + StringBuilder sb = new StringBuilder(); + byte[] bytes = text.getBytes(StandardCharsets.UTF_8); + for (byte b : bytes) { + sb.appendCodePoint(BYTE_ENCODER.get(Byte.toUnsignedInt(b))); + } + return encodeImpl(sb.toString()); + } + + /* + public static String replaceControlCharacters(int[] codePoints) { + // we don't want to print control characters + // which distort the output (e.g. \n or much worse) + // https://stackoverflow.com/questions/4324790/removing-control-characters-from-a-string-in-python/19016117#19016117 + // http://www.unicode.org/reports/tr44/#GC_Values_Table\ + StringBuilder chars = new StringBuilder(); + for (int cp : codePoints) { + if (Character.getType(cp) == Character.CONTROL && cp != '\n') { + chars.append("\\u").append(HexFormat.of().toHexDigits(cp, 4)); // escape + } else { + chars.appendCodePoint(cp); // this character is ok + } + } + return chars.toString(); + } + + public static String replaceControlCharacters(String str) { + return replaceControlCharacters(str.codePoints().toArray()); + } + */ + + public List<Integer> encodeAsList(String text) { + return Arrays.stream(encode(text)).boxed().collect(Collectors.toList()); + } + + public String decode(List<Integer> tokens) { + String decoded = decodeImpl(tokens); + int[] decodedBytesAsInts = decoded.codePoints().map(BYTE_DECODER::get).toArray(); + byte[] rawBytes = new byte[decodedBytesAsInts.length]; + for (int i = 0; i < decoded.length(); i++) { + rawBytes[i] = (byte) decodedBytesAsInts[i]; + } + return new String(rawBytes, StandardCharsets.UTF_8); + } +} diff --git a/source/net/yacy/ai/llama3/Model/Vocabulary.java b/source/net/yacy/ai/llama3/Model/Vocabulary.java new file mode 100644 index 000000000..f2ca7e4dd --- /dev/null +++ b/source/net/yacy/ai/llama3/Model/Vocabulary.java @@ -0,0 +1,65 @@ +/** + * Vocabulary.java + + * This file was extracted from the llama3/qwen2 projects + * https://github.com/mukel/llama3.java + * https://github.com/mukel/qwen2.svm.java + * + * License: MIT License + * + * Copyright (c) 2024 Andrej Karpathy (for llama2.c) + * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2) + * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp) + * Copyright (c) 2025 Michael Peter Christen for modifications: + * The code was modified to fit the YaCy AI project: + * - back-port to Java 11 (removal of Vector API operations and record types) + * - removal of interactive mode and system.out printing + * - separation of the classes in the single java and refactoring + * - run-time performance optimizations for dot product computation of quantized values + * - joining of llama3/qwen2 into one code base; multi-arch options + * - alignment with code from https://github.com/ggml-org/llama.cpp/ + */ + +package net.yacy.ai.llama3.Model; + +import java.util.*; +import java.util.stream.*; + +public final class Vocabulary { + private final String[] tokens; + private final float[] scores; + private final Map<String, Integer> tokenToIndex; + + // Primary constructor + public Vocabulary(String[] tokens, float[] scores, Map<String, Integer> tokenToIndex) { + this.tokens = tokens == null ? null : Arrays.copyOf(tokens, tokens.length); + this.scores = scores == null ? null : Arrays.copyOf(scores, scores.length); + this.tokenToIndex = tokenToIndex == null ? null : new HashMap<>(tokenToIndex); + } + + // Secondary constructor + public Vocabulary(String[] vocabulary, float[] scores) { + this(vocabulary, scores, + IntStream.range(0, vocabulary.length) + .boxed() + .collect(Collectors.toMap(i -> vocabulary[i], i -> i)) + ); + } + + public String[] tokens() { + return Arrays.copyOf(tokens, tokens.length); + } + + public String get(int tokenIndex) { + return tokens[tokenIndex]; + } + + public OptionalInt getIndex(String token) { + Integer value = tokenToIndex.get(token); + return value != null ? OptionalInt.of(value) : OptionalInt.empty(); + } + + public int size() { + return tokens.length; + } +}
\ No newline at end of file diff --git a/source/net/yacy/ai/llama3/Sampler.java b/source/net/yacy/ai/llama3/Sampler.java new file mode 100644 index 000000000..c94876cab --- /dev/null +++ b/source/net/yacy/ai/llama3/Sampler.java @@ -0,0 +1,175 @@ +/** + * Sampler.java + + * This file was extracted from the llama3/qwen2 projects + * https://github.com/mukel/llama3.java + * https://github.com/mukel/qwen2.svm.java + * + * License: MIT License + * + * Copyright (c) 2024 Andrej Karpathy (for llama2.c) + * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2) + * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp) + * Copyright (c) 2025 Michael Peter Christen for modifications: + * The code was modified to fit the YaCy AI project: + * - back-port to Java 11 (removal of Vector API operations and record types) + * - removal of interactive mode and system.out printing + * - separation of the classes in the single java and refactoring + * - run-time performance optimizations for dot product computation of quantized values + * - joining of llama3/qwen2 into one code base; multi-arch options + * - alignment with code from https://github.com/ggml-org/llama.cpp/ + */ + +package net.yacy.ai.llama3; + +import java.util.Comparator; +import java.util.Random; + +import net.yacy.ai.llama3.Tensor.FloatTensor; + +@FunctionalInterface +interface Sampler { + int sampleToken(FloatTensor logits); + + Sampler ARGMAX = FloatTensor::argmax; + + static Sampler selectSampler(int vocabularySize, float temperature, float topp, long rngSeed) { + Sampler sampler; + if (temperature == 0.0f) { + // greedy argmax sampling: take the token with the highest probability + sampler = Sampler.ARGMAX; + } else { + // we sample from this distribution to get the next token + // RandomGeneratorFactory.getDefault().create(rngSeed); requires additonal native-image configuration. + Random rng = new Random(rngSeed); + Sampler innerSampler; + if (topp <= 0 || topp >= 1) { + // simply sample from the predicted probability distribution + innerSampler = new CategoricalSampler(rng); + } else { + // top-p (nucleus) sampling, clamping the least likely tokens to zero + innerSampler = new ToppSampler(vocabularySize, topp, rng); + } + sampler = logits -> { + // apply the temperature to the logits + logits.divideInPlace(0, logits.size(), temperature); + // apply softmax to the logits to get the probabilities for next token + logits.softmaxInPlace(0, logits.size()); + return innerSampler.sampleToken(logits); + }; + } + return sampler; + } + + static class CategoricalSampler implements Sampler { + + final Random rng; + + public CategoricalSampler(Random rng) { + this.rng = rng; + } + + @Override + public int sampleToken(FloatTensor logits) { + // sample index from probabilities (they must sum to 1!) + float random0to1 = rng.nextFloat(); + float cdf = 0.0f; + for (int i = 0; i < logits.size(); i++) { + cdf += logits.getFloat(i); + if (random0to1 < cdf) { + return i; + } + } + return logits.size() - 1; // in case of rounding errors + } + } + + static class ToppSampler implements Sampler { + + final int[] indices; + final float topp; + final Random rng; + + public ToppSampler(int maxNumberOfElements, float topp, Random rng) { + this.indices = new int[maxNumberOfElements]; + this.topp = topp; + this.rng = rng; + } + + static void swap(int[] array, int from, int to) { + int tmp = array[from]; + array[from] = array[to]; + array[to] = tmp; + } + + static void siftDown(int[] array, int from, int n, Comparator<Integer> comparator) { + int prev = from, next; + while ((next = 2 * prev + 1) < n) { + int r = 2 * prev + 2; + if (r < n && comparator.compare(array[r], array[next]) < 0) { + next = r; + } + if (comparator.compare(array[next], array[prev]) < 0) { + swap(array, prev, next); + prev = next; + } else { + break; + } + } + } + + @Override + public int sampleToken(FloatTensor logits) { + // top-p sampling (or "nucleus sampling") samples from the smallest set of + // tokens that exceed probability topp. This way we never sample tokens that + // have very low probabilities and are less likely to go "off the rails". + Comparator<Integer> comparator = Comparator.comparingDouble(logits::getFloat).reversed(); + + int n = logits.size(); + int head = 0; + int tail = n - 1; + // values smaller than (1 - topp) / (n - 1) cannot be part of the result + // so for efficiency we crop these out as candidates before sorting + float cutoff = (1.0f - topp) / (n - 1); + for (int i = 0; i < indices.length; i++) { + if (logits.getFloat(i) >= cutoff) { + indices[head++] = i; + } else { + indices[tail--] = i; + } + } + + int n0 = head; + // build heap O(n0) + for (int i = n0 / 2 - 1; i >= 0; --i) { + siftDown(indices, i, n0, comparator); + } + + // truncate the list where cumulative probability of the largest k elements exceeds topp + // O(k lg n0) + float cumulativeProb = 0.0f; + int lastIndex = 0; + for (int i = n0 - 1; i >= 0; i--) { + swap(indices, 0, i); + cumulativeProb += logits.getFloat(indices[i]); + if (cumulativeProb > topp) { + lastIndex = i; + break; // we've exceeded topp by including lastIndex + } + siftDown(indices, 0, i - 1, comparator); + } + + // sample from the truncated list + float r = rng.nextFloat() * cumulativeProb; + float cdf = 0.0f; + for (int i = n0 - 1; i >= lastIndex; i--) { + cdf += logits.getFloat(indices[i]); + if (r < cdf) { + return indices[i]; + } + } + + return indices[lastIndex]; // in case of rounding errors + } + } +} diff --git a/source/net/yacy/ai/llama3/Tensor/ArrayFloatTensor.java b/source/net/yacy/ai/llama3/Tensor/ArrayFloatTensor.java new file mode 100644 index 000000000..9e0155eba --- /dev/null +++ b/source/net/yacy/ai/llama3/Tensor/ArrayFloatTensor.java @@ -0,0 +1,177 @@ +/** + * ArrayFloatTensor.java + + * This file was extracted from the llama3/qwen2 projects + * https://github.com/mukel/llama3.java + * https://github.com/mukel/qwen2.svm.java + * + * License: MIT License + * + * Copyright (c) 2024 Andrej Karpathy (for llama2.c) + * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2) + * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp) + * Copyright (c) 2025 Michael Peter Christen for modifications: + * The code was modified to fit the YaCy AI project: + * - back-port to Java 11 (removal of Vector API operations and record types) + * - removal of interactive mode and system.out printing + * - separation of the classes in the single java and refactoring + * - run-time performance optimizations for dot product computation of quantized values + * - joining of llama3/qwen2 into one code base; multi-arch options + * - alignment with code from https://github.com/ggml-org/llama.cpp/ + */ + +package net.yacy.ai.llama3.Tensor; + +import java.util.Arrays; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; + +import net.yacy.ai.llama3.Model.GGMLType; + +public final class ArrayFloatTensor extends FloatTensor { + + public final float[] values; + private static final VarHandle FLOAT_ARRAY_HANDLE; + + static { + try { + FLOAT_ARRAY_HANDLE = MethodHandles.arrayElementVarHandle(float[].class); + } catch (Exception e) { + throw new RuntimeException("Failed to get VarHandle for float[]", e); + } + } + + public ArrayFloatTensor(final float[] values) { + this.values = values; + } + + public static FloatTensor allocate(final int... dims) { + int numberOfElements = FloatTensor.numberOfElements(dims); + return new ArrayFloatTensor(new float[numberOfElements]); + } + + @Override + public final int size() { + return this.values.length; + } + + @Override + public final float getFloat(final int index) { + return (float) FLOAT_ARRAY_HANDLE.get(this.values, index); + } + + @Override + public final void setFloat(final int index, final float value) { + FLOAT_ARRAY_HANDLE.set(this.values, index, value); + } + + @Override + public GGMLType type() { + return GGMLType.F32; + } + + @Override + public final FloatTensor fillInPlace(final int thisOffset, final int size, final float value) { + Arrays.fill(this.values, thisOffset, thisOffset + size, value); + return this; + } + + @Override + public final FloatTensor mapInPlace(final int thisOffset, final int size, MapFunction mapFunction) { + int endIndex = thisOffset + size; + for (int i = thisOffset; i < endIndex; ++i) { + this.values[i] = mapFunction.apply(this.values[i]); + } + return this; + } + + @Override + public final void copyTo(final int thisOffset, final FloatTensor that, final int thatOffset, final int size) { + final int delta = thisOffset - thatOffset; + if (that instanceof ArrayFloatTensor) { + final ArrayFloatTensor aft = (ArrayFloatTensor) that; + int endOffset = thatOffset + size; + for (int i = thatOffset; i < endOffset; ++i) { + FLOAT_ARRAY_HANDLE.set(aft.values, i, (float) FLOAT_ARRAY_HANDLE.get(this.values, i + delta)); + } + } else { + int endOffset = thatOffset + size; + for (int i = thatOffset; i < endOffset; ++i) { + that.setFloat(i, (float) FLOAT_ARRAY_HANDLE.get(this.values, i + delta)); + } + } + } + + @Override + public final float dot(final int thisOffset, final FloatTensor that, final int thatOffset, final int size) { + float result = 0f; + if (that instanceof ArrayFloatTensor) { + final ArrayFloatTensor aft = (ArrayFloatTensor) that; + final float[] a = this.values; + final float[] b = aft.values; + + for (int i = 0; i < size; i++) { + final float valA = (float) FLOAT_ARRAY_HANDLE.get(a, thisOffset + i); + final float valB = (float) FLOAT_ARRAY_HANDLE.get(b, thatOffset + i); + result += valA * valB; + } + } else { + final float[] a = this.values; + for (int i = 0; i < size; i++) { + final float valA = (float) FLOAT_ARRAY_HANDLE.get(a, thisOffset + i); + result += valA * that.getFloat(thatOffset + i); + } + } + return result; + } + + @Override + public final void matmul(final FloatTensor that, final FloatTensor out, final int dim0, final int dim1) { + if (that instanceof ArrayFloatTensor) { + parallelFor(0, dim0, i -> ((ArrayFloatTensor) out).values[i] = this.dot(i * dim1, that, 0, dim1)); + } else { + parallelFor(0, dim0, i -> out.setFloat(i, this.dot(i * dim1, that, 0, dim1))); + } + } + + @Override + public final void matmul(final int context, final FloatTensor[] that, final FloatTensor[] out, final int dim0, final int dim1) { + if (that.length != out.length) { + throw new IllegalArgumentException(String.format("that.len=%d, out.len=%d", that.length, out.length)); + } + parallelForLong(0, dim0 * context, ti -> { + int idxArr = (int) (ti / dim0); + int i = (int) (ti % dim0); + out[idxArr].setFloat(i, this.dot(i * dim1, that[idxArr], 0, dim1)); + }); + } + + @Override + public final FloatTensor saxpyInPlace(final int thisOffset, final FloatTensor that, final int thatOffset, final int size, final float a) { + if (that instanceof Q4_0FloatTensor) { + Q4_0FloatTensor qft = (Q4_0FloatTensor) that; + final float[] decodedBlock = Q4_0FloatTensor.scratchBuffer.get(); + int remaining = size; + int i = 0; + + while (remaining > 0) { + int chunkSize = Math.min(remaining, GGMLType.Q4_0.blockSize); + qft.getFloatArray(thatOffset + i, decodedBlock, 0, chunkSize); + for (int j = 0; j < chunkSize; ++j) { + int dstIdx = thisOffset + i + j; + this.setFloat(dstIdx, a * decodedBlock[j] + this.getFloat(dstIdx)); + } + i += chunkSize; + remaining -= chunkSize; + } + } else { + for (int i = 0; i < size; ++i) { + int idx = thisOffset + i; + this.setFloat(idx, a * that.getFloat(thatOffset + i) + this.getFloat(idx)); + } + } + + return this; + } + +}
\ No newline at end of file diff --git a/source/net/yacy/ai/llama3/Tensor/BF16FloatTensor.java b/source/net/yacy/ai/llama3/Tensor/BF16FloatTensor.java new file mode 100644 index 000000000..0de5ac525 --- /dev/null +++ b/source/net/yacy/ai/llama3/Tensor/BF16FloatTensor.java @@ -0,0 +1,74 @@ +/** + * BF16FloatTensor.java + + * This file was extracted from the llama3/qwen2 projects + * https://github.com/mukel/llama3.java + * https://github.com/mukel/qwen2.svm.java + * + * License: MIT License + * + * Copyright (c) 2024 Andrej Karpathy (for llama2.c) + * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2) + * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp) + * Copyright (c) 2025 Michael Peter Christen for modifications: + * The code was modified to fit the YaCy AI project: + * - back-port to Java 11 (removal of Vector API operations and record types) + * - removal of interactive mode and system.out printing + * - separation of the classes in the single java and refactoring + * - run-time performance optimizations for dot product computation of quantized values + * - joining of llama3/qwen2 into one code base; multi-arch options + * - alignment with code from https://github.com/ggml-org/llama.cpp/ + */ + +package net.yacy.ai.llama3.Tensor; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import net.yacy.ai.llama3.Model.GGMLType; + +public final class BF16FloatTensor extends FloatTensor { + + final int size; + final ByteBuffer buffer; + + public BF16FloatTensor(int size, ByteBuffer buffer) { + if (buffer.remaining() < size * GGMLType.BFLOAT16_BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + this.size = size; + this.buffer = buffer.duplicate().order(ByteOrder.nativeOrder()); + } + + public BF16FloatTensor(final float[] values) { + this.size = values.length; + this.buffer = ByteBuffer.allocateDirect(size * GGMLType.BFLOAT16_BYTES).order(ByteOrder.nativeOrder()); + for (int i = 0; i < size; i++) { + setFloat(i, values[i]); + } + } + + @Override + public int size() { + return size; + } + + @Override + public GGMLType type() { + return GGMLType.BF16; + } + + @Override + public final void setFloat(int index, float value) { + assert 0 <= index && index < size; + int hBits = Float.floatToIntBits(value) >>> 16; // convert float to bfloat16 + buffer.putShort(index * GGMLType.BFLOAT16_BYTES, (short) hBits); + } + + @Override + public final float getFloat(int index) { + assert 0 <= index && index < size; + return Float.intBitsToFloat(buffer.getShort(index * GGMLType.BFLOAT16_BYTES) << 16); + } + +} diff --git a/source/net/yacy/ai/llama3/Tensor/DirectBufferFloatTensor.java b/source/net/yacy/ai/llama3/Tensor/DirectBufferFloatTensor.java new file mode 100644 index 000000000..64c615fec --- /dev/null +++ b/source/net/yacy/ai/llama3/Tensor/DirectBufferFloatTensor.java @@ -0,0 +1,111 @@ +/** + * DirectBufferFloatTensor + * Copyright 2025 by Michael Peter Christen + * First released 19.06.2025 at https://yacy.net + * + ** This class was not part of the original llama3 implementation, + ** but added later by the author to support different architectures. + ** It therefore does not inherit the llama3 copyright. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.ai.llama3.Tensor; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.FloatBuffer; + +import net.yacy.ai.llama3.Model.GGMLType; + +public class DirectBufferFloatTensor extends FloatTensor { + + final FloatBuffer floatBuffer; + + public DirectBufferFloatTensor(ByteBuffer byteBuffer) { + if (byteBuffer.isDirect()) { + this.floatBuffer = byteBuffer.asFloatBuffer(); + } else { + int capacityBytes = byteBuffer.remaining(); + ByteBuffer directByteBuffer = ByteBuffer.allocateDirect(capacityBytes).order(byteBuffer.order()); + directByteBuffer.put(byteBuffer.duplicate()); + directByteBuffer.flip(); + this.floatBuffer = directByteBuffer.asFloatBuffer(); + } + } + + public DirectBufferFloatTensor(final float[] values) { + int capacityBytes = values.length * Float.BYTES; + ByteBuffer directByteBuffer = ByteBuffer.allocateDirect(capacityBytes).order(ByteOrder.nativeOrder()); + this.floatBuffer = directByteBuffer.asFloatBuffer(); + this.floatBuffer.put(values); + } + + public static FloatTensor allocate(final int... dims) { + int numberOfElements = FloatTensor.numberOfElements(dims); + int bytesNeeded = numberOfElements * Float.BYTES; + ByteBuffer buffer = ByteBuffer.allocateDirect(bytesNeeded).order(ByteOrder.nativeOrder()); + return new DirectBufferFloatTensor(buffer); + } + + @Override + public final int size() { + return this.floatBuffer.capacity(); + } + + @Override + public final float getFloat(final int index) { + return this.floatBuffer.get(index); + } + + @Override + public final void setFloat(final int index, final float value) { + this.floatBuffer.put(index, value); + } + + @Override + public final GGMLType type() { + return GGMLType.F32; + } + + @Override + public final float dot(final int thisOffset, final FloatTensor that, final int thatOffset, final int size) { + float result = 0f; + for (int j = 0; j < size; j++) { + result += this.floatBuffer.get(thisOffset + j) * that.getFloat(thatOffset + j); + } + return result; + } + + @Override + public final FloatTensor fillInPlace(final int thisOffset, final int size, final float value) { + int end = thisOffset + size; + for (int i = thisOffset; i < end; i++) { + floatBuffer.put(i, value); + } + return this; + } + + @Override + public final FloatTensor mapInPlace(final int thisOffset, final int size, MapFunction mapFunction) { + int end = thisOffset + size; + for (int i = thisOffset; i < end; i++) { + float current = floatBuffer.get(i); + floatBuffer.put(i, mapFunction.apply(current)); + } + return this; + } + +}
\ No newline at end of file diff --git a/source/net/yacy/ai/llama3/Tensor/F16FloatTensor.java b/source/net/yacy/ai/llama3/Tensor/F16FloatTensor.java new file mode 100644 index 000000000..d41fdbc87 --- /dev/null +++ b/source/net/yacy/ai/llama3/Tensor/F16FloatTensor.java @@ -0,0 +1,75 @@ +/** + * F16FloatTensor.java + + * This file was extracted from the llama3/qwen2 projects + * https://github.com/mukel/llama3.java + * https://github.com/mukel/qwen2.svm.java + * + * License: MIT License + * + * Copyright (c) 2024 Andrej Karpathy (for llama2.c) + * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2) + * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp) + * Copyright (c) 2025 Michael Peter Christen for modifications: + * The code was modified to fit the YaCy AI project: + * - back-port to Java 11 (removal of Vector API operations and record types) + * - removal of interactive mode and system.out printing + * - separation of the classes in the single java and refactoring + * - run-time performance optimizations for dot product computation of quantized values + * - joining of llama3/qwen2 into one code base; multi-arch options + * - alignment with code from https://github.com/ggml-org/llama.cpp/ + */ + + +package net.yacy.ai.llama3.Tensor; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import net.yacy.ai.llama3.Model.GGMLType; + +public final class F16FloatTensor extends FloatTensor { + + final int size; + final ByteBuffer buffer; + + public F16FloatTensor(int size, ByteBuffer buffer) { + if (buffer.remaining() < size * GGMLType.FLOAT16_BYTES) { + throw new IllegalArgumentException("Buffer too small"); + } + this.size = size; + this.buffer = buffer.duplicate().order(ByteOrder.nativeOrder()); + } + + public F16FloatTensor(final float[] values) { + this.size = values.length; + this.buffer = ByteBuffer.allocateDirect(size * GGMLType.FLOAT16_BYTES).order(ByteOrder.nativeOrder()); + for (int i = 0; i < size; i++) { + setFloat(i, values[i]); + } + } + + @Override + public int size() { + return size; + } + + @Override + public GGMLType type() { + return GGMLType.F16; + } + + @Override + public final void setFloat(int index, float value) { + assert 0 <= index && index < size; + short hBits = floatToFloat16(value); + buffer.putShort(index * GGMLType.FLOAT16_BYTES, hBits); + } + + @Override + public final float getFloat(int index) { + assert 0 <= index && index < size; + return float16ToFloat(buffer.getShort(index * GGMLType.FLOAT16_BYTES)); + } + +} diff --git a/source/net/yacy/ai/llama3/Tensor/FloatTensor.java b/source/net/yacy/ai/llama3/Tensor/FloatTensor.java new file mode 100644 index 000000000..4457b10d0 --- /dev/null +++ b/source/net/yacy/ai/llama3/Tensor/FloatTensor.java @@ -0,0 +1,286 @@ +/** + * FloatTensor.java + + * This file was extracted from the llama3/qwen2 projects + * https://github.com/mukel/llama3.java + * https://github.com/mukel/qwen2.svm.java + * + * License: MIT License + * + * Copyright (c) 2024 Andrej Karpathy (for llama2.c) + * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2) + * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp) + * Copyright (c) 2025 Michael Peter Christen for modifications: + * The code was modified to fit the YaCy AI project: + * - back-port to Java 11 (removal of Vector API operations and record types) + * - removal of interactive mode and system.out printing + * - separation of the classes in the single java and refactoring + * - run-time performance optimizations for dot product computation of quantized values + * - joining of llama3/qwen2 into one code base; multi-arch options + * - alignment with code from https://github.com/ggml-org/llama.cpp/ + */ + +package net.yacy.ai.llama3.Tensor; + +import java.util.Arrays; +import java.util.function.IntConsumer; +import java.util.function.LongConsumer; +import java.util.stream.IntStream; +import java.util.stream.LongStream; + +import net.yacy.ai.llama3.Model.GGMLType; + +/** + * Over-simplified, shapeless, float tensor. + * <p> + * Not a strict tensor, but rather just a sequence of floats, not required to be backed by memory + * e.g. can represent a sequence of quantized floats. + */ +public abstract class FloatTensor { + + /** + * Converts a 16-bit float (half-precision) to a 32-bit float (single-precision). + * + * @param h the half-precision float as a short + * @return the single-precision float + */ + public final static float float16ToFloat(short h) { + + final int hBits = h & 0xFFFF; // treat as unsigned + final int sign = (hBits >>> 15) & 0x00000001; + + int exp = (hBits >>> 10) & 0x0000001F; + int mant = hBits & 0x000003FF; + int fBits; + + if (exp == 0) { + if (mant == 0) { + // zero + fBits = sign << 31; + } else { + // subnormal + while ((mant & 0x00000400) == 0) { + mant <<= 1; + exp -= 1; + } + exp += 1; + mant &= ~0x00000400; + fBits = (sign << 31) | ((exp + 127 - 15) << 23) | (mant << 13); + } + } else if (exp == 31) { + // Inf/NaN + fBits = (sign << 31) | 0x7F800000 | (mant << 13); + } else { + // normalized number + fBits = (sign << 31) | ((exp + 127 - 15) << 23) | (mant << 13); + } + + return Float.intBitsToFloat(fBits); + } + + /** + * Converts a 32-bit float (single-precision) to a 16-bit float (half-precision). + * + * @param f the single-precision float + * @return the half-precision float as a short + */ + public final static short floatToFloat16(final float f) { + final int fBits = Float.floatToIntBits(f); + final int sign = (fBits >>> 31) & 0x00000001; + final int exp = (fBits >>> 23) & 0x000000FF; + final int mant = fBits & 0x007FFFFF; + + short hBits; + + if (exp == 0xFF) { + // Inf/NaN + hBits = (short) ((sign << 15) | 0x7C00 | (mant >>> 13)); + } else if (exp < 112) { + // subnormal or zero + hBits = (short) (sign << 15); + } else if (exp > 143) { + // overflow to Inf + hBits = (short) ((sign << 15) | 0x7C00); + } else { + // normalized number + hBits = (short) ((sign << 15) | ((exp - 112) << 10) | (mant >>> 13)); + } + + return hBits; + } + + public static void parallelFor(final int startInclusive, final int endExclusive, final IntConsumer action) { + if (startInclusive == 0 && endExclusive == 1) { + action.accept(0); + return; + } + IntStream.range(startInclusive, endExclusive).parallel().forEach(action); + } + + public static void parallelForLong(final long startInclusive, final long endExclusive, final LongConsumer action) { + if (startInclusive == 0 && endExclusive == 1) { + action.accept(0); + return; + } + LongStream.range(startInclusive, endExclusive).parallel().forEach(action); + } + + public abstract int size(); + + public abstract float getFloat(final int index); + + public abstract void setFloat(final int index, final float value); + + abstract GGMLType type(); + + public static int numberOfElements(final int... dimensions) { + assert Arrays.stream(dimensions).allMatch(i -> i > 0); + return Arrays.stream(dimensions).reduce(Math::multiplyExact).orElseThrow(); + } + + public static float scalarDot(final FloatTensor thiz, final int thisOffset, final FloatTensor that, final int thatOffset, final int size) { + float result = 0f; + for (int j = 0; j < size; j++) { + result += thiz.getFloat(thisOffset + j) * that.getFloat(thatOffset + j); + } + return result; + } + + public float dot(final int thisOffset, final FloatTensor that, final int thatOffset, final int size) { + return scalarDot(this, thisOffset, that, thatOffset, size); + } + + public void matmul(final FloatTensor that, final FloatTensor out, final int dim0, final int dim1) { + parallelFor(0, dim0, i -> out.setFloat(i, dot(i * dim1, that, 0, dim1))); + } + + public void matmul(final int context, final FloatTensor[] that, final FloatTensor[] out, final int dim0, final int dim1) { + if (that.length != out.length) { + throw new IllegalArgumentException(String.format("that.len=%d, out.len=%d", that.length, out.length)); + } + parallelForLong(0, dim0 * context, ti -> { + int idxArr = (int) (ti / dim0); + int i = (int) (ti % dim0); + out[idxArr].setFloat(i, dot(i * dim1, that[idxArr], 0, dim1)); + }); + } + + @FunctionalInterface + public interface AggregateFunction { + float apply(float acc, float value); + } + + public float reduce(final int thisOffset, final int size, final float seed, final AggregateFunction reduce) { + float result = seed; + for (int i = 0; i < size; ++i) { + result = reduce.apply(result, getFloat(thisOffset + i)); + } + return result; + } + + private float sum(final int thisOffset, final int size) { + return reduce(thisOffset, size, 0f, Float::sum); + } + + private float max(final int thisOffset, final int size) { + return reduce(thisOffset, size, Float.NEGATIVE_INFINITY, Float::max); + } + + public void copyTo(final int thisOffset, final FloatTensor that, final int thatOffset, final int size) { + int endOffset = thatOffset + size; + for (int i = thatOffset; i < endOffset; ++i) { + that.setFloat(i, this.getFloat(i - thatOffset + thisOffset)); + } + } + + private int argmax(final int thisOffset, final int size) { + assert size > 0; + int maxIndex = thisOffset; + float maxValue = this.getFloat(maxIndex); + int endIndex = thisOffset + size; + for (int i = thisOffset; i < endIndex; ++i) { + float f = this.getFloat(i); + if (f > maxValue) { + maxValue = f; + maxIndex = i; + } + } + return maxIndex; + } + + public int argmax() { + return argmax(0, size()); + } + + @FunctionalInterface + public interface MapFunction { + float apply(float value); + } + + @FunctionalInterface + public interface MapWithIndexFunction { + float apply(float value, int index); + } + + public FloatTensor mapInPlace(final int thisOffset, final int size, MapFunction mapFunction) { + int endIndex = thisOffset + size; + for (int i = thisOffset; i < endIndex; ++i) { + setFloat(i, mapFunction.apply(getFloat(i))); + } + return this; + } + + public final FloatTensor mapInPlace(final MapFunction mapFunction) { + return mapInPlace(0, size(), mapFunction); + } + + public final FloatTensor mapWithIndexInPlace(final int thisOffset, final int size, final FloatTensor.MapWithIndexFunction mapWithIndexFunction) { + int endOffset = thisOffset + size; + for (int i = thisOffset; i < endOffset; ++i) { + setFloat(i, mapWithIndexFunction.apply(getFloat(i), i)); + } + return this; + } + + private final FloatTensor addInPlace(final int thisOffset, final FloatTensor that, final int thatOffset, int size) { + return mapWithIndexInPlace(thisOffset, size, (value, index) -> value + that.getFloat(index - thisOffset + thatOffset)); + } + + public final FloatTensor addInPlace(final FloatTensor that) { + return addInPlace(0, that, 0, size()); + } + + private final FloatTensor multiplyInPlace(final int thisOffset, final FloatTensor that, final int thatOffset, final int size) { + return mapWithIndexInPlace(thisOffset, size, (value, index) -> value * that.getFloat(index - thisOffset + thatOffset)); + } + + public final FloatTensor multiplyInPlace(final FloatTensor that) { + return multiplyInPlace(0, that, 0, size()); + } + + public final FloatTensor divideInPlace(final int thisOffset, final int size, final float value) { + return mapInPlace(thisOffset, size, f -> f / value); + } + + public FloatTensor fillInPlace(final int thisOffset, final int size, final float value) { + return mapInPlace(thisOffset, size, unused -> value); + } + + public final FloatTensor softmaxInPlace(final int thisOffset, final int size) { + // find max value (for numerical stability) + float maxVal = max(thisOffset, size); + // exp and sum + mapInPlace(thisOffset, size, f -> (float) Math.exp(f - maxVal)); + float sum = sum(thisOffset, size); + // normalize + return divideInPlace(thisOffset, size, sum); + } + + public FloatTensor saxpyInPlace(final int thisOffset, final FloatTensor that, final int thatOffset, final int size, final float a) { + // this[thatOffset ... thatOffset + size) = a * that[thatOffset ... thatOffset + size) + this[thisOffset ... thisOffset + size) + for (int i = 0; i < size; ++i) { + this.setFloat(thisOffset + i, a * that.getFloat(thatOffset + i) + this.getFloat(thisOffset + i)); + } + return this; + } +}
\ No newline at end of file diff --git a/source/net/yacy/ai/llama3/Tensor/Q4_0FloatTensor.java b/source/net/yacy/ai/llama3/Tensor/Q4_0FloatTensor.java new file mode 100644 index 000000000..b4e60379a --- /dev/null +++ b/source/net/yacy/ai/llama3/Tensor/Q4_0FloatTensor.java @@ -0,0 +1,219 @@ +/** + * Q4_0FloatTensor.java + + * This file was extracted from the llama3/qwen2 projects + * https://github.com/mukel/llama3.java + * https://github.com/mukel/qwen2.svm.java + * + * License: MIT License + * + * Copyright (c) 2024 Andrej Karpathy (for llama2.c) + * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2) + * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp) + * Copyright (c) 2025 Michael Peter Christen for modifications: + * The code was modified to fit the YaCy AI project: + * - back-port to Java 11 (removal of Vector API operations and record types) + * - removal of interactive mode and system.out printing + * - separation of the classes in the single java and refactoring + * - run-time performance optimizations for dot product computation of quantized values + * - joining of llama3/qwen2 into one code base; multi-arch options + * - alignment with code from https://github.com/ggml-org/llama.cpp/ + */ + +package net.yacy.ai.llama3.Tensor; + +import java.nio.ByteBuffer; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; + +import net.yacy.ai.llama3.Model.GGMLType; + +public final class Q4_0FloatTensor extends FloatTensor { + + private final int size; + private final ByteBuffer buffer; + + public Q4_0FloatTensor(final int size, final ByteBuffer buffer) { + this.size = size; + this.buffer = buffer; + } + + private static final VarHandle FLOAT_ARRAY_HANDLE; + + static { + try { + FLOAT_ARRAY_HANDLE = MethodHandles.arrayElementVarHandle(float[].class); + } catch (Exception e) { + throw new RuntimeException("Failed to get VarHandle for float[]", e); + } + } + + @Override + public final int size() { + return size; + } + + @Override + public final void setFloat(final int index, final float value) { + throw new UnsupportedOperationException("setFloat"); + } + + @Override + public final GGMLType type() { + return GGMLType.Q4_0; + } + + private final static int LOG2_QUANT_BLOCK_SIZE = Integer.numberOfTrailingZeros(GGMLType.Q4_0.blockSize); // only if QUANT_BLOCK_SIZE == 2^LOG2_QUANT_BLOCK_SIZE + private final static int QUANT_HALF_BLOCK = GGMLType.Q4_0.blockSize / 2; // 16 + private final static int QUANT_FLOAT16_BYTES = GGMLType.FLOAT16_BYTES; // 2 + + @Override + public final float getFloat(final int index) { + assert 0 <= index && index < size; + + int blockIndex = index >>> LOG2_QUANT_BLOCK_SIZE; // index / QUANT_BLOCK_SIZE; + int blockOffset = blockIndex * GGMLType.Q4_0.typeSize; + final long offset = blockOffset; + float scale = FloatTensor.float16ToFloat(buffer.getShort((int) offset)); + final int modIndex = index & (GGMLType.Q4_0.blockSize - 1); //index % QUANT_BLOCK_SIZE; + final boolean isLow = modIndex < QUANT_HALF_BLOCK; + final int adjustedIndex = modIndex - (isLow ? 0 : QUANT_HALF_BLOCK); + final int dataIndex = blockOffset + QUANT_FLOAT16_BYTES + adjustedIndex; + final int packed = (buffer.get((int) (long) dataIndex)) & 0xFF; + final int nibble = isLow ? (packed & 0x0F) : ((packed >>> 4) & 0x0F); + //final float quant = nibble - 8; + return (nibble - 8) * scale; + } + + public final void getFloatArray(final int index, final float[] out, final int outOffset, final int length) { + int inPos = index; + int outPos = outOffset; + final int end = index + length; + + while (inPos < end) { + final int blockIndex = inPos >>> LOG2_QUANT_BLOCK_SIZE; + final int blockOffset = blockIndex * GGMLType.Q4_0.typeSize; + float scale = FloatTensor.float16ToFloat(buffer.getShort((int) (long) blockOffset)); + + final int blockStart = blockIndex * GGMLType.Q4_0.blockSize; + final int blockEnd = Math.min(blockStart + GGMLType.Q4_0.blockSize, end); + + // Process two values at a time + int i = inPos; + while (i + 1 < blockEnd) { + int blockMod0 = i & (GGMLType.Q4_0.blockSize - 1); + int blockMod1 = blockMod0 + 1; + + int byteOffset0 = blockOffset + QUANT_FLOAT16_BYTES + (blockMod0 < QUANT_HALF_BLOCK ? blockMod0 : blockMod0 - QUANT_HALF_BLOCK); + int byteOffset1 = blockOffset + QUANT_FLOAT16_BYTES + (blockMod1 < QUANT_HALF_BLOCK ? blockMod1 : blockMod1 - QUANT_HALF_BLOCK); + final long offset = byteOffset0; + + int packed0 = buffer.get((int) offset) & 0xFF; + final long offset1 = byteOffset1; + int packed1 = (byteOffset1 == byteOffset0) ? packed0 : (buffer.get((int) offset1) & 0xFF); + + // Decode first value from packed0 + int nibble0 = (blockMod0 < QUANT_HALF_BLOCK) ? (packed0 & 0x0F) : ((packed0 >>> 4) & 0x0F); + out[outPos++] = (nibble0 - 8) * scale; + + // Decode second value from packed1 + int nibble1 = (blockMod1 < QUANT_HALF_BLOCK) ? (packed1 & 0x0F) : ((packed1 >>> 4) & 0x0F); + out[outPos++] = (nibble1 - 8) * scale; + + i += 2; + } + + // Handle last element if length is odd + if (i < blockEnd) { + int blockMod = i & (GGMLType.Q4_0.blockSize - 1); + int byteOffset = blockOffset + QUANT_FLOAT16_BYTES + (blockMod < QUANT_HALF_BLOCK ? blockMod : blockMod - QUANT_HALF_BLOCK); + final long offset = byteOffset; + int packed = buffer.get((int) offset) & 0xFF; + int nibble = (blockMod < QUANT_HALF_BLOCK) ? (packed & 0x0F) : ((packed >>> 4) & 0x0F); + out[outPos++] = (nibble - 8) * scale; + i++; + } + + int copied = blockEnd - inPos; + inPos += copied; + } + } + + public static final ThreadLocal<float[]> scratchBuffer = ThreadLocal.withInitial(() -> new float[GGMLType.Q4_0.blockSize]); + + @Override + public void copyTo(final int thisOffset, final FloatTensor that, final int thatOffset, final int size) { + final float[] decoded = scratchBuffer.get(); + int remaining = size; + int srcIndex = thisOffset; + int dstIndex = thatOffset; + + // Decode and copy in QUANT_BLOCK_SIZE chunks + while (remaining >= GGMLType.Q4_0.blockSize) { + getFloatArray(srcIndex, decoded, 0, GGMLType.Q4_0.blockSize); + for (int i = 0; i < GGMLType.Q4_0.blockSize; i++) { + that.setFloat(dstIndex + i, (float) FLOAT_ARRAY_HANDLE.get(decoded, i)); + } + srcIndex += GGMLType.Q4_0.blockSize; + dstIndex += GGMLType.Q4_0.blockSize; + remaining -= GGMLType.Q4_0.blockSize; + } + + // Decode and copy any leftover elements one by one + if (remaining > 0) { + getFloatArray(srcIndex, decoded, 0, remaining); + for (int i = 0; i < remaining; i++) { + that.setFloat(dstIndex + i, (float) FLOAT_ARRAY_HANDLE.get(decoded, i)); + } + } + } + + /** + * dot product which has the getFloat method inlined in such a way that it processes full blocks at once. + * This gains a > 2.5 times token/s performance increase compared to the generic dot-getFloat implementation. + */ + public final float dot(final int thisOffset, final FloatTensor that, final int thatOffset, final int size) { + float result = 0.0f; + int index = 0; + final int blockLimit = size - (size % GGMLType.Q4_0.blockSize); + + // Process full blocks, one block are 32 elements, 16 quantized values and 1 scale + while (index < blockLimit) { + + // Get this block + final int thisBlockIndex = (thisOffset + index) >>> LOG2_QUANT_BLOCK_SIZE; // (thisOffset + index) / QUANT_BLOCK_SIZE; + final int thisBlockOffset = thisBlockIndex * GGMLType.Q4_0.typeSize; + final float thisScale = FloatTensor.float16ToFloat(buffer.getShort((int) (long) thisBlockOffset)); + + // Process block: read all quantized values from this block at once + final int quantOffset = thisBlockOffset + QUANT_FLOAT16_BYTES; + float blockResult = 0.0f; + final int thatIndex = thatOffset + index; + if (that instanceof ArrayFloatTensor) { + final ArrayFloatTensor thatArray = (ArrayFloatTensor) that; + final float[] b = thatArray.values; + for (int i = 0; i < QUANT_HALF_BLOCK; ++i) { + final byte packed = buffer.get(quantOffset + i); + final float valB0 = (float) FLOAT_ARRAY_HANDLE.get(b, thatIndex + i); + final float valB1 = (float) FLOAT_ARRAY_HANDLE.get(b, thatIndex + i + QUANT_HALF_BLOCK); + blockResult += ((packed & 0x0F) - 8) * valB0 + (((packed >>> 4) & 0x0F) - 8) * valB1; + } + } else { + for (int i = 0; i < QUANT_HALF_BLOCK; ++i) { + final byte packed = buffer.get(quantOffset + i); + blockResult += ((packed & 0x0F) - 8) * that.getFloat(thatIndex + i) + (((packed >>> 4) & 0x0F) - 8) * that.getFloat(thatIndex + i + QUANT_HALF_BLOCK); + } + } + result += blockResult * thisScale; + index += GGMLType.Q4_0.blockSize; + } + + // Process remaining elements + for (; index < size; index++) { + result += this.getFloat(thisOffset + index) * that.getFloat(thatOffset + index); + } + + return (float) result; + } + +}
\ No newline at end of file diff --git a/source/net/yacy/ai/llama3/Tensor/Q8_0FloatTensor.java b/source/net/yacy/ai/llama3/Tensor/Q8_0FloatTensor.java new file mode 100644 index 000000000..5e84b86fb --- /dev/null +++ b/source/net/yacy/ai/llama3/Tensor/Q8_0FloatTensor.java @@ -0,0 +1,134 @@ +/** + * Q8_0FloatTensor.java + + * This file was extracted from the llama3/qwen2 projects + * https://github.com/mukel/llama3.java + * https://github.com/mukel/qwen2.svm.java + * + * License: MIT License + * + * Copyright (c) 2024 Andrej Karpathy (for llama2.c) + * Copyright (c) 2024 Alfonso² Peterssen (for llama3/qwen2) + * Copyright (c) 2023 Georgi Gerganov et al. (for llama.cpp) + * Copyright (c) 2025 Michael Peter Christen for modifications: + * The code was modified to fit the YaCy AI project: + * - back-port to Java 11 (removal of Vector API operations and record types) + * - removal of interactive mode and system.out printing + * - separation of the classes in the single java and refactoring + * - run-time performance optimizations for dot product computation of quantized values + * - joining of llama3/qwen2 into one code base; multi-arch options + * - alignment with code from https://github.com/ggml-org/llama.cpp/ + */ + +package net.yacy.ai.llama3.Tensor; + +import java.nio.ByteBuffer; + +import net.yacy.ai.llama3.Model.GGMLType; + +public final class Q8_0FloatTensor extends FloatTensor { + + final int size; + final ByteBuffer buffer; + + + public Q8_0FloatTensor(final int size, final ByteBuffer buffer) { + this.size = size; + this.buffer = buffer; + } + + @Override + public int size() { + return size; + } + + @Override + public void setFloat(final int index, final float value) { + throw new UnsupportedOperationException("setFloat"); + } + + @Override + public GGMLType type() { + return GGMLType.Q8_0; + } + + @Override + public final float getFloat(final int index) { + assert 0 <= index && index < size; + int blockIndex = index / GGMLType.Q8_0.blockSize; + int withinBlockIndex = index % GGMLType.Q8_0.blockSize; + int blockOffset = blockIndex * GGMLType.Q8_0.typeSize; + byte quant = buffer.get((int) (long) (blockOffset + GGMLType.FLOAT16_BYTES + withinBlockIndex)); + final long offset = blockOffset; + float scale = FloatTensor.float16ToFloat(buffer.getShort((int) offset)); + return quant * scale; + } + + @Override + public float dot(final int thisOffset, final FloatTensor that, final int thatOffset, final int size) { + assert 0 <= thisOffset && thisOffset + size <= this.size; + assert 0 <= thatOffset && thatOffset + size <= that.size(); + + float result = 0f; + + // Calculate first and last block indices + int firstBlock = thisOffset / GGMLType.Q8_0.blockSize; + int lastBlock = (thisOffset + size - 1) / GGMLType.Q8_0.blockSize; + + for (int block = firstBlock; block <= lastBlock; block++) { + // Calculate block boundaries and overlaps + int blockStart = block * GGMLType.Q8_0.blockSize; + int blockEnd = blockStart + GGMLType.Q8_0.blockSize; + int start = Math.max(thisOffset, blockStart); + int end = Math.min(thisOffset + size, blockEnd); + int length = end - start; + + // Get common scale factor for this block + int blockOffset = block * GGMLType.Q8_0.typeSize; + final long offset = blockOffset; + float thisScale = FloatTensor.float16ToFloat(buffer.getShort((int) offset)); + + // Compute sum of products for this block + float blockSum = 0f; + int withinBlockStart = start % GGMLType.Q8_0.blockSize; + + int memOffset = blockOffset + GGMLType.FLOAT16_BYTES + withinBlockStart; + int thatoffset = thatOffset + (start - thisOffset); + if (that instanceof ArrayFloatTensor) { + ArrayFloatTensor thatArray = (ArrayFloatTensor) that; + for (int i = 0; i < length; i++) { + blockSum += buffer.get((int) (long) memOffset++) * thatArray.getFloat(thatoffset++); + } + } else { + for (int i = 0; i < length; i++) { + blockSum += buffer.get((int) (long) memOffset++) * that.getFloat(thatoffset++); + } + } + + // Apply scale to the entire block sum + result += thisScale * blockSum; + } + + return result; + } + + @Override + public void copyTo(final int thisOffset, final FloatTensor that, final int thatOffset, final int size) { + assert 0 <= thisOffset && thisOffset + size <= this.size; + assert 0 <= thatOffset && thatOffset + size <= that.size(); + + final int endOffset = thatOffset + size; + + for (int i = thatOffset; i < endOffset; ++i) { + int index = i - thatOffset + thisOffset; + int blockIndex = index / GGMLType.Q8_0.blockSize; + int withinBlockIndex = index % GGMLType.Q8_0.blockSize; + int blockOffset = blockIndex * GGMLType.Q8_0.typeSize; + + byte quant = buffer.get((int) (long) (blockOffset + GGMLType.FLOAT16_BYTES + withinBlockIndex)); + final long offset = blockOffset; + float scale = FloatTensor.float16ToFloat(buffer.getShort((int) offset)); + that.setFloat(i, quant * scale); + } + } +} diff --git a/source/net/yacy/cora/document/id/MultiProtocolURL.java b/source/net/yacy/cora/document/id/MultiProtocolURL.java index 653d65eba..0ff750b09 100644 --- a/source/net/yacy/cora/document/id/MultiProtocolURL.java +++ b/source/net/yacy/cora/document/id/MultiProtocolURL.java @@ -2660,7 +2660,6 @@ public class MultiProtocolURL implements Serializable, Comparable<MultiProtocolU return splitpattern.split(normalizedURL.toLowerCase()); // word components of the url } - @SuppressWarnings("deprecation") public static void main(final String[] args) { final String[][] test = new String[][]{ new String[]{null, "file://y:/"}, diff --git a/source/net/yacy/cora/util/Html2Image.java b/source/net/yacy/cora/util/Html2Image.java index 10b9afb52..83efe559b 100644 --- a/source/net/yacy/cora/util/Html2Image.java +++ b/source/net/yacy/cora/util/Html2Image.java @@ -141,7 +141,6 @@ public class Html2Image { private static boolean wkhtmltopdfAvailableInPath() { boolean available = false; try { - @SuppressWarnings("deprecation") final Process p = Runtime.getRuntime().exec(WKHTMLTOPDF_COMMAND + " -V"); available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0; } catch (final IOException e) { @@ -184,7 +183,6 @@ public class Html2Image { boolean available = false; if(!OS.isWindows) { // on MS Windows convert is a system tool to convert volumes from FAT to NTFS try { - @SuppressWarnings("deprecation") final Process p = Runtime.getRuntime().exec(CONVERT_COMMAND + " -version"); available = p.waitFor(2, TimeUnit.SECONDS) && p.exitValue() == 0; } catch (final IOException e) { @@ -296,7 +294,6 @@ public class Html2Image { * @throws IOException when an unexpected error occurred */ private static boolean execWkhtmlToPdf(final String proxy, final File destination, final String commandline, final long maxSeconds) throws IOException { - @SuppressWarnings("deprecation") final Process p = Runtime.getRuntime().exec(commandline); try { diff --git a/source/net/yacy/gui/framework/Browser.java b/source/net/yacy/gui/framework/Browser.java index 80d566127..fc396c741 100644 --- a/source/net/yacy/gui/framework/Browser.java +++ b/source/net/yacy/gui/framework/Browser.java @@ -136,7 +136,6 @@ public class Browser { * It is part of the LSB (Linux Standard Base) and therefore included in all recent Linux Distributions supporting it * (see https://www.linuxbase.org/navigator/browse/cmd_single.php?cmd=list-by-name&Section=ABI&Cname=xdg-open) */ final String cmd = "xdg-open " + url; - @SuppressWarnings("deprecation") final Process p = Runtime.getRuntime().exec(cmd); p.waitFor(); if (p.exitValue() != 0) { @@ -153,7 +152,6 @@ public class Browser { cmd = "rundll32 url.dll,FileProtocolHandler \"" + url + "\""; } //cmd = "cmd.exe /c start javascript:document.location='" + url + "'"; - @SuppressWarnings("deprecation") final Process p = Runtime.getRuntime().exec(cmd); p.waitFor(); if (p.exitValue() != 0) { diff --git a/source/net/yacy/htroot/IndexImportZim_p.java b/source/net/yacy/htroot/IndexImportZim_p.java new file mode 100644 index 000000000..8517121d5 --- /dev/null +++ b/source/net/yacy/htroot/IndexImportZim_p.java @@ -0,0 +1,85 @@ +// IndexImportZim_p.java +// ------------------------- +// (c) 2025 by Michael Peter Christen, mc@yacy.net +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package net.yacy.htroot; + +import java.io.File; +import java.io.IOException; + +import net.yacy.cora.protocol.RequestHeader; +import net.yacy.document.importer.ZimImporter; +import net.yacy.server.serverObjects; +import net.yacy.server.serverSwitch; + +public class IndexImportZim_p { + + public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader request, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) { + + // read multipart data from post request + + final serverObjects prop = new serverObjects(); + + if (ZimImporter.job != null && ZimImporter.job.isAlive()) { + // one import is running, no option to insert anything + prop.put("import", 1); + prop.put("import_thread", "running"); + prop.put("import_warcfile", ZimImporter.job.source()); + prop.put("import_count", ZimImporter.job.count()); + prop.put("import_speed", ZimImporter.job.speed()); + prop.put("import_runningHours", (ZimImporter.job.runningTime() / 60) / 60); + prop.put("import_runningMinutes", (ZimImporter.job.runningTime() / 60) % 60); + prop.put("import_remainingHours", (ZimImporter.job.remainingTime() / 60) / 60); + prop.put("import_remainingMinutes", (ZimImporter.job.remainingTime() / 60) % 60); + if (post != null && post.containsKey("abort")) { + ZimImporter.job.quit(); + } + } else { + prop.put("import", 0); + if (post != null) { + if (post.containsKey("file")) { + final String filename = post.get("file"); + if (filename != null && filename.length() > 0) { + final File sourcefile = new File(filename); + if (sourcefile.exists()) { + try { + final ZimImporter zi = new ZimImporter(sourcefile.getAbsolutePath()); + zi.start(); + prop.put("import_thread", "started"); + } catch (final IOException ex) { + prop.put("import_thread", "Error: file not found [" + filename + "]"); + } + prop.put("import", 1); + prop.put("import_zimfile", filename); + } else { + prop.put("import_zimfile", ""); + prop.put("import_thread", "Error: file not found [" + filename + "]"); + } + } + + prop.put("import_count", 0); + prop.put("import_speed", 0); + prop.put("import_runningHours", 0); + prop.put("import_runningMinutes", 0); + prop.put("import_remainingHours", 0); + prop.put("import_remainingMinutes", 0); + } + } + } + return prop; + } +} diff --git a/source/net/yacy/http/servlets/RAGProxyServlet.java b/source/net/yacy/http/servlets/RAGProxyServlet.java index e7de7d61d..166a69aea 100644 --- a/source/net/yacy/http/servlets/RAGProxyServlet.java +++ b/source/net/yacy/http/servlets/RAGProxyServlet.java @@ -211,10 +211,12 @@ public class RAGProxyServlet extends HttpServlet { String question = "Make a list of a maximum of four search words for the following question; use a JSON Array: " + prompt; try { OpenAIClient oaic = new OpenAIClient(LLM_API_HOST); - String[] a = OpenAIClient.stringsFromChat(oaic.chat(model, question, 80)); + OpenAIClient.Context context = new OpenAIClient.Context(LLM_SYSTEM_PREFIX); + context.addPrompt(question); + String[] a = OpenAIClient.stringsFromChat(oaic.chat(model, context, OpenAIClient.listSchema, 80)); for (String s: a) query.append(s).append(' '); return query.toString().trim(); - } catch (IOException e) { + } catch (IOException | JSONException e) { e.printStackTrace(); return ""; } diff --git a/source/net/yacy/kelondro/logging/ThreadDump.java b/source/net/yacy/kelondro/logging/ThreadDump.java index 00a4eb8dc..a4d842d02 100644 --- a/source/net/yacy/kelondro/logging/ThreadDump.java +++ b/source/net/yacy/kelondro/logging/ThreadDump.java @@ -236,7 +236,6 @@ public class ThreadDump extends HashMap<ThreadDump.StackTrace, List<String>> imp while (tracename.length() < 20) tracename = tracename + "_"; tracename = "[" + tracename + "] "; } - @SuppressWarnings("deprecation") final String threadtitle = tracename + "Thread= " + thread.getName() + " " + (thread.isDaemon()?"daemon":"") + " id=" + thread.getId() + " " + thread.getState().toString(); String className; boolean cutcore = true; diff --git a/source/net/yacy/kelondro/util/FileUtils.java b/source/net/yacy/kelondro/util/FileUtils.java index 408ee459d..372793e27 100644 --- a/source/net/yacy/kelondro/util/FileUtils.java +++ b/source/net/yacy/kelondro/util/FileUtils.java @@ -982,7 +982,6 @@ public final class FileUtils { // deleting files on windows sometimes does not work with java
try {
final String command = "cmd /C del /F /Q \"" + p + "\"";
- @SuppressWarnings("deprecation")
final Process r = Runtime.getRuntime().exec(command);
if ( r == null ) {
ConcurrentLog.severe("FileUtils", "cannot execute command: " + command);
diff --git a/source/net/yacy/server/http/HTTPDProxyHandler.java b/source/net/yacy/server/http/HTTPDProxyHandler.java index e300ec9e2..3f01a7ead 100644 --- a/source/net/yacy/server/http/HTTPDProxyHandler.java +++ b/source/net/yacy/server/http/HTTPDProxyHandler.java @@ -90,7 +90,6 @@ import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.search.Switchboard;
import net.yacy.server.serverObjects;
-@SuppressWarnings("deprecation")
public final class HTTPDProxyHandler {
|