added YaCy Pack Downloader

This does a direct scan of the dataset repositories at huggingface. At this time there is one repo which has the String "YaCy-Pack" inside the repo name. All repositories that have that String in its name are listed. A download button exist which downloads the YaCy Pack to the hold path.
author: Michael Peter Christen <mc@yacy.net> 2025-08-19 22:53:53 -0700
committer: Michael Peter Christen <mc@yacy.net> 2025-08-19 22:53:53 -0700
commit: c2060cc15d95a2870af910b489ac3a1d042d2257 (patch)
tree: cbcc63cd24f0c97142750dbd01b0468838928c2b
parent: 7a6dd868f53d330f29d742245d0c67e7cbc23f67 (diff)
5 files changed, 337 insertions, 3 deletions
diff --git a/htroot/IndexPackDownloader_p.html b/htroot/IndexPackDownloader_p.html
new file mode 100644
index 000000000..cef218ec1
--- /dev/null
+++ b/htroot/IndexPackDownloader_p.html
@@ -0,0 +1,31 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "DTD/xhtml1-transitional.dtd">
+<!-- This page is only XHTML 1.0 Transitional because target is being used in a links -->
+<html xmlns="http://www.w3.org/1999/xhtml">
+#(reload)#::<meta http-equiv="REFRESH" content="5; url=/IndexPackDownloader_p.html">#(/reload)#
+  <head>
+    <title>YaCy '#[clientname]#': Index Pack Downloader</title>
+    #%env/templates/metas.template%#
+  </head>
+  <body id="IndexControl">
+    #%env/templates/header.template%#
+    #%env/templates/submenuIndexImport.template%#
+
+    <h2>YaCy Pack Downloader</h2>
+
+
+    <fieldset><legend>Available Packs</legend>
+        <table class="table table-striped" summary="Packs: Download Sources">
+            <thead class="thead-dark">
+            <tr class="TableHeader"><th width="120">Source</th><th width="360">Repo ID</th><th width="720">File</th><th width="360">Process</th></tr>
+            </thead>
+            <tbody>
+        #{packs-source}#
+            <tr class="TableCell#(dark)#Light::Dark#(/dark)#"><td>#[source]#</td><td><a href="#[repourl]#" target="_blank">#[repoid]#</a></td><td>#[file]#</td><td>#[process]#</td></tr>
+        #{/packs-source}#
+            </tbody>
+        </table>
+    </fieldset>
+
+    #%env/templates/footer.template%#
+  </body>
+</html>
diff --git a/htroot/env/templates/submenuIndexImport.template b/htroot/env/templates/submenuIndexImport.template
index 5ff6a80f1..81bd31a63 100644
--- a/htroot/env/templates/submenuIndexImport.template
+++ b/htroot/env/templates/submenuIndexImport.template
@@ -3,10 +3,11 @@
 </div>
 <div class="SubMenu">
   <div class="SubMenugroup">
-    <h3>Index Packs</h3>
+    <h3>YaCy Packs</h3>
     <ul class="SubMenu">
-      <li><a href="IndexPackGenerator_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Index Pack Generator</a></li>
-      <li><a href="IndexPackManager_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Index Pack Manager</a></li>
+      <li><a href="IndexPackGenerator_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Pack Generator</a></li>
+      <li><a href="IndexPackDownloader_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Pack Downloader</a></li>
+      <li><a href="IndexPackManager_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Pack Manager</a></li>
     </ul>
   </div>
   <div class="SubMenugroup">
diff --git a/source/net/yacy/htroot/IndexPackDownloader_p.java b/source/net/yacy/htroot/IndexPackDownloader_p.java
new file mode 100644
index 000000000..9c22c4dbc
--- /dev/null
+++ b/source/net/yacy/htroot/IndexPackDownloader_p.java
@@ -0,0 +1,112 @@
+// IndexPackDownloader_p.java
+// -----------------------
+// (C) 2025 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
+//
+// This is a part of YaCy, a peer-to-peer based web search engine
+//
+// LICENSE
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+package net.yacy.htroot;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import net.yacy.cora.protocol.RequestHeader;
+import net.yacy.kelondro.util.FileUtils;
+import net.yacy.kelondro.util.HFClient;
+import net.yacy.search.Switchboard;
+import net.yacy.server.serverObjects;
+import net.yacy.server.serverSwitch;
+
+public class IndexPackDownloader_p {
+
+    public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
+        // return variable that accumulates replacements
+        final Switchboard sb = (Switchboard) env;
+        final serverObjects prop = new serverObjects();
+
+        if (post != null) {
+            final String dlsource = post.get("dlsource", "");
+            final String dlrepoid = post.get("dlrepoid", "");
+            final String dlfile = post.get("dlfile", "");
+
+            if (!dlsource.isEmpty() && !dlrepoid.isEmpty() && !dlfile.isEmpty()) {
+                if (dlsource.equals("Huggingface")) {
+                    try {
+                        final byte[] b = HFClient.downloadFile(dlrepoid, dlfile);
+                        if (b == null || b.length == 0) throw new IOException("downloaded file is empty");
+                        // save the file to the hold directory
+                        final File holdFile = new File(sb.packsHoldPath, dlfile);
+                        if (!holdFile.getParentFile().exists()) holdFile.getParentFile().mkdirs();
+                        FileUtils.copy(b, holdFile);
+                        prop.put("reload", 1);
+                    } catch (final Exception e) {
+                        sb.log.warn("error downloading from HuggingFace: " + e.getMessage(), e);
+                    }
+                }
+            }
+        }
+
+        // set default values
+        prop.put("reload", 0);
+
+
+        // show Pack folder contents
+        int i = 0; boolean dark = true;
+        Map<String, List<String>> hfmap;
+        try {
+            hfmap = HFClient.getAllPacks(true);
+        } catch (IOException|InterruptedException e) {
+            hfmap = new HashMap<>();
+        }
+        final Map<String, String> packsMap = sb.packsMap();
+        for (final Map.Entry<String, List<String>> entry : hfmap.entrySet()) {
+            final String repoid = entry.getKey();
+            final List<String> files = entry.getValue();
+            for (final String file: files) {
+                // skip empty files
+                if (file == null || file.isEmpty()) continue;
+                prop.put("packs-source_" + i + "_source", "Huggingface");
+                prop.put("packs-source_" + i + "_repoid", repoid);
+                prop.put("packs-source_" + i + "_repourl", "https://huggingface.co/datasets/" + repoid);
+                prop.put("packs-source_" + i + "_file", file);
+                final String localPath = packsMap.get(file);
+                if (localPath == null) {
+                    prop.put("packs-source_" + i + "_process",
+                        "<a class=\"btn btn-primary\" href=\"IndexPackDownloader_p.html?dlsource=Huggingface&dlrepoid=" + repoid + "&dlfile=" + file + "\" role=\"button\">download</a>"
+                    );
+                } else {
+                    prop.put("packs-source_" + i + "_process",
+                        "file is downloaded; location: " + localPath
+                    );
+                }
+                prop.put("packs_source" + i + "_dark", dark ? "1" : "0");
+                i++;
+                dark = !dark;
+            }
+        }
+        prop.put("packs-source", i);
+
+
+        // return rewrite properties
+        return prop;
+    }
+
+}
+\ No newline at end of file
diff --git a/source/net/yacy/kelondro/util/HFClient.java b/source/net/yacy/kelondro/util/HFClient.java
new file mode 100644
index 000000000..b8cb8f176
--- /dev/null
+++ b/source/net/yacy/kelondro/util/HFClient.java
@@ -0,0 +1,180 @@
+/**
+ *  HFClient
+ *  Copyright 2025 by Michael Peter Christen
+ *  First released 19.08.2025 at https://yacy.net
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public License
+ *  along with this program in the file lgpl21.txt
+ *  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.kelondro.util;
+
+import java.io.IOException;
+import java.net.URLEncoder;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import net.yacy.cora.protocol.ClientIdentification;
+import net.yacy.cora.protocol.http.HTTPClient;
+
+
+public class HFClient {
+
+    private static final String BASE = "https://huggingface.co";
+
+    public enum RepoType {
+        MODEL("models"),
+        DATASET("datasets"),
+        SPACE("spaces");
+
+        private final String pluralSegment;
+        RepoType(String s) { this.pluralSegment = s; }
+        public String segment() { return this.pluralSegment; }
+    }
+
+
+    private static String textOrNull(JsonNode n) {
+        return (n != null && !n.isNull()) ? n.asText() : null;
+    }
+
+
+    public static List<String> listYaCyPacks() throws IOException, InterruptedException {
+        List<String> repos = listRepositories(null, RepoType.DATASET, 20, "YaCy-Pack");
+        return repos;
+    }
+
+    private static List<String> listRepositories(String author, RepoType type, int limit, String search) throws IOException, InterruptedException {
+        if (limit <= 0) limit = 100;
+        StringBuilder url = new StringBuilder(BASE).append("/api/").append(type.segment()).append("?limit=").append(limit);
+        if (author != null && !author.isBlank()) url.append("&author=").append(URLEncoder.encode(author, java.nio.charset.StandardCharsets.UTF_8));
+        if (search != null && !search.isBlank()) url.append("&search=").append(URLEncoder.encode(search, java.nio.charset.StandardCharsets.UTF_8));
+
+        HTTPClient hc = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
+        byte[] body = hc.GETbytes(url.toString(), null, null, false);
+        hc.close();
+
+        ObjectMapper mapper = new ObjectMapper();
+        JsonNode arr = mapper.readTree(body);
+        List<String> ids = new ArrayList<>();
+        if (arr.isArray()) {
+            for (JsonNode n : arr) {
+                // The field is "modelId" for models, "id" for datasets/spaces; normalize both.
+                String id = textOrNull(n.get("modelId"));
+                if (id == null) id = textOrNull(n.get("id"));
+                if (id != null) ids.add(id);
+            }
+        }
+        return ids;
+    }
+
+    public static List<String> listFiles(String repoId, boolean only_jsonl) throws IOException, InterruptedException {
+        List<String>  files = listFiles(repoId, RepoType.DATASET);
+        if (!only_jsonl) return files;
+        for (int i = files.size() - 1; i >= 0; i--) {
+            String file = files.get(i);
+            if (!file.contains(".jsonl")) {
+                files.remove(i); // remove non-jsonl files
+            }
+        }
+        return files;
+    }
+
+    public static List<String> listFiles(String repoId, RepoType type) throws IOException, InterruptedException {
+        String url = BASE + "/api/" + type.segment() + "/" + repoId;
+        HTTPClient hc = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
+        byte[] body = hc.GETbytes(url, null, null, false);
+        hc.close();
+
+        ObjectMapper mapper = new ObjectMapper();
+        JsonNode root = mapper.readTree(body);
+        JsonNode filesArray = root.get("siblings");
+        if (filesArray == null) filesArray = root.get("files");
+
+        List<String> out = new ArrayList<>();
+        if (filesArray != null && filesArray.isArray()) {
+            for (JsonNode f : filesArray) {
+                // models/datasets commonly expose "rfilename"
+                String path = textOrNull(f.get("rfilename"));
+                if (path == null) {
+                    // fallback to "path" if present
+                    path = textOrNull(f.get("path"));
+                }
+                if (path != null) out.add(path);
+            }
+        }
+
+        return out;
+    }
+
+    public static Map<String, List<String>> getAllPacks(boolean only_jsonl) throws IOException, InterruptedException {
+        Map<String, List<String>> map = new LinkedHashMap<>();
+        List<String> repos = listYaCyPacks();
+        for (String repoId : repos) {
+            List<String> files = listFiles(repoId, only_jsonl);
+            if (!files.isEmpty()) {
+                map.put(repoId, files);
+            }
+        }
+        return map;
+    }
+
+    public static byte[] downloadFile(String repoId, String filePath) throws IOException, InterruptedException {
+        return downloadFile(repoId, RepoType.DATASET, "main", filePath);
+    }
+
+    private static byte[] downloadFile(String repoId, RepoType type, String revision, String filePath) throws IOException, InterruptedException {
+
+        Objects.requireNonNull(repoId);
+        Objects.requireNonNull(filePath);
+        if (revision == null || revision.isBlank()) revision = "main";
+
+        // Raw file URL pattern:
+        // https://huggingface.co/{models|datasets|spaces}/{repoId}/resolve/{revision}/{filePath}
+        String url = BASE + "/" + RepoType.DATASET.segment() + "/" + repoId + "/resolve/" + revision + "/" + filePath + "?download=true";
+        HTTPClient hc = new HTTPClient(ClientIdentification.browserAgent);
+        hc.setRedirecting(true);
+        byte[] body = hc.GETbytes(url, null, null, false);
+        hc.close();
+        return body;
+    }
+
+    // test
+    public static void main(String[] args) throws Exception {
+        List<String> repos = listYaCyPacks();
+        System.out.println("Repos: " + repos);
+        String test_repo_id = repos.isEmpty() ? null : repos.get(0);
+        List<String> all_files = test_repo_id == null ? new ArrayList(0) : listFiles(test_repo_id, false);
+        List<String> jsonl_files = test_repo_id == null ? new ArrayList(0) : listFiles(test_repo_id, true);
+        System.out.println("all   files in " + test_repo_id + ": " + all_files);
+        System.out.println("jsonl files in " + test_repo_id + ": " + jsonl_files);
+        if (!jsonl_files.isEmpty()) {
+            byte[] data = downloadFile(test_repo_id, RepoType.DATASET, "main", jsonl_files.get(0));
+            System.out.println("Downloaded " + data.length + " bytes from " + jsonl_files.get(0) + " in " + test_repo_id);
+        }
+
+        Map<String, List<String>> map = getAllPacks(true);
+        for (Map.Entry<String, List<String>> entry : map.entrySet()) {
+            String repo = entry.getKey();
+            List<String> files = entry.getValue();
+            System.out.println("Repo: " + repo + ", Files: " + files);
+        }
+
+    }
+}
diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java
index 9ce73fbe4..5a633e310 100644
--- a/source/net/yacy/search/Switchboard.java
+++ b/source/net/yacy/search/Switchboard.java
@@ -2381,6 +2381,16 @@ public final class Switchboard extends serverSwitch {
         return list;
     }
 
+    public Map<String, String> packsMap() {
+        Map<String, String> map = new LinkedHashMap<>();
+        for (String pack: this.packsInHold()) map.put(pack, "hold");
+        for (String pack: this.packsInLive()) map.put(pack, "live");
+        for (String pack: this.packsInLoad()) map.put(pack, "load");
+        for (String pack: this.packsInLoaded()) map.put(pack, "loaded");
+        for (String pack: this.packsInUnload()) map.put(pack, "unload");
+        return map;
+    }
+
     public boolean packProcess() {
         // work off fresh entries from the proxy or from the crawler
         final String cautionCause = this.onlineCaution();
author	Michael Peter Christen <mc@yacy.net>	2025-08-19 22:53:53 -0700
committer	Michael Peter Christen <mc@yacy.net>	2025-08-19 22:53:53 -0700
commit	c2060cc15d95a2870af910b489ac3a1d042d2257 (patch)
tree	cbcc63cd24f0c97142750dbd01b0468838928c2b
parent	7a6dd868f53d330f29d742245d0c67e7cbc23f67 (diff)