add cache to sha256

.github/workflows/deploy.yml

@@ -23,6 +23,13 @@ jobs:
       - name: Install dependencies
         run: |
           npm install
+
+      - name: Cache SHA256
+        id: cache-sha256
+        uses: actions/cache@v4
+        with:
+            path: .script/cache.json
+            key: sha256-node-${{ hashFiles('**/package-lock.json') }}
 
       - name: Build
         run: |

.gitignore

@@ -1,2 +1,3 @@
 node_modules/
 dist/
+.script/cache.json

.script/cache.ts

@@ -0,0 +1,40 @@
+import { readFile, writeFile } from "fs/promises";
+
+const cache: Record<string, string> = {}
+
+export function getCache(key: string): string | undefined {
+    return cache[key];
+}
+
+export function setCache(key: string, value: string): void {
+    cache[key] = value;
+}
+
+export function clearCache(): void {
+    for (const key in cache) {
+        delete cache[key];
+    }
+}
+
+export function deleteCache(key: string): void {
+    delete cache[key];
+}
+
+export async function loadCache() {
+    try {
+        const data = await readFile(".script/cache.json", "utf-8");
+        const parsed = JSON.parse(data);
+        for (const key in parsed) {
+            cache[key] = parsed[key];
+        }
+    } catch (e) {
+    }
+}
+
+export async function saveCache() {
+    try {
+        await writeFile(".script/cache.json", JSON.stringify(cache), "utf-8");
+    } catch (e) {
+        console.error(e);
+    }
+}

.script/hub.ts

@@ -1,6 +1,7 @@
 import { downloadFile, listFiles } from "@huggingface/hub";
 import { HF_TOKEN } from "./env";
 import { getSha256FromResponse } from "./sha256";
+import { getCache, setCache } from "./cache";
 
 interface HubFileMetadata {
     filename: string;
@@ -20,22 +21,24 @@ export async function getFiles(repoId: string, revision: string) {
         let sha256 = file.lfs?.oid;
 
         if (!sha256) {
-            const response = await downloadFile({
-                repo: repoId,
-                revision,
-                path: file.path,
-                credentials: { accessToken: HF_TOKEN },
-            });
+            const cached = getCache(file.oid);
+            if (cached) {
+                sha256 = cached;
+            } else {
+                const response = await downloadFile({
+                    repo: repoId,
+                    revision,
+                    path: file.path,
+                    credentials: { accessToken: HF_TOKEN },
+                });
 
-            if (!response) {
-                throw new Error(
-                    `Failed to download file ${file.path}`
-                );
-            }
+                if (!response) {
+                    throw new Error(`Failed to download file ${file.path}`);
+                }
 
-            sha256 = await getSha256FromResponse(
-                response
-            );
+                sha256 = await getSha256FromResponse(response);
+                setCache(file.oid, sha256);
+            }
         }
 
         files.push({

.script/index.ts

@@ -5,12 +5,15 @@ import { readFile, mkdir, writeFile, rm } from "fs/promises";
 import { parse } from "yaml";
 import type { IProvider, Resolution, ResolutionFile, Rule } from "./interfaces";
 import { getFiles } from "./hub";
+import { loadCache, saveCache } from "./cache";
 
 const root = resolve(fileURLToPath(import.meta.url), "../..")
 
 async function main() {
     console.log(root);
 
+    await loadCache();
+
     const providers: Record<string, IProvider> = {};
 
     const providerFiles = globStream("./providers/*.ts", {
@@ -67,6 +70,8 @@ async function main() {
         }
 
     }
+
+    await saveCache();
 }
 
 main();