From 6a74e80ecd82da7138851233f85bce310d2578c8 Mon Sep 17 00:00:00 2001
From: wfjsw <wfjsw@users.noreply.github.com>
Date: Sat, 10 Aug 2024 01:32:04 -0500
Subject: [PATCH] add cache to sha256

---
 .github/workflows/deploy.yml |  7 +++++++
 .gitignore                   |  1 +
 .script/cache.ts             | 40 ++++++++++++++++++++++++++++++++++++
 .script/hub.ts               | 31 +++++++++++++++-------------
 .script/index.ts             |  5 +++++
 5 files changed, 70 insertions(+), 14 deletions(-)
 create mode 100644 .script/cache.ts

diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index 7370171..ed1b888 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -23,6 +23,13 @@ jobs:
       - name: Install dependencies
         run: |
           npm install
+
+      - name: Cache SHA256
+        id: cache-sha256
+        uses: actions/cache@v4
+        with:
+            path: .script/cache.json
+            key: sha256-node-${{ hashFiles('**/package-lock.json') }}
         
       - name: Build
         run: |
diff --git a/.gitignore b/.gitignore
index b947077..7900daf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 node_modules/
 dist/
+.script/cache.json
diff --git a/.script/cache.ts b/.script/cache.ts
new file mode 100644
index 0000000..6ba364a
--- /dev/null
+++ b/.script/cache.ts
@@ -0,0 +1,40 @@
+import { readFile, writeFile } from "fs/promises";
+
+const cache: Record<string, string> = {}
+
+export function getCache(key: string): string | undefined {
+    return cache[key];
+}
+
+export function setCache(key: string, value: string): void {
+    cache[key] = value;
+}
+
+export function clearCache(): void {
+    for (const key in cache) {
+        delete cache[key];
+    }
+}
+
+export function deleteCache(key: string): void {
+    delete cache[key];
+}
+
+export async function loadCache() {
+    try {
+        const data = await readFile(".script/cache.json", "utf-8");
+        const parsed = JSON.parse(data);
+        for (const key in parsed) {
+            cache[key] = parsed[key];
+        }
+    } catch (e) {
+    }
+}
+
+export async function saveCache() {
+    try {
+        await writeFile(".script/cache.json", JSON.stringify(cache), "utf-8");
+    } catch (e) {
+        console.error(e);
+    }
+}
diff --git a/.script/hub.ts b/.script/hub.ts
index 143e473..2aa00ed 100644
--- a/.script/hub.ts
+++ b/.script/hub.ts
@@ -1,6 +1,7 @@
 import { downloadFile, listFiles } from "@huggingface/hub";
 import { HF_TOKEN } from "./env";
 import { getSha256FromResponse } from "./sha256";
+import { getCache, setCache } from "./cache";
 
 interface HubFileMetadata {
     filename: string;
@@ -20,22 +21,24 @@ export async function getFiles(repoId: string, revision: string) {
         let sha256 = file.lfs?.oid;
 
         if (!sha256) {
-            const response = await downloadFile({
-                repo: repoId,
-                revision,
-                path: file.path,
-                credentials: { accessToken: HF_TOKEN },
-            });
+            const cached = getCache(file.oid);
+            if (cached) {
+                sha256 = cached;
+            } else {
+                const response = await downloadFile({
+                    repo: repoId,
+                    revision,
+                    path: file.path,
+                    credentials: { accessToken: HF_TOKEN },
+                });
 
-            if (!response) {
-                throw new Error(
-                    `Failed to download file ${file.path}`
-                );
-            }
+                if (!response) {
+                    throw new Error(`Failed to download file ${file.path}`);
+                }
 
-            sha256 = await getSha256FromResponse(
-                response
-            );
+                sha256 = await getSha256FromResponse(response);
+                setCache(file.oid, sha256);
+            }
         }
 
         files.push({
diff --git a/.script/index.ts b/.script/index.ts
index 909d35d..cb980e8 100644
--- a/.script/index.ts
+++ b/.script/index.ts
@@ -5,12 +5,15 @@ import { readFile, mkdir, writeFile, rm } from "fs/promises";
 import { parse } from "yaml";
 import type { IProvider, Resolution, ResolutionFile, Rule } from "./interfaces";
 import { getFiles } from "./hub";
+import { loadCache, saveCache } from "./cache";
 
 const root = resolve(fileURLToPath(import.meta.url), "../..")
 
 async function main() {
     console.log(root);
 
+    await loadCache();
+
     const providers: Record<string, IProvider> = {};
 
     const providerFiles = globStream("./providers/*.ts", {
@@ -67,6 +70,8 @@ async function main() {
         }
 
     }
+
+    await saveCache();
 }
 
 main();