finalize

.github/workflows/deploy.yml

@@ -0,0 +1,38 @@
+name: Deploy
+
+on:
+  push:
+    branches:
+      - master
+  workflow_dispatch:
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22.5.1'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: |
+          npm install
+        
+      - name: Build
+        run: |
+          npm run build
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+
+      - name: Publish with Wrangler
+        uses: cloudflare/wrangler-action@v3
+        with:
+          apiToken: ${{ secrets.CF_API_TOKEN }}
+          command: pages deploy dist --project-name=oystermercury-hfredir
+          accountId: ${{ secrets.CF_ACCOUNT_ID }}

.gitignore

@@ -1 +1,2 @@
 node_modules/
+dist/

.script/hub.ts

@@ -1,14 +1,48 @@
-import { listFiles } from "@huggingface/hub";
+import { downloadFile, listFiles } from "@huggingface/hub";
 import { HF_TOKEN } from "./env";
+import { getSha256FromResponse } from "./sha256";
 
-export async function getMetadata(repoId: string, revision: string) {
-    const files = [];
-    const remoteFiles = listFiles({ repo: repoId, revision, recursive: true, credentials: { accessToken: HF_TOKEN } });
+interface HubFileMetadata {
+    filename: string;
+    size: number;
+    sha256: string;
+}
+
+export async function getFiles(repoId: string, revision: string) {
+    const files: HubFileMetadata[] = [];
+    const remoteFiles = listFiles({
+        repo: repoId,
+        revision,
+        recursive: true,
+        credentials: { accessToken: HF_TOKEN },
+    });
     for await (const file of remoteFiles) {
+        let sha256 = file.lfs?.oid;
+
+        if (!sha256) {
+            const response = await downloadFile({
+                repo: repoId,
+                revision,
+                path: file.path,
+                credentials: { accessToken: HF_TOKEN },
+            });
+
+            if (!response) {
+                throw new Error(
+                    `Failed to download file ${file.path}`
+                );
+            }
+
+            sha256 = await getSha256FromResponse(
+                response
+            );
+        }
+
         files.push({
             filename: file.path,
             size: file.size,
-            sha256: file.lfs?.oid,
+            sha256,
         });
     }
+    return files;
 }

.script/index.ts

@@ -0,0 +1,72 @@
+import { globStream } from "glob";
+import { fileURLToPath, pathToFileURL } from "url";
+import { resolve } from "path";
+import { readFile, mkdir, writeFile, rm } from "fs/promises";
+import { parse } from "yaml";
+import type { IProvider, Resolution, ResolutionFile, Rule } from "./interfaces";
+import { getFiles } from "./hub";
+
+const root = resolve(fileURLToPath(import.meta.url), "../..")
+
+async function main() {
+    console.log(root);
+
+    const providers: Record<string, IProvider> = {};
+
+    const providerFiles = globStream("./providers/*.ts", {
+        cwd: root,
+        nodir: true,
+        posix: true,
+    });
+
+    for await (const providerFile of providerFiles) {
+        const fileUrl = pathToFileURL(resolve(root, providerFile));
+        const provider = await import(fileUrl.toString());
+        const instance = new provider.default() as IProvider;
+        providers[instance.identifier] = instance;
+    }
+
+    await rm(resolve(root, "dist"), { recursive: true, force: true });
+    await mkdir(resolve(root, "dist"));
+
+    const rules = globStream("./rules/**/*.yaml", {
+        cwd: root,
+        nodir: true,
+        posix: true,
+    });
+
+    for await (const rule of rules) {
+        const repoId = rule.slice(6, -5);
+        // console.log(repoId);
+        const content = await readFile(resolve(root, rule), "utf-8");
+        const parsed = parse(content) as Rule;
+        await mkdir(resolve(root, "dist", repoId), {
+            recursive: true,
+        });
+
+        for (const revision of parsed.revisions) {
+            const files: ResolutionFile[] = []
+            const filename = resolve(root, `dist/${repoId}/${revision}.json`);
+            const hubFiles = await getFiles(repoId, revision);
+            for (const file of hubFiles) {
+                const resources: string[] = [];
+                for (const destination of parsed.destination) {
+                    const provider = providers[destination.provider];
+                    if (!provider) {
+                        throw new Error(`Provider ${destination.provider} not found`);
+                    }
+                    resources.push(provider.toProviderUrl(file.filename, revision, destination.args));
+                }
+                files.push({
+                    ...file,
+                    resources,
+                });
+            }
+
+            await writeFile(filename, JSON.stringify(files));
+        }
+
+    }
+}
+
+main();

.script/interfaces.ts

@@ -7,3 +7,24 @@ export interface GlobalArgs {
     whitelist?: string[];
     blacklist?: string[];
 }
+
+export interface RuleDestination {
+    provider: string;
+    args: unknown;
+}
+
+export interface Rule {
+    revisions: string[]
+    destination: RuleDestination[];
+}
+
+export interface ResolutionFile {
+    filename: string;
+    size: number;
+    sha256: string;
+    resources: string[];
+}
+
+export interface Resolution {
+    [key: string]: ResolutionFile[];
+}

.script/sha256.ts

@@ -0,0 +1,22 @@
+import { createHash } from "crypto";
+
+export async function getSha256FromResponse(res: Response) {
+    const reader = res.body?.getReader();
+
+    if (!reader) {
+        throw new Error("Failed to get reader from response body");
+    }
+
+    const hash = createHash("sha256");
+
+    while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        if (!value) {
+            throw new Error("Failed to get value from reader");
+        }
+        hash.update(value);
+    }
+
+    return hash.digest("hex");
+}

README.md

@@ -0,0 +1,2 @@
+# Huggingface Redirections
+