refactor: Update imports and class names in ipfs_model_manager.js

fireproofdb_kit_js/index.js

@@ -1 +1 @@
-export * from './main.js';
+export { fireproofDbKitJs } from './main.js';

fireproofdb_kit_js/main.js

@@ -1,4 +1,4 @@
-export class FireproofKitJs {
+export class fireproofDbKitJs {
     constructor(resources, metadata) {
         this.resources = resources;
         this.metadata = metadata;
@@ -16,4 +16,4 @@ export class FireproofKitJs {
     }
 }
 
-export default FireproofKitJs;
+export default fireproofDbKitJs;

index.js

@@ -1,5 +1,6 @@
 
-export { ipfsModelManager } from './ipfs_model_manager_js/ipfs_model_manager.js';
 export * from './ipfs_model_manager_js/ipfs_model_manager.js';
 export * from './ipfs_model_manager_js/s3_kit.js';
-export * from './ipfs_model_manager_js/test_fio.js';
+export * from './ipfs_model_manager_js/test_fio.js';
+export { ipfsModelManagerJs } from './ipfs_model_manager_js/ipfs_model_manager.js';
+export { ipfsModelManagerJs as default } from './ipfs_model_manager_js/ipfs_model_manager.js';

ipfs_model_manager_js/index.js

@@ -0,0 +1,3 @@
+export * from './ipfs_model_manager.js';
+export { ipfsModelManagerJs } from './ipfs_model_manager.js';
+export { ipfsModelManagerJs as default } from './ipfs_model_manager.js';

ipfs_model_manager_js/ipfs_model_manager.js

@@ -4,9 +4,10 @@ import path from 'path';
 import util from 'util';
 import { promisify } from 'util';
 import { exec, execSync } from 'child_process';
+import { libp2pKitJs } from 'libp2p_kit_js';
 import { ipfsKitJs, installIpfs } from 'ipfs_kit_js';
-import ipfsHuggingfaceScraper from  'ipfs_huggingface_scraper_js';
-import orbitdbKit from 'orbitdb_kit';
+import ipfsHuggingfaceScraperJs from  'ipfs_huggingface_scraper_js';
+import orbitDbKitJs from 'orbitdb_kit_js';
 import * as testFio from './test_fio.js';
 import * as s3Kit from './s3_kit.js';
 import fsExtra from 'fs-extra';
@@ -21,7 +22,7 @@ const stat = util.promisify(fs.stat);
 const moveFile = util.promisify(fs.rename);
 const tmpFile = new temp_file.TempFileManager()
 
-export class ipfsModelManager {
+export class ipfsModelManagerJs {
     constructor(resources = null, meta = null) {
         this.thisDir = path.dirname(import.meta.url);
         if (this.thisDir.startsWith("file://")) {
@@ -123,14 +124,15 @@ export class ipfsModelManager {
                 "cache": this.cache,
             };
         }
-
         let homeDir = os.homedir();
         let homeDirFiles = fs.readdirSync(homeDir);
+        let ipfsPath = this.ipfsPath;
         this.testFio = new testFio.TestFio(resources, meta);
-        this.s3Kit = new s3Kit.s3Kit(resources, meta);
         this.ipfsKitJs = new ipfsKitJs(resources, meta);
         this.installIpfs = new installIpfs(resources, meta);
-        let ipfsPath = this.ipfsPath;
+        this.libp2pKit = new libp2pKit(resources, meta);
+        this.s3Kit = this.libp2pKit.s3Kit(resources, meta);
+        // this.s3Kit = new s3Kit.s3Kit(resources, meta);
         if (!fs.existsSync(this.ipfsPath)) {
             fs.mkdirSync(this.ipfsPath, { recursive: true });
         }
@@ -2032,4 +2034,6 @@ export class ipfsModelManager {
         return null;
     }
 
-}
+}
+
+export default ipfsModelManagerJs;

ipfs_model_manager_js/test_fio.js

@@ -1,7 +1,7 @@
 import { execSync } from 'child_process';
 import fs from 'fs';
 import path from 'path';
-
+import tmp from 'tmp';
 
 export class TestFio {
     constructor(resources, meta = null) {
@@ -12,6 +12,7 @@ export class TestFio {
         this.path = process.env.PATH;
         this.path = this.path + ":" + path.join(this.thisDir, "bin")
         this.pathString = "PATH="+ this.path
+        this.tmp = tmp;
     }
 
     call(method, kwargs = {}) {
@@ -105,7 +106,7 @@ export class TestFio {
     }
 
     diskSpeed4k(location) {
-        const tempFile = tmp.fileSync({ postfix: '.iso', dir: location });
+        const tempFile = this.tmp.fileSync({ postfix: '.iso', dir: location });
         const timestamp_0 = Date.now();
         const command = `dd if=/dev/zero of=${tempFile.name} bs=4k count=8k conv=fdatasync`;
         execSync(command);
@@ -140,10 +141,3 @@ export class TestFio {
 
 }
 
-function test(){
-    const thisTest = new TestFio(null);
-    const results = thisTest.test("/tmp/");
-    console.log(results);
-    console.log("Test complete");
-    //process.exit(0);
-}

package.json

@@ -1,9 +1,9 @@
 {
 	"name": "ipfs_model_manager_js",
-	"version": "1.0.9",
+	"version": "1.0.15",
 	"description": "This is a model manager and wrapper for huggingface, and it maintains an index of models from collections of models store retrieved through local/https/s3/ipfs/orbitdb, then maintains a state of which what models are currently accesible, and it will choose what files should be cached through local/s3/ipfs/ based on configuration settings.",
 	"type": "module",
-	"main": "model_manager.js",
+	"main": "index.js",
 	"scripts": {
 		"test": "node tests/test.js"
 	},
@@ -24,19 +24,23 @@
 		"deasync": "^0.1.29",
 		"debug": "^4.3.4",
 		"esm": "^3.2.25",
-		"ipfs_huggingface_scraper_js": "^1.0.1",
-		"ipfs_kit_js": "^1.0.21",
+		"ipfs_faiss_js": "^1.0.4",
+		"ipfs_huggingface_scraper_js": "^1.0.10",
+		"ipfs_kit_js": "^1.0.22",
+		"ipfs_model_manager_js": "^1.0.9",
 		"is-stream": "^3.0.0",
+		"libp2p_kit_js": "^1.0.5",
 		"moment": "^2.30.1",
 		"nodejs": "^0.0.0",
 		"npm": "^10.8.2",
-		"orbitdb_kit": "^1.0.1",
+		"orbitdb_kit_js": "^1.0.9",
 		"prompt-sync": "^4.2.0",
 		"prompt-sync-history": "^1.0.1",
 		"shutil": "^0.0.1",
 		"subprocess": "^0.3.0",
 		"sync-request": "6.1.0",
-		"tar": "^7.1.0"
+		"tar": "^7.1.0",
+		"tmp": "^0.2.3"
 	},
 	"author": "Benjamin Barber",
 	"license": "AGPL-3.0-only",

tests/test.js

@@ -1,17 +1,18 @@
-import libp2pKitJs from libp2p_kit_js
-import orbitDBKitJs from orbit_db_kit_js
-import ipfsFaissJs from ipfs_faiss_js
+import libp2pKitJs from 'libp2p_kit_js'
+import orbitDbKitJs from 'orbitdb_kit_js'
+import ipfsFaissJs from 'ipfs_faiss_js'
+import ipfsKitJs from 'ipfs_kit_js'
 import { storachaKitJs } from '../storacha_kit_js/main.js';
 import { fireproofDbKitJs } from '../fireproofdb_kit_js/main.js';
-import { ipfsModelManagerJs } from '../ipfs_model_manager_js/ipfs_model_manager.js';
+import { ipfsModelManagerJs } from 'ipfs_model_manager_js';
 import { requireConfig } from "../config/config.js";
 import fs from "fs";
 import path from "path";
 
 export default class testIpfsModelManager {
-    constructor() {
+    constructor(resources = {}, metadata = {}) {
         this.ipfsModelManager = new ipfsModelManagerJs(resources, metadata)
-        this.orbitDbKit = new orbitDBKitJs(resources, metadata)
+        this.orbitDbKit = new orbitDbKitJs(resources, metadata)
         this.ipfsFaiss = new ipfsFaissJs(resources, metadata)
         this.libp2pKit = new libp2pKitJs(resources, metadata)
         this.ipfsKit = new ipfsKitJs(resources, metadata)
@@ -78,7 +79,7 @@ export default class testIpfsModelManager {
                 meta[key] = this.config[key];
             }
         }
-        this.modelManager = new ipfsModelManager(null, meta);
+        this.modelManager = new ipfsModelManagerJs(resources, metadata);
     }
 
 
@@ -348,7 +349,7 @@ export default class testIpfsModelManager {
 }
 
 export class testS3Kit {
-    constructor() {
+    constructor(resources = {}, metadata = {}) {
         const endpoint = "https://object.ord1.coreweave.com"
         const access_key = "CWVFBNRZEEDYTAUM"
         const secret_key = "cwoBNj1ILmRGxcm18EsWE5Qth4hVtmtNJPkLVW2AETU"
@@ -410,8 +411,8 @@ export class testS3Kit {
                 meta[key] = this.config[key];
             }
         }
-        this.s3Kit = new s3Kit(s3cfg);
-        this.modelManager = new ipfsModelManager(null, meta);
+        this.libp2pKit = new libp2pKitJs(null, meta);
+        this.s3Kit = new this.libp2pKit.s3Kit(s3cfg);
     }
 
 	async test() {
@@ -508,6 +509,64 @@ export class testS3Kit {
 
 }
 
+export async function test_fio() {
+    const thisTest = new TestFio(null);
+    const results = thisTest.test("/tmp/");
+    console.log(results);
+    console.log("Test complete");
+}
+
+export async function test(){
+    const endpoint = "https://object.ord1.coreweave.com"
+    const access_key = "CWVFBNRZEEDYTAUM"
+    const secret_key = "cwoBNj1ILmRGxcm18EsWE5Qth4hVtmtNJPkLVW2AETU"
+    const host_bucket = "%(bucket)s.object.ord1.coreweave.com"
+    const bucket = "cloudkit-beta";
+    const ipfs_src = "QmXBUkLywjKGTWNDMgxknk6FJEYu9fZaEepv3djmnEqEqD";
+    const s3cfg = {
+        "endpoint": endpoint,
+        "accessKey": access_key,
+        "secretKey": secret_key,
+        "hostBucket": host_bucket,   
+        "bucket": bucket
+    };
+    const cluster_name = "cloudkit_storage";
+    //let ipfs_path = "/storage/";
+    const localPath = "/storage/cloudkit-models";
+    //ipfs_path = "/storage/ipfs/";
+    const ten_mins = 600;
+    const ten_hours = 36000;
+    const ten_days = 864000;
+    const never =  100000000;
+    const role = "worker";
+    const cache = {
+        "local": "/storage/cloudkit-models/collection.json",
+        "s3": "s3://cloudkit-beta/collection.json",
+        "ipfs": ipfs_src,
+        "https": "https://huggingface.co/endomorphosis/cloudkit-collection/resolve/main/collection.json"
+    };
+    const timing = {
+        "local_time": ten_mins,
+        "s3_time": ten_hours,
+        "ipfs_time": ten_days,
+        "https_time": never,
+    };
+    const meta = {
+        // "s3cfg": s3cfg,
+        "ipfs_src": ipfs_src,
+        "timing": timing,
+        "cache": cache,
+        "role": role,
+        "cluster_name": cluster_name,
+        //"ipfs_path": ipfs_path,
+        //"localPath": localPath,
+        //"ipfs_path": ipfs_path
+    };
+
+    const models_manager = new ipfsModelManagerJs(null, meta);
+    const results = await models_manager.test();
+    console.log(results);
+}
 
 if (import.meta.url === 'file://' + process.argv[1]) {
     const testModelManager = new testIpfsModelManager();