chore: Update dependencies and refactor S3 module to use AWS SDK v3

index.js

@@ -0,0 +1,4 @@
+export * from '/ipfs_model_manager_js/model_manager.js';
+export * from '/ipfs_model_manager_js/s3.js';
+export * from '/ipfs_model_manager_js/s3_kit.js';
+export * from '/ipfs_model_manager_js/test_fio.js';

ipfs_model_manager_js/model_manager.js

@@ -4,23 +4,16 @@ import path from 'path';
 import util from 'util';
 import { promisify } from 'util';
 import { exec } from 'child_process';
-import { ipfs_kit_js } from 'ipfs_kit_js';
-//from './s3_kit.js' import S3Kit;
-//import AWS from 'aws-sdk';
-//import { S3 } from 'aws-sdk';
-//import ipfsClient from 'ipfs-http-client';
+import { ipfsKitJs, installIpfs } from 'ipfs_kit_js';
 import * as test_fio from './test_fio.js';
 import * as s3_kit from './s3_kit.js';
-import * as ipfs_kit from './ipfs_kit.js';
-import * as install_ipfs from './ipfs_kit_lib/install_ipfs.js';
+// import * as install_ipfs from './ipfs_kit_lib/install_ipfs.js';
 import fsExtra from 'fs-extra';
 import crypto from 'crypto';
 import rimraf from 'rimraf';
 import _ from 'lodash';
 import * as temp_file from "./tmp_file.js";
 
-//const s3 = new AWS.S3();
-//const ipfs = ipfsClient('http://localhost:5001');
 const readFile = util.promisify(fs.readFile);
 const writeFile = util.promisify(fs.writeFile);
 const stat = util.promisify(fs.stat);
@@ -59,7 +52,8 @@ class ModelManager {
             this.localPath = "/root/";
             localPath = this.localPath;
         } else {
-            this.localPath = path.join(os.homeDir(), username);
+            this.homeDir = os.homedir();
+            this.localPath = this.homeDir;
             localPath = this.localPath;
         }
         if (meta !== null && typeof meta === 'object') {

ipfs_model_manager_js/test_fio.js

@@ -0,0 +1,149 @@
+import { execSync } from 'child_process';
+import fs from 'fs';
+import path from 'path';
+
+
+export class TestFio {
+    constructor(resources, meta = null) {
+        this.thisDir = path.dirname(import.meta.url);
+        if (this.thisDir.startsWith("file://")) {
+            this.thisDir = this.thisDir.replace("file://", "");
+        }
+        this.path = process.env.PATH;
+        this.path = this.path + ":" + path.join(this.thisDir, "bin")
+        this.pathString = "PATH="+ this.path
+    }
+
+    call(method, kwargs = {}) {
+        if (method === "test") {
+            return this.test(kwargs);
+        }
+    }
+
+    diskDeviceNameFromLocation(location) {
+        let directory_tree = location.split("/");
+
+        const command = "df -h";
+        let df = execSync(command).toString();
+        df = df.split("\n");
+        for (let line of df) {
+            if (line.includes(location)) {
+                const device = line.split(" ")[0];
+                return device;
+            } else {
+                while (directory_tree.length > 1) {
+                    directory_tree.pop();
+                    location = directory_tree.join("/");
+                    for (let line of df) {
+                        if (directory_tree.length === 1 && location === "") {
+                            location = "/";
+                        }
+                        if (line.includes(location)) {
+                            while (line.includes("  ")) {
+                                line = line.replace("  ", " ");
+                            }
+                            const mount = line.split(" ");
+                            if (mount[5] === location) {
+                                const device = mount[0];
+                                return device;
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        return "rootfs";
+    }
+
+
+    diskDeviceTotalCapacity(device) {
+        const command = "df -h";
+        let df = execSync(command).toString();
+        df = df.split("\n");
+        for (let line of df) {
+            if (line.includes(device)) {
+                while (line.includes("  ")) {
+                    line = line.replace("  ", " ");
+                }
+                const capacity = line.split(" ")[1];
+                return capacity;
+            }
+        }
+        return null;
+    }
+
+    diskDeviceUsedCapacity(device) {
+        const command = "df -h";
+        let df = execSync(command).toString();
+        df = df.split("\n");
+        for (let line of df) {
+            if (line.includes(device)) {
+                while (line.includes("  ")) {
+                    line = line.replace("  ", " ");
+                }
+                const capacity = line.split(" ")[2];
+                return capacity;
+            }
+        }
+        return null;
+    }
+
+    diskDeviceAvailCapacity(device) {
+        const command = "df -h";
+        let df = execSync(command).toString();
+        df = df.split("\n");
+        for (let line of df) {
+            if (line.includes(device)) {
+                while (line.includes("  ")) {
+                    line = line.replace("  ", " ");
+                }
+                const capacity = line.split(" ")[3];
+                return capacity;
+            }
+        }
+        return null;
+    }
+
+    diskSpeed4k(location) {
+        const tempFile = tmp.fileSync({ postfix: '.iso', dir: location });
+        const timestamp_0 = Date.now();
+        const command = `dd if=/dev/zero of=${tempFile.name} bs=4k count=8k conv=fdatasync`;
+        execSync(command);
+        const timestamp_1 = Date.now();
+        const write_speed = 32 / ((timestamp_1 - timestamp_0) / 1000);
+        const command2 = `dd if=${tempFile.name} of=/dev/null bs=4k`;
+        execSync(command2);
+        const timestamp_2 = Date.now();
+        const read_speed = 32 / ((timestamp_2 - timestamp_1) / 1000);
+        fs.unlinkSync(tempFile.name);
+        return { read_speed, write_speed };
+    }
+
+
+    stats(location, kwargs = {}) {
+        const disk_device = this.diskDeviceNameFromLocation(location);
+        const disk_capacity = this.diskDeviceTotalCapacity(disk_device);
+        const disk_used = this.diskDeviceUsedCapacity(disk_device);
+        const disk_avail = this.diskDeviceAvailCapacity(disk_device);
+        const { read_speed: disk_read_speed, write_speed: disk_write_speed } = this.diskSpeed4k(location);
+        const results = {
+            disk_device,
+            disk_capacity,
+            disk_used,
+            disk_avail,
+            disk_write_speed,
+            disk_read_speed
+        };
+        return results;
+    }
+
+
+}
+
+function test(){
+    const thisTest = new TestFio(null);
+    const results = thisTest.test("/tmp/");
+    console.log(results);
+    console.log("Test complete");
+    //process.exit(0);
+}

ipfs_model_manager_js/tmp_file.js

@@ -0,0 +1,36 @@
+import os from 'os';
+import path from 'path';
+import fs from 'fs';
+import crypto from 'crypto';
+
+
+export class TempFileManager {
+    constructor(resources, meta = null) {
+        this.createTempDirectory = this.createTempDirectory.bind(this);
+        this.createTempFile = this.createTempFile.bind(this);
+    }
+
+    async createTempFile({ postfix, dir, }) {
+        const randomString = crypto.randomBytes(12).toString('hex');
+        const tempFileName = `${randomString}.${postfix}`;
+        const tempFilePath = path.join(dir, tempFileName);
+        const fd = fs.openSync(tempFilePath, 'w');
+        const cleanupCallback = () => {
+            fs.unlinkSync(tempFilePath);
+        };
+        let results = { tempFilePath, fd, cleanupCallback };
+        return (results);
+    }
+
+    async createTempDirectory({ dir }) {
+        const randomString = crypto.randomBytes(12).toString('hex');
+        const tempDirectoryName = `${randomString}`;
+        const tempDirectoryPath = path.join(dir, tempDirectoryName);
+        const fd = fs.openSync(tempDirectoryPath, 'w');
+        const cleanupCallback = () => {
+            fs.rmdirSync(tempDirectoryPath);
+        };
+        return ({ tempDirectoryPath , fd, cleanupCallback});
+    }
+}
+

package.json

@@ -1,44 +1,45 @@
 {
-  "name": "ipfs_model_manager_js",
-  "version": "1.0.2",
-  "description": "This is a model manager and wrapper for huggingface, and it maintains an index of models from collections of models store retrieved through local/https/s3/ipfs/orbitdb, then maintains a state of which what models are currently accesible, and it will choose what files should be cached through local/s3/ipfs/ based on configuration settings.",
-  "main": "model_manager.js",
-  "scripts": {
-    "test": " node tests/test.js"
-  },
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/endomorphosis/ipfs_model_manager_js.git"
-  },
-  "author": "",
-  "dependencies": {
-    "@aws-sdk/client-s3": "^3.315.0",
-    "@aws-sdk/lib-storage": "^3.315.0",
-    "@chainsafe/libp2p-gossipsub": "^13.0.0",
-    "@mwni/toml": "^1.0.0",
-    "@orbitdb/core": "^2.1.0",
-    "@orbitdb/quickstart": "^1.0.0",
-    "aws-sdk": "^2.1597.0",
-    "commander": "^11.0.0",
-    "crypto": "^1.0.1",
-    "deasync": "^0.1.29",
-    "debug": "^4.3.4",
-    "ipfs_kit_js": "^1.0.0",
-    "ipfs-cluster-api": "^0.0.9",
-    "ipfs-http-client": "^60.0.1",
-    "is-stream": "^3.0.0",
-    "nodejs": "^0.0.0",
-    "npm": "^10.8.2",
-    "prompt-sync": "^4.2.0",
-    "prompt-sync-history": "^1.0.1",
-    "shutil": "^0.0.1",
-    "subprocess": "^0.3.0",
-    "sync-request": "6.1.0",
-    "tar": "^7.1.0"
-  },
-  "license": "ISC",
-  "bugs": {
-    "url": "https://github.com/endomorphosis/ipfs_model_manager_js/issues"
-  },
-  "homepage": "https://github.com/endomorphosis/ipfs_model_manager_js#readme"
+	"name": "ipfs_model_manager_js",
+	"version": "1.0.2",
+	"description": "This is a model manager and wrapper for huggingface, and it maintains an index of models from collections of models store retrieved through local/https/s3/ipfs/orbitdb, then maintains a state of which what models are currently accesible, and it will choose what files should be cached through local/s3/ipfs/ based on configuration settings.",
+	"type": "module",
+	"main": "model_manager.js",
+	"scripts": {
+		"test": "node tests/test.js"
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/endomorphosis/ipfs_model_manager_js.git"
+	},
+	"author": "",
+	"dependencies": {
+		"@aws-sdk/client-s3": "^3.315.0",
+		"@aws-sdk/lib-storage": "^3.315.0",
+		"@chainsafe/libp2p-gossipsub": "^13.0.0",
+		"@mwni/toml": "^1.0.0",
+		"@orbitdb/core": "^2.1.0",
+		"@orbitdb/quickstart": "^1.0.0",
+		"aws-sdk": "^2.1597.0",
+		"commander": "^11.0.0",
+		"crypto": "^1.0.1",
+		"deasync": "^0.1.29",
+		"debug": "^4.3.4",
+		"ipfs_kit_js": "^1.0.0",
+		"ipfs-cluster-api": "^0.0.9",
+		"ipfs-http-client": "^60.0.1",
+		"is-stream": "^3.0.0",
+		"nodejs": "^0.0.0",
+		"npm": "^10.8.2",
+		"prompt-sync": "^4.2.0",
+		"prompt-sync-history": "^1.0.1",
+		"shutil": "^0.0.1",
+		"subprocess": "^0.3.0",
+		"sync-request": "6.1.0",
+		"tar": "^7.1.0"
+	},
+	"license": "ISC",
+	"bugs": {
+		"url": "https://github.com/endomorphosis/ipfs_model_manager_js/issues"
+	},
+	"homepage": "https://github.com/endomorphosis/ipfs_model_manager_js#readme"
 }