Skip to content

Commit

Permalink
refactor: Remove unused imports and commented code in ipfs_model_mana…
Browse files Browse the repository at this point in the history 
…ger.js
  • Loading branch information
@endomorphosis
endomorphosis committed Aug 18, 2024
1 parent e8f391e commit 78b0bde
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 72 deletions.
66 changes: 37 additions & 29 deletions ipfs_model_manager_js/ipfs_model_manager.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import { promisify } from 'util';
import { exec } from 'child_process';
import { ipfsKitJs, installIpfs } from 'ipfs_kit_js';
import * as testFio from './test_fio.js';
// import * as s3Kit from './s3_kit.js';
import * as s3Kit from './s3_kit.js';
// import * as install_ipfs from './ipfs_kit_lib/install_ipfs.js';
import fsExtra from 'fs-extra';
import crypto from 'crypto';
Expand All @@ -16,7 +16,6 @@ import * as temp_file from "./tmp_file.js";
import { execSync } from 'child_process';
import { requireConfig } from '../config/config.js';


const readFile = util.promisify(fs.readFile);
const writeFile = util.promisify(fs.writeFile);
const stat = util.promisify(fs.stat);
Expand All @@ -33,6 +32,7 @@ class ModelManager {
this.parentDir = path.dirname(this.thisDir);
if (fs.existsSync(path.join(this.parentDir, "config", "config.toml"))) {
this.config = new requireConfig({config: path.join(this.parentDir, "config", "config.toml")});
this.s3cfg = this.config.s3;
}
else{
// this.config = new requireConfig();
Expand All @@ -58,7 +58,6 @@ class ModelManager {
this.thisModelPath = null;
this.thisModel = null;
this.thisModelName = null;
this.s3cfg = null;
let username = os.userInfo().username;
let localPath
if (username === "root") {
Expand All @@ -70,7 +69,10 @@ class ModelManager {
localPath = this.localPath;
}
if (meta !== null && typeof meta === 'object') {
this.s3cfg = meta.s3cfg || null;
// for (let key in meta) {
// this[key] = meta[key];
// }
// this.s3cfg = meta.s3cfg || null;
this.ipfsSrc = meta.ipfs_src || null;
this.timing = meta.timing || null;
this.collectionCache = meta.cache || null;
Expand All @@ -94,12 +96,14 @@ class ModelManager {
meta.ipfsPath = path.join(this.localPath, ".cache/ipfs");
}
this.ipfsPath = meta.ipfsPath || (this.localPath + "/.cache/ipfs");
this.s3cfg = meta.s3cfg || null;
// this.s3cfg = meta.s3cfg || null;
if (Object.keys(meta).includes("s3_cfg")){
this.s3cfg = meta.s3_cfg;
}
}
else {
this.localPath = path.join(this.ipfsPath , "cloudkit-models");
// get the username of the current user and determine if its root
this.s3cfg = null;
this.role = "leecher";
this.clusterName = "cloudkit_storage";
this.cache = {
Expand All @@ -121,7 +125,7 @@ class ModelManager {
let homeDir = os.homedir();
let homeDirFiles = fs.readdirSync(homeDir);
this.testFio = new testFio.TestFio(resources, meta);
// this.s3Kit = new s3Kit.s3Kit(resources, meta);
this.s3Kit = new s3Kit.s3Kit(resources, meta);
this.ipfsKitJs = new ipfsKitJs(resources, meta);
this.installIpfs = new installIpfs(resources, meta);
let ipfsPath = this.ipfsPath;
Expand Down Expand Up @@ -1361,7 +1365,7 @@ class ModelManager {
}

async lsLocalModels(kwargs) {
let lsModels = this.lsModels();
let lsModels = await this.lsModels();
let localModels = {};
let timestamps = {};

Expand Down Expand Up @@ -1726,7 +1730,7 @@ class ModelManager {


async checkHistoryModels(kwargs = {}) {
const lsModels = this.lsModels();
const lsModels = await this.lsModels();
const currentTimestamp = Date.now() / 1000;
const historyJsonPath = path.join(this.ipfsPath, "history.json");

Expand Down Expand Up @@ -1769,7 +1773,7 @@ class ModelManager {


async checkZombies(kwargs = {}) {
const lsModels = this.lsModels();
const lsModels = await this.lsModels();
const localFiles = fs.readdirSync(this.localPath, { withFileTypes: true });
const lsLocalFiles = [];
const collectionFiles = ["collection.json"];
Expand Down Expand Up @@ -1799,7 +1803,7 @@ class ModelManager {

const s3Files = await this.s3Kit.s3LsDir("", this.s3cfg["bucket"]);
const s3FileNames = s3Files.map(file => file["key"]);
const ipfsFiles = await this.ipfsKit.ipfsLsPath("/");
const ipfsFiles = await this.ipfsKitJs.ipfsLsPath("/");
const ipfsFileNames = ipfsFiles["ipfsLsPath"].map(file => file["name"]);

const collectionPins = this.collectionPins;
Expand Down Expand Up @@ -1834,7 +1838,7 @@ class ModelManager {
}

async checkExpired(kwargs = {}) {
const lsModels = this.lsModels();
const lsModels = await this.lsModels();
const currentTimestamp = Date.now() / 1000;
const expired = {
"local" : [],
Expand Down Expand Up @@ -1868,7 +1872,7 @@ class ModelManager {
}

async checkPinnedModels(kwargs = {}) {
const lsModels = this.lsModels();
const lsModels = await this.lsModels();
while (Object.keys(this.pinnedModels).length < 5) {
const randomNumber = Math.random();
const calculate = Math.round(randomNumber * lsModels.length);
Expand Down Expand Up @@ -1917,22 +1921,26 @@ class ModelManager {
async downloadMissing(kwargs = {}) {
const currentTimestamp = Date.now() / 1000;
const notFound = this.checkNotFound();
for (const model of notFound["local"]) {
if (model in this.pinnedModels) {
this.download_model(model);
this.models["local_models"][model] = Date.now() / 1000;
} else if (this.historyModels[model] > currentTimestamp - this.timing["local_time"]) {
this.download_model(model);
this.models["local_models"][model] = Date.now() / 1000;
if (Object.keys(notFound).includes("local")){
for (const model of notFound["local"]) {
if (model in this.pinnedModels) {
this.download_model(model);
this.models["local_models"][model] = Date.now() / 1000;
} else if (this.historyModels[model] > currentTimestamp - this.timing["local_time"]) {
this.download_model(model);
this.models["local_models"][model] = Date.now() / 1000;
}
}
}
for (const model of notFound["s3"]) {
if (model in this.pinnedModels) {
this.s3Kit.s3UlDir(this.localPath + "/" + model, this.s3cfg["bucket"], this.models["s3_models"][model]["folderData"]);
this.models["s3_models"][model] = Date.now() / 1000;
} else if (this.historyModels[model] > currentTimestamp - this.timing["s3_time"]) {
this.s3Kit.s3UlDir(this.localPath + "/" + model, this.s3cfg["bucket"], this.models["s3_models"][model]["folderData"]);
this.models["s3_models"][model] = Date.now() / 1000;
if(Object.keys(notFound).includes("s3")){
for (const model of notFound["s3"]) {
if (model in this.pinnedModels) {
this.s3Kit.s3UlDir(this.localPath + "/" + model, this.s3cfg["bucket"], this.models["s3_models"][model]["folderData"]);
this.models["s3_models"][model] = Date.now() / 1000;
} else if (this.historyModels[model] > currentTimestamp - this.timing["s3_time"]) {
this.s3Kit.s3UlDir(this.localPath + "/" + model, this.s3cfg["bucket"], this.models["s3_models"][model]["folderData"]);
this.models["s3_models"][model] = Date.now() / 1000;
}
}
}
return null;
Expand Down Expand Up @@ -1981,7 +1989,7 @@ class ModelManager {
// this.download_model('stablelm-zephyr-3b-GGUF-Q2_K');
await this.downloadMissing();
await this.evictExpiredModels();
await this.evictZombies();
// await this.evictZombies();
return this;
}

Expand Down Expand Up @@ -2022,7 +2030,7 @@ const timing = {
"https_time": never,
};
const meta = {
"s3cfg": s3cfg,
// "s3cfg": s3cfg,
"ipfs_src": ipfs_src,
"timing": timing,
"cache": cache,
Expand Down
37 changes: 0 additions & 37 deletions ipfs_model_manager_js/s3.js
View file

This file was deleted.

8 changes: 2 additions & 6 deletions ipfs_model_manager_js/s3_kit.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ export class s3Kit {
this.s3cfgToBoto = this.configToBoto;
this.getSession = this.getSession;
if (meta !== null) {
if ('s3cfg' in meta) {
if (Object.keys(meta).includes('s3cfg')) {
if (meta['s3cfg'] !== null) {
this.s3cfg = meta['s3cfg'];
this.getSession(meta['s3cfg']);
Expand Down Expand Up @@ -125,10 +125,6 @@ export class s3Kit {

async s3LsDir(dir, bucketName, kwargs) {
let s3Config = kwargs && kwargs.s3cfg ? kwargs.s3cfg : this.s3cfg;
AWS.config.update({
accessKeyId: 'YOUR_ACCESS_KEY_ID',
secretAccessKey: 'YOUR_SECRET_ACCESS_KEY'
});

let s3 = new AWS.S3(this.s3cfgToBoto(s3Config));

Expand Down Expand Up @@ -749,5 +745,5 @@ async function test() {
await testThis.test();
}

// test();
test();
// main();

0 comments on commit 78b0bde

Please sign in to comment.