From 2a01ed3ab53730eb7cc291ea6a468ecc8211be63 Mon Sep 17 00:00:00 2001 From: endomorphosis Date: Sat, 3 Aug 2024 00:59:55 -0700 Subject: [PATCH] chore: Refactor S3 module to use AWS SDK v3 --- .../model_manager.js | 1 + s3.js => ipfs_model_manager_js/s3.js | 0 s3_kit.js => ipfs_model_manager_js/s3_kit.js | 0 utils.js => ipfs_model_manager_js/utils.js | 104 ------------------ package.json | 45 ++++---- 5 files changed, 24 insertions(+), 126 deletions(-) rename model_manager.js => ipfs_model_manager_js/model_manager.js (99%) rename s3.js => ipfs_model_manager_js/s3.js (100%) rename s3_kit.js => ipfs_model_manager_js/s3_kit.js (100%) rename utils.js => ipfs_model_manager_js/utils.js (86%) diff --git a/model_manager.js b/ipfs_model_manager_js/model_manager.js similarity index 99% rename from model_manager.js rename to ipfs_model_manager_js/model_manager.js index ef647b9..8dbd3f4 100644 --- a/model_manager.js +++ b/ipfs_model_manager_js/model_manager.js @@ -4,6 +4,7 @@ import path from 'path'; import util from 'util'; import { promisify } from 'util'; import { exec } from 'child_process'; +import { ipfs_kit_js } from 'ipfs_kit_js'; //from './s3_kit.js' import S3Kit; //import AWS from 'aws-sdk'; //import { S3 } from 'aws-sdk'; diff --git a/s3.js b/ipfs_model_manager_js/s3.js similarity index 100% rename from s3.js rename to ipfs_model_manager_js/s3.js diff --git a/s3_kit.js b/ipfs_model_manager_js/s3_kit.js similarity index 100% rename from s3_kit.js rename to ipfs_model_manager_js/s3_kit.js diff --git a/utils.js b/ipfs_model_manager_js/utils.js similarity index 86% rename from utils.js rename to ipfs_model_manager_js/utils.js index ff3f58f..e540758 100644 --- a/utils.js +++ b/ipfs_model_manager_js/utils.js @@ -9,25 +9,6 @@ import request from 'sync-request' import {createClient, createUploadStream} from './s3.js' import process from 'process' -export function open_ended_question(question){ - - let prompt = prompt_sync(({ - history: prompt_sync_history(), - autocomplete: complete([]), - sigint: true - })) - - let answer = prompt(question) - - console.log("confirm answer: " + answer) - let confirm = prompt("Confirm? (y/n): ") - if (confirm != 'y' && confirm != 'Y'){ - answer = open_ended_question(question) - } - - return answer -} - export function prepare_source(generate){ } @@ -36,24 +17,6 @@ export function extract_source(generate){ } -export function multiple_select_question(question,choices){ - let tmp_choices = choices - let done = false - let selections = [] - let selection = multiple_choice_question(question, tmp_choices) - while(!done){ - if (selection == 'None' || selection == '' || selection == null){ - done = true - } - else{ - selections.push(selection) - tmp_choices = tmp_choices.filter(function(value, index, arr){ return value != selection;}) - selection = multiple_choice_question(question, tmp_choices) - } - } - return selections -} - export function folder_data(generate, manifest, build_path){ let files let parent_dir = path.dirname(build_path) @@ -90,46 +53,6 @@ export function folder_data(generate, manifest, build_path){ return fileDict } -export function multiple_choice_question(question, choices){ - - let prompt = prompt_sync(({ - history: prompt_sync_history(), - autocomplete: complete(choices), - sigint: true - })) - - let new_choices = [] - new_choices.push("None") - for (var choice in choices){ - new_choices.push(choices[choice]) - } - - let index = 0 - console.log(question) - - for( var choice in new_choices){ - console.log(index + ". " + new_choices[choice]) - index += 1 - } - - let answer = prompt("Select:") - - if (!new_choices.includes(answer)){ - if (parseInt(answer) >= 0 && parseInt(answer) < new_choices.length){ - answer = new_choices[parseInt(answer)] - } - else{ - console.log("Invalid Selection") - answer = multiple_choice_question(question, choices) - } - } - - if (answer == 'None'){ - answer = '' - } - return answer -} - export function generate_cache_paths(generate, local_path){ let s3bucket_name = JSON.parse(process.env.s3_creds)["bucket"] @@ -148,33 +71,6 @@ export function generate_cache_paths(generate, local_path){ return dict } -export function complete(commands){ - return function (str) { - var i; - var ret = []; - for (i=0; i< commands.length; i++) { - if (commands[i].indexOf(str) == 0) - ret.push(commands[i]); - } - return ret; - }; -}; - -export function parse_templates(templates){ - let results = {} - - let chat_templates = ["llama-2", "vicuna", "alpaca", "chatlm"] - let instruct_templates = [] - - for( var template in templates){ - this_template = templates[template] - if(chat_templates.includes(this_template)){ - results["chat"] = this_template - } - } - return results -} - export function folder_to_bytestream(folder){ let files = fs.readdirSync(folder) let bytestream = [] diff --git a/package.json b/package.json index 9cde148..0f914d9 100644 --- a/package.json +++ b/package.json @@ -12,29 +12,30 @@ }, "author": "", "dependencies": { - "@aws-sdk/client-s3": "^3.315.0", - "@aws-sdk/lib-storage": "^3.315.0", - "@chainsafe/libp2p-gossipsub": "^13.0.0", - "@mwni/toml": "^1.0.0", + "@aws-sdk/client-s3": "^3.315.0", + "@aws-sdk/lib-storage": "^3.315.0", + "@chainsafe/libp2p-gossipsub": "^13.0.0", + "@mwni/toml": "^1.0.0", + "@orbitdb/core": "^2.1.0", + "@orbitdb/quickstart": "^1.0.0", + "aws-sdk": "^2.1597.0", + "commander": "^11.0.0", + "crypto": "^1.0.1", + "deasync": "^0.1.29", + "debug": "^4.3.4", "ipfs_kit_js": "^1.0.0", - "@orbitdb/core": "^2.1.0", - "@orbitdb/quickstart": "^1.0.0", - "aws-sdk": "^2.1597.0", - "commander": "^11.0.0", - "crypto": "^1.0.1", - "deasync": "^0.1.29", - "debug": "^4.3.4", - "ipfs-cluster-api": "^0.0.9", - "ipfs-http-client": "^60.0.1", - "is-stream": "^3.0.0", - "nodejs": "^0.0.0", - "prompt-sync": "^4.2.0", - "prompt-sync-history": "^1.0.1", - "shutil": "^0.0.1", - "subprocess": "^0.3.0", - "sync-request": "6.1.0", - "tar": "^7.1.0" - }, + "ipfs-cluster-api": "^0.0.9", + "ipfs-http-client": "^60.0.1", + "is-stream": "^3.0.0", + "nodejs": "^0.0.0", + "npm": "^10.8.2", + "prompt-sync": "^4.2.0", + "prompt-sync-history": "^1.0.1", + "shutil": "^0.0.1", + "subprocess": "^0.3.0", + "sync-request": "6.1.0", + "tar": "^7.1.0" + }, "license": "ISC", "bugs": { "url": "https://github.com/endomorphosis/ipfs_model_manager_js/issues"