chore: Refactor S3 module to use AWS SDK v3

endomorphosis · Aug 3, 2024 · 2a01ed3 · 2a01ed3
1 parent 5d4aa95
commit 2a01ed3
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 126 deletions.
diff --git a/model_manager.js → ipfs_model_manager_js/model_manager.js b/model_manager.js → ipfs_model_manager_js/model_manager.js
@@ -4,6 +4,7 @@ import path from 'path';
 import util from 'util';
 import { promisify } from 'util';
 import { exec } from 'child_process';
+import { ipfs_kit_js } from 'ipfs_kit_js';
 //from './s3_kit.js' import S3Kit;
 //import AWS from 'aws-sdk';
 //import { S3 } from 'aws-sdk';

diff --git a/s3.js → ipfs_model_manager_js/s3.js b/s3.js → ipfs_model_manager_js/s3.js
diff --git a/s3_kit.js → ipfs_model_manager_js/s3_kit.js b/s3_kit.js → ipfs_model_manager_js/s3_kit.js
diff --git a/utils.js → ipfs_model_manager_js/utils.js b/utils.js → ipfs_model_manager_js/utils.js
@@ -9,25 +9,6 @@ import request from 'sync-request'
 import {createClient, createUploadStream} from './s3.js'
 import process from 'process'
 
-export function open_ended_question(question){
-
-    let prompt = prompt_sync(({
-        history: prompt_sync_history(),
-        autocomplete: complete([]),
-        sigint: true
-    }))
-
-    let answer = prompt(question)
-
-    console.log("confirm answer: " + answer)
-    let confirm = prompt("Confirm? (y/n): ")
-    if (confirm != 'y' && confirm != 'Y'){
-        answer = open_ended_question(question)
-    }
-
-    return answer
-}
-
 export function prepare_source(generate){
 
 }
@@ -36,24 +17,6 @@ export function extract_source(generate){
 
 }
 
-export function multiple_select_question(question,choices){
-    let tmp_choices = choices
-    let done = false
-    let selections = []
-    let selection = multiple_choice_question(question, tmp_choices)
-    while(!done){
-        if (selection == 'None' || selection == '' || selection == null){
-            done = true
-        }
-        else{
-            selections.push(selection)
-            tmp_choices = tmp_choices.filter(function(value, index, arr){ return value != selection;})
-            selection = multiple_choice_question(question, tmp_choices)
-        }
-    }
-    return selections
-}
-
 export function folder_data(generate, manifest, build_path){
     let files 
     let parent_dir = path.dirname(build_path)
@@ -90,46 +53,6 @@ export function folder_data(generate, manifest, build_path){
     return  fileDict   
 }
 
-export function multiple_choice_question(question, choices){
-
-    let prompt = prompt_sync(({
-        history: prompt_sync_history(),
-        autocomplete: complete(choices),
-        sigint: true
-    }))
-
-    let new_choices = []
-    new_choices.push("None")
-    for (var choice in choices){
-        new_choices.push(choices[choice])
-    }
-
-    let index = 0
-    console.log(question)
-
-    for( var choice in new_choices){
-        console.log(index + ". " + new_choices[choice])
-        index += 1
-    }
-
-    let answer = prompt("Select:")
-
-    if (!new_choices.includes(answer)){
-        if (parseInt(answer) >= 0 && parseInt(answer) < new_choices.length){
-            answer = new_choices[parseInt(answer)]
-        }
-        else{
-            console.log("Invalid Selection")
-            answer = multiple_choice_question(question, choices)
-        }
-    }
-
-    if (answer == 'None'){
-        answer = ''
-    }
-    return answer
-}
-
 export function generate_cache_paths(generate, local_path){
 
     let s3bucket_name = JSON.parse(process.env.s3_creds)["bucket"]
@@ -148,33 +71,6 @@ export function generate_cache_paths(generate, local_path){
     return dict
 }
 
-export function complete(commands){
-    return function (str) {
-      var i;
-      var ret = [];
-      for (i=0; i< commands.length; i++) {
-        if (commands[i].indexOf(str) == 0)
-          ret.push(commands[i]);
-      }
-      return ret;
-    };
-};
-
-export function parse_templates(templates){
-    let results = {}
-
-    let chat_templates = ["llama-2", "vicuna", "alpaca", "chatlm"]
-    let instruct_templates = []
-
-    for( var template in templates){
-        this_template = templates[template]
-        if(chat_templates.includes(this_template)){
-            results["chat"] = this_template
-        }
-    }
-    return results
-}
-
 export function folder_to_bytestream(folder){
     let files = fs.readdirSync(folder)
     let bytestream = []

diff --git a/package.json b/package.json
@@ -12,29 +12,30 @@
   },
   "author": "",
   "dependencies": {
-		"@aws-sdk/client-s3": "^3.315.0",
-		"@aws-sdk/lib-storage": "^3.315.0",
-		"@chainsafe/libp2p-gossipsub": "^13.0.0",
-		"@mwni/toml": "^1.0.0",
+    "@aws-sdk/client-s3": "^3.315.0",
+    "@aws-sdk/lib-storage": "^3.315.0",
+    "@chainsafe/libp2p-gossipsub": "^13.0.0",
+    "@mwni/toml": "^1.0.0",
+    "@orbitdb/core": "^2.1.0",
+    "@orbitdb/quickstart": "^1.0.0",
+    "aws-sdk": "^2.1597.0",
+    "commander": "^11.0.0",
+    "crypto": "^1.0.1",
+    "deasync": "^0.1.29",
+    "debug": "^4.3.4",
     "ipfs_kit_js": "^1.0.0",
-		"@orbitdb/core": "^2.1.0",
-		"@orbitdb/quickstart": "^1.0.0",
-		"aws-sdk": "^2.1597.0",
-		"commander": "^11.0.0",
-		"crypto": "^1.0.1",
-		"deasync": "^0.1.29",
-		"debug": "^4.3.4",
-		"ipfs-cluster-api": "^0.0.9",
-		"ipfs-http-client": "^60.0.1",
-		"is-stream": "^3.0.0",
-		"nodejs": "^0.0.0",
-		"prompt-sync": "^4.2.0",
-		"prompt-sync-history": "^1.0.1",
-		"shutil": "^0.0.1",
-		"subprocess": "^0.3.0",
-		"sync-request": "6.1.0",
-		"tar": "^7.1.0"
-	},
+    "ipfs-cluster-api": "^0.0.9",
+    "ipfs-http-client": "^60.0.1",
+    "is-stream": "^3.0.0",
+    "nodejs": "^0.0.0",
+    "npm": "^10.8.2",
+    "prompt-sync": "^4.2.0",
+    "prompt-sync-history": "^1.0.1",
+    "shutil": "^0.0.1",
+    "subprocess": "^0.3.0",
+    "sync-request": "6.1.0",
+    "tar": "^7.1.0"
+  },
   "license": "ISC",
   "bugs": {
     "url": "https://github.com/endomorphosis/ipfs_model_manager_js/issues"