Update IPFS private key and MySQL credentials, add API functions, and…

… fix import statements

README.md

@@ -1,36 +1,27 @@
 # Data Economy Hackathon
 IPFS Huggingface Bridge
 
-Author - Benjamin Barber @endomorphosis
-
-QA / website - Kevin De Haan @coregod360
-
-CLEANUP / Windows compatibility / Breakfix 03/31/2024 - 04/07/2024
+Author - Benjamin Barber
+QA - Kevin De Haan
 
 # About
 
 This is a model manager and wrapper for huggingface, looks up a index of models from an collection of models, and will download a model from either https/s3/ipfs, depending on which source is the fastest.
 
 # How to use
 
-to install
-
-python3 setup.py
+setup.py
 
 In your python script
 
 from transformers import AutoModelForSeq2SeqLM
-
-from ipfs_transformers import AutoModelForSeq2SeqLM
-
+from ipfs_huggingface_bridge import AutoModelForSeq2SeqLM
 model = AutoModelForSeq2SeqLM.from_auto_download("google/t5_11b_trueteacher_and_anli")
 
 or 
 
 from transformers import AutoModelForSeq2SeqLM
-
-from ipfs_transformers import AutoModelForSeq2SeqLM
-
+from ipfs_huggingface_bridge import AutoModelForSeq2SeqLM
 model = AutoModelForSeq2SeqLM.from_ipfs("QmWJr4M1VN5KpJjqCsJsJg7PDmFoqQYs1BKpYxcdMY1qkh")
 
 To scrape huggingface

huggingface_scraper/generate_manifest.js

@@ -3,13 +3,21 @@ import path, { parse } from 'path'
 import prompt_sync from 'prompt-sync'
 import prompt_sync_history from 'prompt-sync-history'
 import hf_embed_calc from './manifests/manifest_hf_embed.js'
+import hf_embed_add from './manifests/manifest_hf_embed.js'
 import llama_cpp_calc from './manifests/manifest_llama_cpp.js'
+import llama_cpp_add from './manifests/manifest_llama_cpp.js'
 import hf_lm_calc from './manifests/manifest_hf_lm.js'
+import hf_lm_add from './manifests/manifest_hf_lm.js'
 import hf_t5_calc from './manifests/manifest_hf_t5.js'
+import hf_t5_add from './manifests/manifest_hf_t5.js'
 import knn_calc from './manifests/manifest_knn.js'
+import knn_add from './manifests/manifest_knn.js'
 import api_calc from './manifests/manifest_api.js'
+import api_add from './manifests/manifest_api.js'
 import hf_faster_whisper_calc from './manifests/manifest_hf_faster_whisper.js'
+import hf_faster_whisper_add from './manifests/manifest_hf_faster_whisper.js'
 import hf_transformers_calc from './manifests/manifest_hf_transformers.js'
+import hf_transformers_add from './manifests/manifest_hf_transformers.js'
 import diffusion_calc from './manifests/manifest_diffusion.js'
 import {complete, parse_templates, generate_test} from './utils.js'
 
@@ -22,15 +30,61 @@ export class Generate_Manifest{
     main(generate){
         let self = this
         let generation = self.generate(generate)
+        let test_generation = false
+        if (test_generation){
+            let add_generation = this.add_generator_to_model_data(self, generation)
+        }
         return generation
     }
 
+    add_generator_to_model_data(self, generation){
+        if (generation.skill == undefined){
+            throw("skill is undefined")
+        }
+        else if (generation.skill == 'hf_transformers'){
+            results = hf_transformers_add(self)
+        }
+        else if (generation.skill == 'hf_embed'){
+            results = hf_embed_add(self)
+        }
+        else if (generation.skill == 'llama_cpp'){
+            results = llama_cpp_add(self)
+        }
+        else if (generation.skill == 'diffusion'){
+            results = diffusion_add(self)
+        }
+        else if (generation.skill == 'knn'){
+            results = knn_add(self)
+        }
+        else if (generation.skill == 'api'){
+            results = api_add(self)
+        }
+        else if (generation.skill == 'custom'){
+            results = custom_add(self)
+        }
+        else if (generation.skill == 'hf_faster_whisper'){
+            results = hf_faster_whisper_add(self)
+        }
+        else if (generation.skill == 'hf_lm'){
+            results = hf_lm_add(self)
+        }
+        else if (generation.skill == 'hf_t5'){
+            results = hf_t5_add(self)
+        }
+        else{
+            throw("skill is not defined")
+        }
+
+
+        return results
+    }
+
     generate(generate){
         let self = {}
         let metadata = self.metadata
         let hwRequirements = self.hwRequirements
         let results
-        let model_types = ['hf_transformers', 'hf_embed', 'llama_cpp', 'diffusion', 'knn', 'api', 'whisper', 'custom']
+        let model_types = ['hf_transformers', 'hf_embed', 'llama_cpp', 'diffusion', 'knn', 'api', 'whisper', 'hf_lm', 'hf_t5']
         // prompt the user for input
         console.log("1. hf_transformers")
         console.log("2. hf_embed")
@@ -39,7 +93,8 @@ export class Generate_Manifest{
         console.log("5. knn")
         console.log("6. api")
         console.log("7. whisper")
-        console.log("8. custom")
+        console.log("8. hf_lm")
+        console.log("9. hf_t5")
         // request console input
         let prompt = prompt_sync(({
             history: prompt_sync_history(),
@@ -103,12 +158,11 @@ export class Generate_Manifest{
         }
 
         try{
-            test = generate_test(results)
+           let test = generate_test(results)
         }
         catch(err){
             console.log(err)
-            throw("Error in testing the manifest " +  model_type)
-
+//            throw("Error in testing the manifest " +  model_type)
         }
         finally{
             return results

huggingface_scraper/main.js

@@ -6,11 +6,11 @@ export class Scraper {
     constructor(s3_creds, hf_creds, mysql_creds, local_model_path, collection_path) {
         this.env = process.env;
         if (s3_creds != undefined) {
-            this.env.s3_creds = s3_creds;
+            process.env.s3_creds = s3_creds;
             this.s3_creds = s3_creds;
         }
         if (hf_creds != undefined) {
-            this.env.hf_creds = hf_creds;
+            process.env.hf_creds = hf_creds;
             this.hf_creds = hf_creds;
         }
         if (mysql_creds != undefined) {
@@ -22,12 +22,13 @@ export class Scraper {
             this.local_model_path = local_model_path;
         }
         if (collection_path != undefined) {
-            this.env.collection_path = collection_path;
+            process.env.collection_path = collection_path;
             this.collection_path = collection_path;
         }
     }
 
     main() {
+        console.log(this)
         let args = process.argv.slice(2);
         let command;
         let source;
@@ -55,37 +56,38 @@ export class Scraper {
             //console.log("No command specified try -h or --help for help");
             let this_generate_manifest = new generate_manifest.Generate_Manifest();
             let this_manifest = this_generate_manifest.main();
+            console.log(this_manifest);
+            console.log(this)
             let this_process_manifest = new manifest.Manifest(
-                s3_creds = this.s3_creds,
-                hf_creds = this.hf_creds,
-                mysql_creds = this.mysql_creds,
-                local_model_path = this.local_model_path,
-                collection_path = this.collection_path
+                this.s3_creds,
+                this.hf_creds,
+                this.mysql_creds,
+                this.local_model_path,
+                this.collection_path
             );
-            this_process_manifest.main();
+            let processed_manifest = this_process_manifest.main(this_manifest);
         }
-
         if (command == 'import' && source != "hf") {
             throw new Error("Only hf is supported as a source");
         }
 
         if (command == "import" && source == "hf" && model == undefined) {
             let this_manifest = new manifest.Manifest(
-                s3_creds = this.s3_creds,
-                hf_creds = this.hf_creds,
-                mysql_creds = this.mysql_creds,
-                local_model_path = this.local_model_path,
-                collection_path = this.collection_path
+                this.s3_creds,
+                this.hf_creds,
+                this.mysql_creds,
+                this.local_model_path,
+                this.collection_path
             );
             this_manifest.import_from_hf();
         }
 
         if (command == "import" && source == "hf" && model != undefined) {
             let this_manifest = new manifest.Manifest(
-                s3_creds = this.s3_creds,
-                hf_creds = this.hf_creds,
-                mysql_creds = this.mysql_creds,
-                local_model_path = this.local_model_path,
+                this.s3_creds,
+                this.hf_creds,
+                this.mysql_creds,
+                this.local_model_path,
                 collection_path = this.collection_path
             );
             this_manifest.import_from_hf(model);

huggingface_scraper/manifest.js

@@ -21,6 +21,7 @@ export class Manifest{
         this.format = ""
         this.hwRequirements = {}
         this.metadata = {}
+        this.env = process.env
         if (s3_creds != undefined){
             this.env.s3_creds = s3_creds
             this.s3_creds = s3_creds
@@ -43,12 +44,15 @@ export class Manifest{
         }
     }
 
-    main(){
+    main(generate){
         let self = this
+        console.log("manifest.js")
+        console.log("main(generate)")
+        console.log(generate)
         let generation = generate_manifest.main(generate)
         let manifest = generation.manifest
         let folder = generation.folder
-        let generate = generation.generate
+        let this_generate = generation.generate
         let processing = new process_manifest.main(manifest, folder, generate)
         return processing
     }
@@ -62,31 +66,31 @@ export class Manifest{
     manifest_from_generator(this_generator){
         let generate = this_generator
         if (generate.skill == "hf_transformers"){
-            return hf_transformers_generate(generate)
+            return manifest_hf_transformers.hf_transformers_generate(generate)
         }
         if (generate.skill == "hf_embed"){
-            return hf_embed_generate(generate)
+            return manifest_hf_embed.hf_embed_generate(generate)
         }
         if (generate.skill == "llama_cpp"){
-            return manifest_llama_cpp(generate)
+            return manifest_llama_cpp.llama_cpp_generate(generate)
         }
         if (generate.skill == "diffusion"){
-            return diffusion_generate(generate)
+            return manifest_diffusion.diffusion_generate(generate)
         }
         if (generate.skill == "knn"){
-            return knn_generate(generate)
+            return manifest_knn.knn_generate(generate)
         }
         if (generate.skill == "hf_lm"){
-            return hf_lm_generate(generate)
+            return manifest_hf_lm.hf_lm_generate(generate)
         }
         if (generate.skill == "hf_t5"){
-            return hf_t5_generate(generate)
+            return manifest_hf_t5.hf_t5_generate(generate)
         }
         if (generate.skill == "api"){
-            return api_generate(generate)
+            return manifest_api.api_generate(generate)
         }
         if (generate.skill == "custom"){
-            return custom_generate(generate)
+            return manifest_custom.custom_generate(generate)
         }
     }
 

huggingface_scraper/manifests/manifest_api.js

@@ -78,3 +78,14 @@ export function api_generate(generate){
     results.skill = "api"
     return results
 }
+
+export function api_add(self, generation){
+    if (generation.modelName != undefined){
+        generate_api[generation.modelName] = generation
+        fs.writeFileSync(path.resolve('./modeldata/generate_api.json'), JSON.stringify(generate_api, null, 2))       
+        return Object.keys(generate_api)
+    }
+    else{
+        throw "model name is undefined"
+    }      
+}

huggingface_scraper/manifests/manifest_diffusion.js

@@ -1,6 +1,7 @@
 import {complete, open_ended_question, multiple_choice_question, parse_templates, generate_template, generate_metadata_template, generate_hwrequirements_template} from '../utils.js'
 import prompt_sync from 'prompt-sync'
 import prompt_sync_history from 'prompt-sync-history'
+import generate_diffusion from './modeldata/generate_diffusion.json' assert { type: 'json' };
 
 export class Manifest_diffusion{
     constructor(){
@@ -279,4 +280,15 @@ export function diffusion_generate(generate){
     results.metadata = diffusion_generate_metadata(generate)
     results.skill = "diffusion"
     return results
+}
+
+export function diffusion_add(self, generation){
+    if (generation.modelName != undefined){
+        generate_diffusion[generation.modelName] = generation
+        fs.writeFileSync(path.resolve('./modeldata/generate_diffusion.json'), JSON.stringify(generate_diffusion, null, 2))       
+        return Object.keys(generate_diffusion)
+    }
+    else{
+        throw "model name is undefined"
+    }      
 }