Moved to a new repository and scrubbed credentials

.gitignore

@@ -0,0 +1,4 @@
+.DS_Store
+node_modules
+package-lock.json
+__pycache__

README.md

@@ -0,0 +1,41 @@
+# Data Economy Hackathon
+IPFS Huggingface Bridge
+
+Author - Benjamin Barber
+QA - Kevin De Haan
+
+CLEANUP / Windows compatibility / Breakfix 03/31/2024 - 04/07/2024
+
+# About
+
+This is a model manager and wrapper for huggingface, looks up a index of models from an collection of models, and will download a model from either https/s3/ipfs, depending on which source is the fastest.
+
+# How to use
+
+setup.py
+
+In your python script
+
+from transformers import AutoModelForSeq2SeqLM
+from ipfs_huggingface_bridge import AutoModelForSeq2SeqLM
+model = AutoModelForSeq2SeqLM.from_auto_download("google/t5_11b_trueteacher_and_anli")
+
+or 
+
+from transformers import AutoModelForSeq2SeqLM
+from ipfs_huggingface_bridge import AutoModelForSeq2SeqLM
+model = AutoModelForSeq2SeqLM.from_ipfs("QmWJr4M1VN5KpJjqCsJsJg7PDmFoqQYs1BKpYxcdMY1qkh")
+
+To scrape huggingface
+
+interactive prompt:
+
+node scraper.js 
+
+import a model:
+
+node scraper.js hf "modelname" (as defined in your .json files)
+
+import all models 
+
+node scraper.js hf 

huggingface_scraper/checkpoint_diffusion_convert.js

@@ -0,0 +1,2 @@
+import fs from 'fs'
+import path from 'path'

huggingface_scraper/checkpoint_gguf_convert.js

@@ -0,0 +1,30 @@
+import fs from 'fs'
+import path from 'path'
+import os from 'os'
+
+export function convert_fp32_to_gguf(path){
+    let this_file = __filename  
+    let this_dir = path.dirname(this_file)
+    dest_path = path.replace(".ggml", ".gguf")
+    let command = "python3 " + this_dir + "/libllama/avx2/convert-ggml-gguf.py " + path + " " + dest_path
+    console.log("command is: " + command)
+    let results = os.system(command)
+}
+
+export function convert_fp16_to_gguf(path){
+    let this_file = __filename  
+    let this_dir = path.dirname(this_file)
+    dest_path = path.replace(".ggml", ".gguf")
+    let command = "python3 "+ this_dir + "/libllama/avx2/convert-ggml-gguf.py " + path + " " + dest_path
+    console.log("command is: " + command)
+    let results = os.system(command)
+}
+
+export function convert_ggml_to_gguf(path){
+    let this_file = __filename  
+    let this_dir = path.dirname(this_file)
+    dest_path = path.replace(".ggml", ".gguf")
+    let command = "python3 "+ this_dir + "/libllama/avx2/convert-ggml-gguf.py " + path + " " + dest_path
+    console.log("command is: " + command)
+    let results = os.system(command)
+}