push update

.gitignore

@@ -2,3 +2,13 @@
 node_modules
 package-lock.json
 __pycache__
+/.idea/
+/orbitdb_kit/node_modules/
+/orbitdb_kit/ipfs/
+/orbitdb_kit/orbitdb/
+/ipfs/
+/orbitdb/
+build
+ipfs_transformers.egg-info
+config/config.toml
+config.toml

config/config template.toml

@@ -0,0 +1,86 @@
+[[PATHS]]
+local_path = "/storage/cloudkit-models"
+ipfs_path = "/storage/ipfs/"
+
+[[PATHS.COLLECTION]]
+"local" = "/storage/cloudkit-models/collection.json",
+"s3": "s3://cloudkit-beta/collection.json",
+"ipfs": "QmXBUkLywjKGTWNDMgxknk6FJEYu9fZaEepv3djmnEqEqD",
+"https": "https://huggingface.co/endomorphosis/cloudkit-collection/resolve/main/collection.json"
+"orbitdb: "/orbitdb/zdpuB31L6gJz49erikZSQT3A1erJbid8oUTBrjLtBwjjXe3R5"
+
+[[HF]]
+hf_account_name = "your_account_name"
+hf_user_key = "your_user_key"
+hf_org_key = "your_org_key"
+hf_org_name = "your_org_name"
+
+[[IPFS]]
+
+[[IPFS.BOOTSTRAP]]
+bootstrap = [
+"/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
+"/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ",
+"/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN",
+"/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa",
+"/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb",
+"/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt"
+]
+
+[[IPFS.CLUSTER]]
+clusterName = "cloudkit_storage"
+clusterLocation = "/ip4/167.99.96.231/tcp/9096/p2p/12D3KooWKw9XCkdfnf8CkAseryCgS3VVoGQ6HUAkY91Qc6Fvn4yv",
+secret = ""
+
+[[IPFS.PEERSTORE]]
+peerlist = [
+"/ip4/10.11.0.1/tcp/9096/p2p/12D3KooWDYKMnVLKnP2SmM8umJEEKdhug93QYybmNUEiSe1Kwjmu",
+"/ip4/10.120.0.2/tcp/9096/p2p/12D3KooWDYKMnVLKnP2SmM8umJEEKdhug93QYybmNUEiSe1Kwjmu",
+"/ip4/10.46.0.5/tcp/9096/p2p/12D3KooWDYKMnVLKnP2SmM8umJEEKdhug93QYybmNUEiSe1Kwjmu",
+"/ip4/127.0.0.1/tcp/9096/p2p/12D3KooWDYKMnVLKnP2SmM8umJEEKdhug93QYybmNUEiSe1Kwjmu",
+"/ip4/167.99.96.231/tcp/9096/p2p/12D3KooWDYKMnVLKnP2SmM8umJEEKdhug93QYybmNUEiSe1Kwjmu",
+"/ip4/172.17.0.1/tcp/9096/p2p/12D3KooWDYKMnVLKnP2SmM8umJEEKdhug93QYybmNUEiSe1Kwjmu",
+"/ip4/10.11.0.1/tcp/9096/p2p/12D3KooWFHz8Ze2LyrkkL7q3AqMU6TASaYcW2nmzpYwQodsq2SCV",
+"/ip4/10.120.0.2/tcp/9096/p2p/12D3KooWFHz8Ze2LyrkkL7q3AqMU6TASaYcW2nmzpYwQodsq2SCV",
+"/ip4/10.46.0.5/tcp/9096/p2p/12D3KooWFHz8Ze2LyrkkL7q3AqMU6TASaYcW2nmzpYwQodsq2SCV",
+"/ip4/127.0.0.1/tcp/9096/p2p/12D3KooWFHz8Ze2LyrkkL7q3AqMU6TASaYcW2nmzpYwQodsq2SCV",
+"/ip4/167.99.96.231/tcp/9096/p2p/12D3KooWFHz8Ze2LyrkkL7q3AqMU6TASaYcW2nmzpYwQodsq2SCV",
+"/ip4/172.17.0.1/tcp/9096/p2p/12D3KooWFHz8Ze2LyrkkL7q3AqMU6TASaYcW2nmzpYwQodsq2SCV",
+"/ip4/172.29.29.10/tcp/9096/p2p/12D3KooWNJN6azoq29bY4J3GjE8xQHdYZeQ72Ga6cXnnQ11Jx8fP",
+"/ip4/192.168.0.20/tcp/57468/p2p/12D3KooWNJN6azoq29bY4J3GjE8xQHdYZeQ72Ga6cXnnQ11Jx8fP",
+"/ip4/97.120.209.166/tcp/18100/p2p/12D3KooWJ6mj5yii47Hedtfajahmcq1jhU1CmycvKoKoRFWyY4Ea/p2p-circuit/p2p/12D3KooWNJN6azoq29bY4J3GjE8xQHdYZeQ72Ga6cXnnQ11Jx8fP",
+"/ip4/97.120.209.166/tcp/28143/p2p/12D3KooWECihD8h6TooNoqJAVFDsP5MgUiVcy7XYRRmkFM3yyYME/p2p-circuit/p2p/12D3KooWNJN6azoq29bY4J3GjE8xQHdYZeQ72Ga6cXnnQ11Jx8fP",
+"/ip4/97.120.209.166/tcp/38053/p2p/12D3KooWJ6mj5yii47Hedtfajahmcq1jhU1CmycvKoKoRFWyY4Ea/p2p-circuit/p2p/12D3KooWNJN6azoq29bY4J3GjE8xQHdYZeQ72Ga6cXnnQ11Jx8fP",
+"/ip4/167.99.96.231/tcp/9096/p2p/12D3KooWKw9XCkdfnf8CkAseryCgS3VVoGQ6HUAkY91Qc6Fvn4yv/p2p-circuit/p2p/12D3KooWJ6mj5yii47Hedtfajahmcq1jhU1CmycvKoKoRFWyY4Ea",
+"/ip4/25.18.152.214/tcp/9096/p2p/12D3KooWECihD8h6TooNoqJAVFDsP5MgUiVcy7XYRRmkFM3yyYME/p2p-circuit/p2p/12D3KooWJ6mj5yii47Hedtfajahmcq1jhU1CmycvKoKoRFWyY4Ea",
+"/ip4/97.120.129.70/tcp/15289/p2p/12D3KooWECihD8h6TooNoqJAVFDsP5MgUiVcy7XYRRmkFM3yyYME/p2p-circuit/p2p/12D3KooWJ6mj5yii47Hedtfajahmcq1jhU1CmycvKoKoRFWyY4Ea",
+"/ip4/97.120.209.166/tcp/28143/p2p/12D3KooWECihD8h6TooNoqJAVFDsP5MgUiVcy7XYRRmkFM3yyYME/p2p-circuit/p2p/12D3KooWJ6mj5yii47Hedtfajahmcq1jhU1CmycvKoKoRFWyY4Ea",
+"/ip4/10.11.0.1/tcp/9096/p2p/12D3KooWKw9XCkdfnf8CkAseryCgS3VVoGQ6HUAkY91Qc6Fvn4yv",
+"/ip4/10.120.0.2/tcp/9096/p2p/12D3KooWKw9XCkdfnf8CkAseryCgS3VVoGQ6HUAkY91Qc6Fvn4yv",
+"/ip4/10.46.0.5/tcp/9096/p2p/12D3KooWKw9XCkdfnf8CkAseryCgS3VVoGQ6HUAkY91Qc6Fvn4yv",
+"/ip4/127.0.0.1/tcp/9096/p2p/12D3KooWKw9XCkdfnf8CkAseryCgS3VVoGQ6HUAkY91Qc6Fvn4yv",
+"/ip4/167.99.96.231/tcp/9096/p2p/12D3KooWKw9XCkdfnf8CkAseryCgS3VVoGQ6HUAkY91Qc6Fvn4yv",
+"/ip4/172.17.0.1/tcp/9096/p2p/12D3KooWKw9XCkdfnf8CkAseryCgS3VVoGQ6HUAkY91Qc6Fvn4yv",
+"/ip4/10.11.0.1/tcp/9096/p2p/12D3KooWDYKMnVLKnP2SmM8umJEEKdhug93QYybmNUEiSe1Kwjmu",
+"/ip4/10.120.0.2/tcp/9096/p2p/12D3KooWDYKMnVLKnP2SmM8umJEEKdhug93QYybmNUEiSe1Kwjmu"
+]
+
+[[ORBITDB]]
+ipAddress = "127.0.0.1"
+dbAddress = "/orbitdb/zdpuB31L6gJz49erikZSQT3A1erJbid8oUTBrjLtBwjjXe3R5"
+
+[[S3]]
+s3_bucket =""
+s3_secret_key = ""
+s3_access_key = ""
+s3_endpoint = ""
+s3_host_bucket = ""
+
+[[PINATA]]
+
+[[LIGHTHOUSE]]
+
+[[WEB3STORAGE]]
+
+[[FILEBASE]]
+

config/config.js

@@ -0,0 +1,52 @@
+import os from 'os'
+import fs from 'fs'
+import path from 'path'
+import { parseToml, overrideToml } from '@mwni/toml'
+
+
+export const baseConfig = {
+	master: {
+		port: 8080,
+		tempPath: fs.mkdtempSync(
+			path.join(os.tmpdir(), 'cloudkit-')
+		)
+	}
+}
+
+export function findConfig(){
+	let paths = [
+		'./config.toml',
+		'../config.toml',
+        '../config/config.toml',
+        './config/config.toml'
+	]
+
+	let foundPath = paths.find(
+		p => fs.existsSync(p)
+	)
+
+	return foundPath
+		? path.resolve(foundPath)
+		: undefined
+}
+
+export function loadConfig(configPath, overrides){
+	return overrideToml(
+		baseConfig,
+		parseToml(fs.readFileSync(configPath), 'camelCase'),
+		overrides
+	)
+}
+
+export function requireConfig(opts){
+	const configPath = findConfig() || opts.config
+
+	if(!configPath){
+		console.error(`no config file found`)
+		console.log(`make sure config.toml is in the working directory`)
+		console.log(`or specify path using --config`)
+		process.exit(1)
+	}
+
+	return loadConfig(configPath)
+}

huggingface_scraper/generate_manifest.js

@@ -19,6 +19,9 @@ import hf_faster_whisper_add from './manifests/manifest_hf_faster_whisper.js'
 import hf_transformers_calc from './manifests/manifest_hf_transformers.js'
 import hf_transformers_add from './manifests/manifest_hf_transformers.js'
 import diffusion_calc from './manifests/manifest_diffusion.js'
+import diffusion_add from './manifests/manifest_diffusion.js'
+import dataset_calc from './manifests/manifest_dataset.js'
+import dataset_add from './manifests/manifest_dataset.js'
 import {complete, parse_templates, generate_test} from './utils.js'
 
 export class Generate_Manifest{
@@ -80,6 +83,9 @@ export class Generate_Manifest{
         else if (generation.skill == 'hf_t5'){
             results = hf_t5_add(this)
         }
+        else if (generation.skill == 'dataset'){
+            results = dataset_add(this)
+        }
         else{
             throw("skill is not defined")
         }
@@ -92,7 +98,7 @@ export class Generate_Manifest{
         let metadata = this.metadata
         let hwRequirements = this.hwRequirements
         let results
-        let model_types = ['hf_transformers', 'hf_embed', 'llama_cpp', 'diffusion', 'knn', 'api', 'whisper', 'hf_lm', 'hf_t5']
+        let model_types = ['hf_transformers', 'hf_embed', 'llama_cpp', 'diffusion', 'knn', 'api', 'whisper', 'hf_lm', 'hf_t5', 'dataset']
         // prompt the user for input
         console.log("1. hf_transformers")
         console.log("2. hf_embed")
@@ -103,6 +109,7 @@ export class Generate_Manifest{
         console.log("7. whisper")
         console.log("8. hf_lm")
         console.log("9. hf_t5")
+        console.log("10. dataset")
         // request console input
         let prompt = prompt_sync(({
             history: prompt_sync_history(),
@@ -114,7 +121,7 @@ export class Generate_Manifest{
         // check if the input is valid
         console.log("confirm model type: " + model_type)
         if (!model_types.includes(model_type)){
-            if (parseInt(model_type) > 0 && parseInt(model_type) < 8){
+            if (parseInt(model_type) > 0 && parseInt(model_type) <= 10 ){
                 model_type = model_types[parseInt(model_type) - 1]
             }
             else{
@@ -124,7 +131,7 @@ export class Generate_Manifest{
         }
 
         this.skill = model_type
-    
+
         if (model_type == 'hf_transformers'){
             results = hf_transformers_calc(this)
         }
@@ -165,10 +172,19 @@ export class Generate_Manifest{
             results = custom_calc(this)
         }
 
+        if (model_type == 'dataset'){
+            results = dataset_calc(this)
+        }
+
         try{
            let test = generate_test(results)
         }
         catch(err){
+            console.log("Error in testing the manifest ")
+            console.log(model_type)
+            console.log("results")
+            console.log(results)
+            console.log("error")
             console.log(err)
 //            throw("Error in testing the manifest " +  model_type)
         }

huggingface_scraper/generator.js

@@ -10,7 +10,7 @@ import generate_hf_t5 from './modeldata/generate_hf_t5.json' assert { type: 'jso
 import generate_hf_diffusion from './modeldata/generate_diffusion.json' assert { type: 'json' };
 import generate_api from './modeldata/generate_api.json' assert { type: 'json' };
 import generate_knn from './modeldata/generate_knn.json' assert { type: 'json' };
-
+import generate_dataset from './modeldata/generate_dataset.json' assert { type: 'json' };
 import { setFlagsFromString } from 'v8';
 
 export function template_llama_cpp(local_model_path, collection_path){
@@ -231,6 +231,30 @@ export function template_api(local_model_path, collection_path){
     return results
 }
 
+
+export function template_dataset(local_model_path, collection_path){
+    let results = []
+    let generate
+    let models = Object.keys(generate_api)
+    let local = local_model_path
+    for (var model in models){
+        let this_model = models[model]
+        generate = {}
+        generate.modelName = this_model
+        generate.skill = "dataset"
+        generate.format = generate_template[this_model].format
+        generate.id = this_model 
+        generate.samples = generate_template[this_model].samples
+        generate.size = generate_template[this_model].size
+        generate.location = "huggingface"
+        generate.units = "MB"
+        generate.destPath = local + "/" + this_model 
+        results.push(generate)
+    }
+    return results
+}
+
+
 export class Generator{
     constructor(local_model_path, collection_path){
         this.llama_cpp = {}
@@ -243,6 +267,7 @@ export class Generator{
         this.hf_lm = {}
         this.hf_t5 = {}
         this.hf_diffusion = {}
+        this.dataset = {}
         this.local_model_path = local_model_path
         this.collection_path = collection_path
     }
@@ -258,6 +283,7 @@ export class Generator{
         this.hf_diffusion = template_hf_diffusion(this.local_model_path, this.collection_path)
         this.api = template_api(this.local_model_path, this.collection_path)
         this.knn = template_knn(this.local_model_path, this.collection_path)
+        this.dataset = template_dataset(this.local_model_path, this.collection_path)
         return this 
     }
 }

huggingface_scraper/ipfs.js

@@ -14,7 +14,6 @@ export class ipfsClusterCtl {
         return this
     }
 
-
     check_collection(collection){
         let status = {}
         let collection_keys = Object.keys(collection)