Skip to content
This repository has been archived by the owner on May 14, 2024. It is now read-only.

Commit

Permalink
Created using Colaboratory
Browse files Browse the repository at this point in the history
  • Loading branch information
@Linaqruf
Linaqruf committed Dec 4, 2022
1 parent 519ea75 commit 487af64
Showing 1 changed file with 70 additions and 55 deletions.
125 changes: 70 additions & 55 deletions kohya-trainer-deepdanbooru.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@
" \"diffusers_fine_tuning_v2\", \\\n",
" \"diffusers_fine_tuning_v3\", \\\n",
" \"diffusers_fine_tuning_v4\"]\n",
"version = \"diffusers_fine_tuning_v4\" #@param [\"diffusers_fine_tuning_v1\",\"diffusers_fine_tuning_v2\",\"diffusers_fine_tuning_v3\",\"diffusers_fine_tuning_v4\"]\n",
"version = \"diffusers_fine_tuning_v3\" #@param [\"diffusers_fine_tuning_v1\",\"diffusers_fine_tuning_v2\",\"diffusers_fine_tuning_v3\",\"diffusers_fine_tuning_v4\"]\n",
"\n",
"customVersion.append((versionDir[versionList.index(version)]))\n",
"\n",
Expand Down Expand Up @@ -249,6 +249,40 @@
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"#@title Datasets cleaner\n",
"#@markdown This will delete unnecessary file and unsupported media like `.bin`, `.mp4`, `.webm`, and `.gif`\n",
"import os\n",
"\n",
"dir_name = \"/content/kohya-trainer/train_data\" #@param {'type' : 'string'}\n",
"test = os.listdir(dir_name)\n",
"\n",
"for item in test:\n",
" if item.endswith(\".mp4\"):\n",
" os.remove(os.path.join(dir_name, item))\n",
"\n",
"for item in test:\n",
" if item.endswith(\".webm\"):\n",
" os.remove(os.path.join(dir_name, item))\n",
"\n",
"for item in test:\n",
" if item.endswith(\".gif\"):\n",
" os.remove(os.path.join(dir_name, item))\n",
" \n",
"for item in test:\n",
" if item.endswith(\".webp\"):\n",
" os.remove(os.path.join(dir_name, item))\n",
"\n"
],
"metadata": {
"cellView": "form",
"id": "Jz2emq6vWnPu"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
Expand All @@ -273,7 +307,7 @@
" !git pull\n",
" \n",
"else:\n",
" !git clone https://github.com/KichangKim/DeepDanbooru\n",
" !git clone https://github.com/KichangKim/DeepDanbooru deepdanbooru\n",
"\n",
"%cd /content/kohya-trainer/deepdanbooru\n",
"!pip install -r requirements.txt\n",
Expand All @@ -299,7 +333,7 @@
"else:\n",
" !mkdir deepdanbooruv3\n",
"\n",
"if os.path.isfile('/content/kohya-trainer/deepdanbooru/deepdanbooruv3.zip'):\n",
"if os.path.isfile('/content/kohya-trainer/deepdanbooru/deepdanbooruv3/deepdanbooruv3.zip'):\n",
" pass\n",
"else:\n",
" !wget -c https://github.com/KichangKim/deepdanbooru/releases/download/v3-20211112-sgd-e28/deepdanbooru-v3-20211112-sgd-e28.zip -O deepdanbooruv3.zip\n",
Expand All @@ -326,56 +360,36 @@
" --save-txt"
],
"metadata": {
"id": "hibZK5NPTjZQ",
"cellView": "form"
"cellView": "form",
"id": "NwxPfDeI3h2_"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"#@title Datasets cleaner\n",
"#@markdown This will delete unnecessary file and unsupported media like `.bin`, `.mp4`, `.webm`, and `.gif`\n",
"import os\n",
"\n",
"dir_name = \"/content/kohya-trainer/train_data\" #@param {'type' : 'string'}\n",
"test = os.listdir(dir_name)\n",
"\n",
"for item in test:\n",
" if item.endswith(\".mp4\"):\n",
" os.remove(os.path.join(dir_name, item))\n",
"\n",
"for item in test:\n",
" if item.endswith(\".webm\"):\n",
" os.remove(os.path.join(dir_name, item))\n",
"\n",
"for item in test:\n",
" if item.endswith(\".gif\"):\n",
" os.remove(os.path.join(dir_name, item))\n",
" \n",
"for item in test:\n",
" if item.endswith(\".webp\"):\n",
" os.remove(os.path.join(dir_name, item))\n",
"\n"
"#@title Create Metadata.json\n",
"%cd /content/kohya-trainer\n",
"!python merge_dd_tags_to_metadata.py train_data meta_cap_dd.json"
],
"metadata": {
"cellView": "form",
"id": "Jz2emq6vWnPu"
"id": "hz2Cmlf2ay9w",
"cellView": "form"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"#@title Create Metadata.json\n",
"#@title Tag cleaner\n",
"%cd /content/kohya-trainer\n",
"!python merge_dd_tags_to_metadata.py train_data meta_cap_dd.json"
"!python clean_captions_and_tags.py train_data meta_cap_dd.json meta_clean.json"
],
"metadata": {
"id": "hz2Cmlf2ay9w",
"cellView": "form"
"cellView": "form",
"id": "Isog9VbN5Le3"
},
"execution_count": null,
"outputs": []
Expand Down Expand Up @@ -421,7 +435,7 @@
" \"Stable-Diffusion-v1-4\", \\\n",
" \"Stable-Diffusion-v1-5-pruned-emaonly\" \\\n",
" \"Waifu-Diffusion-v1-3-fp32\"]\n",
"modelName = \"Anything-V3.0-pruned\" #@param [\"\", \"Animefull-final-pruned\", \"Animesfw-final-pruned\", \"Anything-V3.0-pruned-fp16\", \"Anything-V3.0-pruned-fp32\", \"Anything-V3.0-pruned\", \"Stable-Diffusion-v1-4\", \"Stable-Diffusion-v1-5-pruned-emaonly\", \"Waifu-Diffusion-v1-3-fp32\"]\n",
"modelName = \"Animefull-final-pruned\" #@param [\"\", \"Animefull-final-pruned\", \"Animesfw-final-pruned\", \"Anything-V3.0-pruned-fp16\", \"Anything-V3.0-pruned-fp32\", \"Anything-V3.0-pruned\", \"Stable-Diffusion-v1-4\", \"Stable-Diffusion-v1-5-pruned-emaonly\", \"Waifu-Diffusion-v1-3-fp32\"]\n",
"\n",
"#@markdown ### Custom model\n",
"#@markdown The model URL should be a direct download link.\n",
Expand Down Expand Up @@ -525,16 +539,16 @@
"source": [
"#@title Training begin\n",
"num_cpu_threads_per_process = 8 #@param {'type':'integer'}\n",
"model_path =\"/content/kohya-trainer/checkpoint/Anything-V3.0-pruned.ckpt\" #@param {'type':'string'}\n",
"model_path =\"/content/herigaru/herigaru5k-pruned.ckpt\" #@param {'type':'string'}\n",
"output_dir =\"/content/kohya-trainer/fine_tuned\" #@param {'type':'string'}\n",
"train_batch_size = 1 #@param {type: \"slider\", min: 1, max: 10}\n",
"learning_rate =\"2e-6\" #@param {'type':'string'}\n",
"max_token_length = \"225\" #@param [\"150\", \"225\"] {allow-input: false}\n",
"clip_skip = 2 #@param {type: \"slider\", min: 1, max: 10}\n",
"mixed_precision = \"fp16\" #@param [\"fp16\", \"bf16\"] {allow-input: false}\n",
"max_train_steps = 10000 #@param {'type':'integer'}\n",
"max_train_steps = 5000 #@param {'type':'integer'}\n",
"# save_precision = \"fp16\" #@param [\"float\", \"fp16\", \"bf16\"] {allow-input: false}\n",
"save_every_n_epochs = 50 #@param {'type':'integer'}\n",
"save_every_n_epochs = 100 #@param {'type':'integer'}\n",
"gradient_accumulation_steps = 1 #@param {type: \"slider\", min: 1, max: 10}\n",
"dataset_repeats = 1 #@param {'type':'integer'}\n",
" \n",
Expand All @@ -545,6 +559,7 @@
" --train_data_dir=train_data \\\n",
" --output_dir={output_dir} \\\n",
" --shuffle_caption \\\n",
" --logging_dir=logs \\\n",
" --train_batch_size={train_batch_size} \\\n",
" --learning_rate={learning_rate} \\\n",
" --max_token_length={max_token_length} \\\n",
Expand Down Expand Up @@ -629,8 +644,8 @@
"\n",
"prune = True #@param {'type':'boolean'}\n",
"\n",
"model_src = \"/content/kohya-trainer/fine_tuned/last.ckpt\" #@param {'type' : 'string'}\n",
"model_dst = \"/content/kohya-trainer/fine_tuned/last-pruned.ckpt\" #@param {'type' : 'string'}\n",
"model_src = \"/content/herigaru/herigaru5k.ckpt\" #@param {'type' : 'string'}\n",
"model_dst = \"/content/herigaru/herigaru5k-pruned.ckpt\" #@param {'type' : 'string'}\n",
"\n",
"if prune == True:\n",
" import os\n",
Expand All @@ -640,7 +655,7 @@
" !wget https://raw.githubusercontent.com/lopho/stable-diffusion-prune/main/prune.py\n",
"\n",
"\n",
"!python3 prune.py -p {model_src} {model_dst}\n"
"!python3 prune.py {model_src} {model_dst}\n"
],
"metadata": {
"id": "LUOG7BzQVLKp",
Expand Down Expand Up @@ -748,11 +763,11 @@
"#@title Clone Model\n",
"\n",
"#@markdown Opt-out this cell when run all\n",
"opt_out= True #@param {'type':'boolean'}\n",
"opt_out= False #@param {'type':'boolean'}\n",
"\n",
"if opt_out == False:\n",
" %cd /content\n",
" Repository_url = \"https://huggingface.co/Linaqruf/experimental\" #@param {'type': 'string'}\n",
" Repository_url = \"https://huggingface.co/Linaqruf/herigaru\" #@param {'type': 'string'}\n",
" !git clone {Repository_url}\n",
"else:\n",
" pass\n"
Expand All @@ -769,23 +784,23 @@
"source": [
"#@title Commit to Huggingface\n",
"#@markdown Opt-out this cell when run all\n",
"opt_out= True #@param {'type':'boolean'}\n",
"opt_out= False #@param {'type':'boolean'}\n",
"\n",
"if opt_out == False:\n",
" %cd /content\n",
" #@markdown Go to your model path\n",
" model_path= \"alphanime-diffusion\" #@param {'type': 'string'}\n",
" model_path= \"herigaru\" #@param {'type': 'string'}\n",
"\n",
" #@markdown Your path look like /content/**model_path**\n",
" #@markdown ___\n",
" #@markdown #Git Commit\n",
"\n",
" #@markdown Set **git commit identity**\n",
"\n",
" email= \"your-email\" #@param {'type': 'string'}\n",
" name= \"your-name\" #@param {'type': 'string'}\n",
" email= \"[email protected]\" #@param {'type': 'string'}\n",
" name= \"Linaqruf\" #@param {'type': 'string'}\n",
" #@markdown Set **commit message**\n",
" commit_m= \"this is commit message\" #@param {'type': 'string'}\n",
" commit_m= \"upload newly finetuned model 5k\" #@param {'type': 'string'}\n",
"\n",
" %cd \"/content/{model_path}\"\n",
" !git lfs install\n",
Expand Down Expand Up @@ -843,11 +858,11 @@
"source": [
"#@title Clone Dataset\n",
"#@markdown Opt-out this cell when run all\n",
"opt_out= True #@param {'type':'boolean'}\n",
"opt_out= False #@param {'type':'boolean'}\n",
"\n",
"if opt_out == False:\n",
" %cd /content\n",
" Repository_url = \"https://huggingface.co/datasets/Linaqruf/alphanime-diffusion-tag\" #@param {'type': 'string'}\n",
" Repository_url = \"https://huggingface.co/datasets/Linaqruf/herigaru-tag\" #@param {'type': 'string'}\n",
" !git clone {Repository_url}\n",
"else:\n",
" pass\n"
Expand All @@ -865,23 +880,23 @@
"#@title Commit to Huggingface\n",
"#@markdown Opt-out this cell when run all\n",
"\n",
"opt_out= True #@param {'type':'boolean'}\n",
"opt_out= False #@param {'type':'boolean'}\n",
"\n",
"if opt_out == False:\n",
" %cd /content\n",
" #@markdown Go to your model path\n",
" dataset_path= \"alphanime-diffusion-tag\" #@param {'type': 'string'}\n",
" dataset_path= \"herigaru-tag\" #@param {'type': 'string'}\n",
"\n",
" #@markdown Your path look like /content/**dataset_path**\n",
" #@markdown ___\n",
" #@markdown #Git Commit\n",
"\n",
" #@markdown Set **git commit identity**\n",
"\n",
" email= \"your-email\" #@param {'type': 'string'}\n",
" name= \"your-name\" #@param {'type': 'string'}\n",
" email= \"[email protected]\" #@param {'type': 'string'}\n",
" name= \"Linaqruf\" #@param {'type': 'string'}\n",
" #@markdown Set **commit message**\n",
" commit_m= \"this is commit message\" #@param {'type': 'string'}\n",
" commit_m= \"upload 5k new datasets\" #@param {'type': 'string'}\n",
"\n",
" %cd \"/content/{dataset_path}\"\n",
" !git lfs install\n",
Expand Down

0 comments on commit 487af64

Please sign in to comment.