directory reorg

vanshigupta04 · Dec 11, 2023 · 75ee1dc · 75ee1dc
1 parent 6398f20
commit 75ee1dc
Show file tree

Hide file tree

Showing 13 changed files with 4,827 additions and 1,677 deletions.
diff --git a/notebooks/data/finetune_data.txt → data/finetune_data.txt b/notebooks/data/finetune_data.txt → data/finetune_data.txt
diff --git a/notebooks/data/friends_dataset.jsonl → data/friends_dataset.jsonl b/notebooks/data/friends_dataset.jsonl → data/friends_dataset.jsonl
diff --git a/.../notebook/exp1/data/finetuning_data.jsonl → data/old_data/finetuning_data.jsonl b/.../notebook/exp1/data/finetuning_data.jsonl → data/old_data/finetuning_data.jsonl
diff --git a/...ng/llamacpp/exp1/data/friends_dataset.txt → data/old_data/friends_dataset.txt b/...ng/llamacpp/exp1/data/friends_dataset.txt → data/old_data/friends_dataset.txt
diff --git a/notebooks/Buzfeed_friends_mcq.ipynb → data_processing/Buzfeed_friends_mcq.ipynb b/notebooks/Buzfeed_friends_mcq.ipynb → data_processing/Buzfeed_friends_mcq.ipynb
diff --git a/notebooks/fetch_all_data.ipynb → data_processing/fetch_all_data.ipynb b/notebooks/fetch_all_data.ipynb → data_processing/fetch_all_data.ipynb
diff --git a/notebooks/generate_finetuning_data.ipynb → ...processing/generate_finetuning_data.ipynb b/notebooks/generate_finetuning_data.ipynb → ...processing/generate_finetuning_data.ipynb
diff --git a/finetuning/auth_token.txt b/finetuning/auth_token.txt
@@ -0,0 +1 @@
+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
diff --git a/finetuning/llamacpp/exp1/finetune.log b/finetuning/llamacpp/exp1/finetune.log
diff --git a/finetuning/llamacpp/exp1/run_finetuning.sh → finetuning/lora.sh b/finetuning/llamacpp/exp1/run_finetuning.sh → finetuning/lora.sh
@@ -1,31 +1,25 @@
-#!/bin/bash -l
-#SBATCH -G 4
-#SBATCH --output=finetune.log
-#SBATCH --time=7-00:00:00
-#SBATCH --mem=512G
-keep-job 30
-datadir=/common/home/$USER/Persona-LLM-Chatbot-For-QA/notebooks/data/friends_dataset.txt
+datadir=/common/home/$USER/Persona-LLM-Chatbot-For-QA/finetuning/llamacpp/exp2/data/finetune_data.txt
 model_path=/freespace/local/$USER/llama.cpp/models/llama-2-7b-chat.Q4_K_M.gguf
-new_model_path=llama2-7b-chat-lora.gguf
-cd /freespace/local/$USER/llama.cpp/
-./finetune \
+new_model_path=lama2-7b-chat-lora.gguf
+/freespace/local/$USER/llama.cpp/finetune \
   --model-base $model_path \
   --train-data $datadir \
   --lora-out $new_model_path \
   --save-every 0 \
   --threads 14 \
-  --ctx 256 \
+  --ctx 4096 \
   --rope-freq-base 10000 \
   --rope-freq-scale 1.0 \
   --batch 1 \
   --grad-acc 1 \
-  --adam-iter 256 \
+  --adam-iter 3099 \
   --adam-alpha 0.001 \
-  --lora-r 4 \
-  --lora-alpha 4 \
+  --lora-r 64 \
+  --lora-alpha 128 \
   --use-checkpointing \
   --use-flash \
   --sample-start "\n" \
   --escape \
   --include-sample-start \
-  --seed 1
+  --seed 2 \
+  -ngl 35 > finetune.log 2>&1