From 563aae0eea8da318ebbfc9f6f595d907fed7ebb2 Mon Sep 17 00:00:00 2001
From: abhishekkrthakur <abhishek4@gmail.com>
Date: Mon, 4 Nov 2024 15:09:03 +0100
Subject: [PATCH] guanaco smollm2

---
 configs/llm_finetuning/smollm2_guanaco.yml | 34 ++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 configs/llm_finetuning/smollm2_guanaco.yml

diff --git a/configs/llm_finetuning/smollm2_guanaco.yml b/configs/llm_finetuning/smollm2_guanaco.yml
new file mode 100644
index 0000000000..ca0e5302b2
--- /dev/null
+++ b/configs/llm_finetuning/smollm2_guanaco.yml
@@ -0,0 +1,34 @@
+task: llm-sft
+base_model: HuggingFaceTB/SmolLM2-135M-Instruct
+project_name: autotrain-smollm2-135m-finetune-guanaco
+log: tensorboard
+backend: local
+
+data:
+  path: timdettmers/openassistant-guanaco
+  train_split: train
+  valid_split: null
+  chat_template: null
+  column_mapping:
+    text_column: text
+
+params:
+  block_size: 1024
+  model_max_length: 2048
+  epochs: 1
+  batch_size: 1
+  lr: 1e-5
+  peft: true
+  quantization: int4
+  target_modules: all-linear
+  padding: right
+  optimizer: paged_adamw_8bit
+  scheduler: linear
+  gradient_accumulation: 8
+  mixed_precision: bf16
+  merge_adapter: true
+
+hub:
+  username: ${HF_USERNAME}
+  token: ${HF_TOKEN}
+  push_to_hub: true
\ No newline at end of file