update config hub

config_hub/finetune/falcon-7b/lora.yaml

@@ -113,6 +113,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/falcon-7b/qlora.yaml

@@ -115,6 +115,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/gemma-2b/full.yaml

@@ -84,6 +84,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/gemma-2b/lora.yaml

@@ -114,6 +114,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/gemma-2b/qlora.yaml

@@ -114,6 +114,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/gemma-7b/lora.yaml

@@ -114,6 +114,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/gemma-7b/qlora.yaml

@@ -114,6 +114,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/llama-2-7b/full.yaml

@@ -87,6 +87,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/llama-2-7b/lora.yaml

@@ -113,6 +113,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/llama-2-7b/qlora.yaml

@@ -115,6 +115,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/llama-3-8b/full.yaml

@@ -87,6 +87,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/llama-3-8b/lora.yaml

@@ -113,6 +113,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/llama-3-8b/qlora.yaml

@@ -115,6 +115,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/mistral-7b-v0.2/lora.yaml

@@ -113,6 +113,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/mistral-7b-v0.2/qlora.yaml

@@ -115,6 +115,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/mistral-7b/lora.yaml

@@ -113,6 +113,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/mistral-7b/qlora.yaml

@@ -115,6 +115,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/phi-2/full.yaml

@@ -87,6 +87,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/phi-2/lora.yaml

@@ -114,6 +114,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/phi-2/qlora.yaml

@@ -114,6 +114,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/stablelm-base-alpha-3b/full.yaml

@@ -84,6 +84,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/stablelm-base-alpha-3b/lora.yaml

@@ -113,6 +113,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/stablelm-base-alpha-3b/qlora.yaml

@@ -115,6 +115,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/tiny-llama/full.yaml

@@ -84,6 +84,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/tiny-llama/lora.yaml

@@ -114,6 +114,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/finetune/tiny-llama/qlora.yaml

@@ -114,6 +114,7 @@ seed: 1337
 
 # Optimizer-related arguments
 optimizer:
+
   class_path: torch.optim.AdamW
 
   init_args:

config_hub/pretrain/debug.yaml

@@ -58,18 +58,6 @@ train:
   # Whether to tie the embedding weights with the language modeling head weights. (type: Optional[bool], default: False)
   tie_embeddings:
 
-  #   (type: float, default: 0.0004)
-  learning_rate: 6e-4
-
-  #   (type: float, default: 0.1)
-  weight_decay: 0.1
-
-  #   (type: float, default: 0.9)
-  beta1: 0.9
-
-  #   (type: float, default: 0.95)
-  beta2: 0.95
-
   #   (type: Optional[float], default: 1.0)
   max_norm: 1.0
 
@@ -91,6 +79,24 @@ eval:
   # Whether to evaluate on the validation set at the beginning of the training
   initial_validation: false
 
+# Optimizer-related arguments
+optimizer:
+
+  class_path: torch.optim.AdamW
+
+  init_args:
+
+    #   (type: float, default: 0.001)
+    lr: 6e-4
+
+    #   (type: float, default: 0.01)
+    weight_decay: 0.1
+
+    #   (type: tuple, default: (0.9,0.999))
+    betas:
+      - 0.9
+      - 0.95
+
 # How many devices/GPUs to use. Uses all GPUs by default. (type: Union[int, str], default: auto)
 devices: auto
 

config_hub/pretrain/tinyllama.yaml

@@ -58,18 +58,6 @@ train:
   # Whether to tie the embedding weights with the language modeling head weights. (type: Optional[bool], default: False)
   tie_embeddings:
 
-  #   (type: float, default: 0.0004)
-  learning_rate: 4e-4
-
-  #   (type: float, default: 0.1)
-  weight_decay: 0.1
-
-  #   (type: float, default: 0.9)
-  beta1: 0.9
-
-  #   (type: float, default: 0.95)
-  beta2: 0.95
-
   #   (type: Optional[float], default: 1.0)
   max_norm: 1.0
 
@@ -91,6 +79,24 @@ eval:
   # Whether to evaluate on the validation set at the beginning of the training
   initial_validation: false
 
+# Optimizer-related arguments
+optimizer:
+
+  class_path: torch.optim.AdamW
+
+  init_args:
+
+    #   (type: float, default: 0.001)
+    lr: 4e-4
+
+    #   (type: float, default: 0.01)
+    weight_decay: 0.1
+
+    #   (type: tuple, default: (0.9,0.999))
+    betas:
+      - 0.9
+      - 0.95
+
 # How many devices/GPUs to use. Uses all GPUs by default. (type: Union[int, str], default: auto)
 devices: auto
 

config_hub/pretrain/tinystories.yaml

@@ -74,18 +74,6 @@ train:
   # Whether to tie the embedding weights with the language modeling head weights. (type: Optional[bool], default: False)
   tie_embeddings: true
 
-  #   (type: float, default: 0.0004)
-  learning_rate: 0.0005
-
-  #   (type: float, default: 0.1)
-  weight_decay: 0.1
-
-  #   (type: float, default: 0.9)
-  beta1: 0.9
-
-  #   (type: float, default: 0.95)
-  beta2: 0.95
-
   #   (type: Optional[float], default: 1.0)
   max_norm: 1.0
 
@@ -107,6 +95,24 @@ eval:
   # Whether to evaluate on the validation set at the beginning of the training
   initial_validation: false
 
+# Optimizer-related arguments
+optimizer:
+
+  class_path: torch.optim.AdamW
+
+  init_args:
+
+    #   (type: float, default: 0.001)
+    lr: 0.0005
+
+    #   (type: float, default: 0.01)
+    weight_decay: 0.1
+
+    #   (type: tuple, default: (0.9,0.999))
+    betas:
+      - 0.9
+      - 0.95
+
 # How many devices/GPUs to use. Uses all GPUs by default. (type: Union[int, str], default: auto)
 devices: auto