From 449eb29b111324090fa7066e0b26e9166806b02e Mon Sep 17 00:00:00 2001 From: Sebastian Raschka Date: Mon, 1 Apr 2024 15:44:48 -0500 Subject: [PATCH] Reduce microbatch size (#1212) --- litgpt/finetune/adapter.py | 4 ++-- litgpt/finetune/adapter_v2.py | 4 ++-- litgpt/finetune/full.py | 2 +- litgpt/finetune/lora.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/litgpt/finetune/adapter.py b/litgpt/finetune/adapter.py index 88fd4ecc26..304ea6bd3a 100644 --- a/litgpt/finetune/adapter.py +++ b/litgpt/finetune/adapter.py @@ -46,8 +46,8 @@ def setup( train: TrainArgs = TrainArgs( save_interval=1000, log_interval=1, - global_batch_size=128, - micro_batch_size=4, + global_batch_size=16, + micro_batch_size=1, lr_warmup_steps=100, epochs=5, learning_rate=1e-3, diff --git a/litgpt/finetune/adapter_v2.py b/litgpt/finetune/adapter_v2.py index 97d0e51f16..d925f99ce1 100644 --- a/litgpt/finetune/adapter_v2.py +++ b/litgpt/finetune/adapter_v2.py @@ -46,8 +46,8 @@ def setup( train: TrainArgs = TrainArgs( save_interval=1000, log_interval=1, - global_batch_size=128, - micro_batch_size=4, + global_batch_size=16, + micro_batch_size=1, lr_warmup_steps=100, epochs=5, learning_rate=1e-3, diff --git a/litgpt/finetune/full.py b/litgpt/finetune/full.py index 38aa1ae466..fdcd6bff1e 100644 --- a/litgpt/finetune/full.py +++ b/litgpt/finetune/full.py @@ -44,7 +44,7 @@ def setup( train: TrainArgs = TrainArgs( save_interval=1000, log_interval=1, - global_batch_size=64, + global_batch_size=16, micro_batch_size=1, lr_warmup_steps=100, epochs=5, diff --git a/litgpt/finetune/lora.py b/litgpt/finetune/lora.py index ce8b7764bd..25ae0df839 100644 --- a/litgpt/finetune/lora.py +++ b/litgpt/finetune/lora.py @@ -56,8 +56,8 @@ def setup( train: TrainArgs = TrainArgs( save_interval=1000, log_interval=1, - global_batch_size=128, - micro_batch_size=4, + global_batch_size=16, + micro_batch_size=1, lr_warmup_steps=100, epochs=5, learning_rate=3e-4,