From aa95635b116023ce113d7975e991530369bdcd48 Mon Sep 17 00:00:00 2001 From: Sebastian Raschka Date: Tue, 21 May 2024 18:21:20 -0400 Subject: [PATCH] Simplify code (#1429) --- litgpt/finetune/adapter.py | 3 +-- litgpt/finetune/adapter_v2.py | 3 +-- litgpt/finetune/lora.py | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/litgpt/finetune/adapter.py b/litgpt/finetune/adapter.py index aa02977fb9..2ec71784e7 100644 --- a/litgpt/finetune/adapter.py +++ b/litgpt/finetune/adapter.py @@ -145,7 +145,6 @@ def main( model = fabric.setup_module(model) - trainable_params = [p for p in model.parameters() if p.requires_grad] if isinstance(fabric.strategy.precision, BitsandbytesPrecision): import bitsandbytes as bnb @@ -153,7 +152,7 @@ def main( else: optimizer_cls = torch.optim.AdamW optimizer = optimizer_cls( - trainable_params, lr=train.learning_rate, weight_decay=train.weight_decay, betas=(train.beta1, train.beta2) + model.parameters(), lr=train.learning_rate, weight_decay=train.weight_decay, betas=(train.beta1, train.beta2) ) optimizer = fabric.setup_optimizers(optimizer) scheduler = get_lr_scheduler(optimizer, warmup_steps=train.lr_warmup_steps, max_steps=lr_max_steps) diff --git a/litgpt/finetune/adapter_v2.py b/litgpt/finetune/adapter_v2.py index 197bcd0bba..86526a58e5 100644 --- a/litgpt/finetune/adapter_v2.py +++ b/litgpt/finetune/adapter_v2.py @@ -145,7 +145,6 @@ def main( model = fabric.setup_module(model) - trainable_params = [p for p in model.parameters() if p.requires_grad] if isinstance(fabric.strategy.precision, BitsandbytesPrecision): import bitsandbytes as bnb @@ -153,7 +152,7 @@ def main( else: optimizer_cls = torch.optim.AdamW optimizer = optimizer_cls( - trainable_params, lr=train.learning_rate, weight_decay=train.weight_decay, betas=(train.beta1, train.beta2) + model.parameters(), lr=train.learning_rate, weight_decay=train.weight_decay, betas=(train.beta1, train.beta2) ) optimizer = fabric.setup_optimizers(optimizer) scheduler = get_lr_scheduler(optimizer, warmup_steps=train.lr_warmup_steps, max_steps=lr_max_steps) diff --git a/litgpt/finetune/lora.py b/litgpt/finetune/lora.py index bed58862bb..a2c3ef07b2 100644 --- a/litgpt/finetune/lora.py +++ b/litgpt/finetune/lora.py @@ -175,7 +175,6 @@ def main( model = fabric.setup_module(model) - trainable_params = [p for p in model.parameters() if p.requires_grad] if isinstance(fabric.strategy.precision, BitsandbytesPrecision): import bitsandbytes as bnb @@ -183,7 +182,7 @@ def main( else: optimizer_cls = torch.optim.AdamW optimizer = optimizer_cls( - trainable_params, lr=train.learning_rate, weight_decay=train.weight_decay, betas=(train.beta1, train.beta2) + model.parameters(), lr=train.learning_rate, weight_decay=train.weight_decay, betas=(train.beta1, train.beta2) ) optimizer = fabric.setup_optimizers(optimizer) scheduler = get_lr_scheduler(optimizer, warmup_steps=train.lr_warmup_steps, max_steps=lr_max_steps)