From 37cf8421243df51dafc8d70515b5e4f75d1320cd Mon Sep 17 00:00:00 2001 From: Andrei-Aksionov Date: Tue, 7 May 2024 16:04:47 +0300 Subject: [PATCH] Check that all the tensors are materialized. --- tests/test_lora.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_lora.py b/tests/test_lora.py index d283f1cf44..8f4edba90a 100644 --- a/tests/test_lora.py +++ b/tests/test_lora.py @@ -758,3 +758,13 @@ def test_lora_model_fsdp_init(): model = fabric.setup(model) y = model(x) assert y.shape == torch.Size([2, 8, 512]) + + # verify that all the parameters, buffers and other attributes aren't on `meta` device + for m in model.modules(): + for p_name, parameter in m.named_parameters(): + assert not parameter.is_meta, f"Parameter `{p_name}` isn't materialized." + for b_name, buffer in m._buffers.items(): + assert not buffer.is_meta, f"Buffer `{b_name}` isn't materialized." + for attr_name, attr_value in m.__dict__.items(): + if isinstance(attr_value, torch.Tensor): + assert not attr_value.is_meta, f"Attribute `{attr_name}` isn't materialized."