From 0d5a829c7ecc1f8e79d45a5169c04ee0645037f7 Mon Sep 17 00:00:00 2001 From: Abhishree Date: Mon, 22 Jul 2024 16:48:58 -0700 Subject: [PATCH 1/2] Disable nvFuser setup with PyTorch 23.11 and later Signed-off-by: Abhishree --- .../nlp/models/language_modeling/megatron_base_model.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nemo/collections/nlp/models/language_modeling/megatron_base_model.py b/nemo/collections/nlp/models/language_modeling/megatron_base_model.py index 20d532d4764a..cbdc121003b8 100644 --- a/nemo/collections/nlp/models/language_modeling/megatron_base_model.py +++ b/nemo/collections/nlp/models/language_modeling/megatron_base_model.py @@ -399,7 +399,8 @@ def _enable_nvidia_optimizations(self): self.cfg.persist_layer_norm = False # NVFUSER available starting with 21.11 - if NVIDIA_TORCH_MAJOR >= 21 or (NVIDIA_TORCH_MAJOR == 21 and NVIDIA_TORCH_MINOR >= 11): + if (NVIDIA_TORCH_MAJOR >= 21 or (NVIDIA_TORCH_MAJOR == 21 and NVIDIA_TORCH_MINOR >= 11)) and \ + (NVIDIA_TORCH_MAJOR < 23 or (NVIDIA_TORCH_MAJOR == 23 and NVIDIA_TORCH_MINOR < 11)): # NVFUSER torch._C._jit_set_profiling_executor(True) From 75a91d7527435ad64cc476f0f59dc9905b66d459 Mon Sep 17 00:00:00 2001 From: athitten Date: Mon, 22 Jul 2024 23:50:47 +0000 Subject: [PATCH 2/2] Apply isort and black reformatting Signed-off-by: athitten --- .../nlp/models/language_modeling/megatron_base_model.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nemo/collections/nlp/models/language_modeling/megatron_base_model.py b/nemo/collections/nlp/models/language_modeling/megatron_base_model.py index cbdc121003b8..a615dcef5051 100644 --- a/nemo/collections/nlp/models/language_modeling/megatron_base_model.py +++ b/nemo/collections/nlp/models/language_modeling/megatron_base_model.py @@ -399,8 +399,9 @@ def _enable_nvidia_optimizations(self): self.cfg.persist_layer_norm = False # NVFUSER available starting with 21.11 - if (NVIDIA_TORCH_MAJOR >= 21 or (NVIDIA_TORCH_MAJOR == 21 and NVIDIA_TORCH_MINOR >= 11)) and \ - (NVIDIA_TORCH_MAJOR < 23 or (NVIDIA_TORCH_MAJOR == 23 and NVIDIA_TORCH_MINOR < 11)): + if (NVIDIA_TORCH_MAJOR >= 21 or (NVIDIA_TORCH_MAJOR == 21 and NVIDIA_TORCH_MINOR >= 11)) and ( + NVIDIA_TORCH_MAJOR < 23 or (NVIDIA_TORCH_MAJOR == 23 and NVIDIA_TORCH_MINOR < 11) + ): # NVFUSER torch._C._jit_set_profiling_executor(True)