Add settings to suppress bf16 compile errors in CI on V100 (#8481)

* Add settings to suppress bf16 compile errors in CI on V100 Signed-off-by: Abhishree <[email protected]> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: Abhishree <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
NVIDIA · Feb 22, 2024 · c743c55 · c743c55
1 parent 59aa22d
commit c743c55
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/examples/nlp/language_modeling/megatron_gpt_pretraining.py b/examples/nlp/language_modeling/megatron_gpt_pretraining.py
@@ -13,6 +13,8 @@
 # limitations under the License.
 
 
+# To suppress BF16 compile related issue in the CI runs with turing/V100
+import torch._dynamo
 import torch.multiprocessing as mp
 from omegaconf.omegaconf import OmegaConf, open_dict
 
@@ -22,6 +24,8 @@
 from nemo.utils import logging
 from nemo.utils.exp_manager import exp_manager
 
+torch._dynamo.config.suppress_errors = True
+
 mp.set_start_method("spawn", force=True)