Add settings to suppress bf16 compile errors in CI on V100 (NVIDIA#8481…

…) (NVIDIA#8482) * Add settings to suppress bf16 compile errors in CI on V100 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Signed-off-by: Abhishree <[email protected]> Co-authored-by: Abhishree Thittenamane <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: Zeeshan Patel <[email protected]>
zpx01 · Mar 8, 2024 · 3221c20 · 3221c20
1 parent c517cd2
commit 3221c20
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/examples/nlp/language_modeling/megatron_gpt_pretraining.py b/examples/nlp/language_modeling/megatron_gpt_pretraining.py
@@ -13,6 +13,8 @@
 # limitations under the License.
 
 
+# To suppress BF16 compile related issue in the CI runs with turing/V100
+import torch._dynamo
 import torch.multiprocessing as mp
 from omegaconf.omegaconf import OmegaConf, open_dict
 
@@ -22,6 +24,8 @@
 from nemo.utils import logging
 from nemo.utils.exp_manager import exp_manager
 
+torch._dynamo.config.suppress_errors = True
+
 mp.set_start_method("spawn", force=True)