Check tokenizer is present.

Signed-off-by: Alexandros Koumparoulis <[email protected]>
NVIDIA · Jan 30, 2024 · 01860ff · 01860ff
1 parent 7d82bd4
commit 01860ff
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/scripts/nlp_language_modeling/convert_hf_mixtral_to_nemo.py b/scripts/nlp_language_modeling/convert_hf_mixtral_to_nemo.py
@@ -81,6 +81,9 @@ def load_model(cls, checkpoint, strict, **kwargs):
 
             # register the artifacts
             cfg = checkpoint[cls.CHECKPOINT_HYPER_PARAMS_KEY]
+            assert os.path.exists(
+                cfg.tokenizer.model
+            ), f"Expected cfg.tokenizer.model {cfg.tokenizer.model} to be present"
             if cfg.tokenizer.model is not None:
                 model.register_artifact("tokenizer.tokenizer_model", cfg.tokenizer.model)
             if cfg.tokenizer.vocab_file is not None: