From b0935b6bd6bdd373d5cdc8138619356e61497438 Mon Sep 17 00:00:00 2001 From: Chen Cui Date: Mon, 25 Nov 2024 15:15:25 -0800 Subject: [PATCH] error handling Signed-off-by: Chen Cui --- nemo/lightning/resume.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/nemo/lightning/resume.py b/nemo/lightning/resume.py index 0101ae05da91..375bcfc256fd 100644 --- a/nemo/lightning/resume.py +++ b/nemo/lightning/resume.py @@ -40,11 +40,19 @@ def _try_restore_tokenizer(model, ckpt_path): from nemo.collections.common.tokenizers import TokenizerSpec from nemo.lightning.io import load_context - tokenizer = load_context(ckpt_path, "model.tokenizer") + try: + tokenizer = load_context(ckpt_path, "model.tokenizer") + except ValueError as e: + logging.warning(f"Encountered error while trying to restore tokenizer. Tokenizer is not restored. " + f"Original error: {e}") + return model + if isinstance(tokenizer, TokenizerSpec): - # Ignore if the ckpt doesn't have a tokenizer. type(tokenizer)==TrainerContext in that case. model.tokenizer = tokenizer model.__io__.tokenizer = tokenizer.__io__ + else: + # Ignore if the ckpt doesn't have a tokenizer. type(tokenizer)==TrainerContext in this case. + logging.warning("Checkpoint does not have model.tokenizer field. Tokenizer is not restored.") return model