dynamically determine project root for path to custom normalizer (#48)

sillsdev · Oct 24, 2023 · d7f9d69 · d7f9d69
1 parent b6a6ea1
commit d7f9d69
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/machine/translation/huggingface/hugging_face_nmt_model_trainer.py b/machine/translation/huggingface/hugging_face_nmt_model_trainer.py
@@ -201,7 +201,8 @@ def add_tokens(tokenizer: Any, missing_tokens: List[str]) -> Any:
                 )
             else:
                 norm_tok = PreTrainedTokenizerFast.from_pretrained(
-                    "./machine/translation/huggingface/custom_normalizer", use_fast=True
+                    str(Path(os.path.dirname(os.path.abspath(__file__))) / "custom_normalizer"),
+                    use_fast=True,
                 )
                 # using unofficially supported behavior to set the normalizer
                 tokenizer.backend_tokenizer.normalizer = norm_tok.backend_tokenizer.normalizer  # type: ignore