detect if tokenizer is not loaded and adjust (#1012)

NVIDIA · Nov 20, 2024 · 515da0d · 515da0d
2 parents e28aeca + ead3cb0
commit 515da0d
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/garak/generators/huggingface.py b/garak/generators/huggingface.py
@@ -80,6 +80,13 @@ def _load_client(self):
 
         pipeline_kwargs = self._gather_hf_params(hf_constructor=pipeline)
         self.generator = pipeline("text-generation", **pipeline_kwargs)
+        if self.generator.tokenizer is None:
+            # account for possible model without a stored tokenizer
+            from transformers import AutoTokenizer
+
+            self.generator.tokenizer = AutoTokenizer.from_pretrained(
+                pipeline_kwargs["model"]
+            )
         if not hasattr(self, "deprefix_prompt"):
             self.deprefix_prompt = self.name in models_to_deprefix
         if _config.loaded: