[djl-convert] Exclude models in includeTokenTypes (#3435)

deepjavalibrary · Aug 21, 2024 · 99ed366 · 99ed366
1 parent 29dea2a
commit 99ed366
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/extensions/tokenizers/src/main/python/djl_converter/huggingface_converter.py b/extensions/tokenizers/src/main/python/djl_converter/huggingface_converter.py
@@ -137,7 +137,9 @@ def save_rust_model(self, model_info, args: Namespace, temp_dir: str,
             os.makedirs(temp_dir)
 
         tokenizer = AutoTokenizer.from_pretrained(model_id)
-        include_types = config.model_type != "distilbert"
+        include_types = config.model_type not in [
+            "distilbert", "mistral", "qwen2", "gemma2"
+        ]
         hf_pipeline = PipelineHolder(tokenizer, ModelHolder(config))
         try:
             # Save tokenizer.json to temp dir