Add the import script for the T5 tokenizer. (huggingface#2399)

wonkyDD · Aug 5, 2024 · 59bbc0d · 59bbc0d
1 parent dfdce2b
commit 59bbc0d
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/candle-examples/examples/flux/t5_tokenizer.py b/candle-examples/examples/flux/t5_tokenizer.py
@@ -0,0 +1,6 @@
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+BASE_MODEL = "google/t5-v1_1-xxl"
+tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
+# The tokenizer will be saved in /tmp/tokenizer/tokenizer.json
+tokenizer.save_pretrained("/tmp/tokenizer/")