update

lvdongyi · Oct 21, 2024 · d46655c · d46655c
1 parent 7ff5a17
commit d46655c
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/paddlenlp/transformers/auto/tokenizer.py b/paddlenlp/transformers/auto/tokenizer.py
@@ -107,13 +107,14 @@
             ("tinybert", "TinyBertTokenizer"),
             ("unified_transformer", "UnifiedTransformerTokenizer"),
             ("unimo", "UNIMOTokenizer"),
-            ("gpt", "GPTChineseTokenizer"),
+            ("gpt", (("GPTTokenizer", "GPTChineseTokenizer"), None)),
             ("gau_alpha", "GAUAlphaTokenizer"),
             ("artist", "ArtistTokenizer"),
             ("chineseclip", "ChineseCLIPTokenizer"),
             ("ernie_vil", "ErnieViLTokenizer"),
             ("glm", "GLMGPT2Tokenizer"),
             ("qwen", "QWenTokenizer"),
+            ("qwen2", "Qwen2Tokenizer"),
             ("yuan", "YuanTokenizer"),
         ]
     )