Falcon3: fix test_tokenizer.py

Lightning-AI · Dec 22, 2024 · d0571e4 · d0571e4
1 parent d849632
commit d0571e4
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py
@@ -56,7 +56,7 @@ def test_tokenizer_against_hf(config):
     else:
         assert ours.vocab_size == config.vocab_size
 
-    if config.name.startswith(("falcon", "stablecode", "Qwen2.5", "QwQ")):
+    if config.name.startswith(("Falcon3", "falcon", "stablecode", "Qwen2.5", "QwQ")):
         # even though their config defines it, it's set as None in HF
         assert isinstance(ours.bos_id, int)
         assert theirs.bos_token_id is None