From 5c726b052e2f1a43b4d418dff6e26e2809164251 Mon Sep 17 00:00:00 2001
From: Yu Shi Jie <yshijie1999@gmail.com>
Date: Mon, 2 Dec 2024 21:01:11 -0500
Subject: [PATCH] revert

---
 litgpt/tokenizer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/litgpt/tokenizer.py b/litgpt/tokenizer.py
index ed78ca550d..a81c59aa2d 100644
--- a/litgpt/tokenizer.py
+++ b/litgpt/tokenizer.py
@@ -94,7 +94,7 @@ def check_if_bos_token_used(self, checkpoint_dir: Path) -> bool:
             config = json.load(fp)
         # for LlaMA-3 tokenizer there is no `add_bos_token` at all and `tokenizer_class` is only
         # `PreTrainedTokenizerFast`
-        if checkpoint_dir.stem.startswith(("Meta-Llama-3", "Llama-3", "SmolLM2")):
+        if checkpoint_dir.stem.startswith(("Meta-Llama-3", "Llama-3")):
             return True
         if "add_bos_token" in config:
             return config["add_bos_token"]