avoid EOS turning into bytes

microsoft · Jun 24, 2024 · 8c2c035 · 8c2c035
1 parent 7246614
commit 8c2c035
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/py/llguidance/rust/py.rs b/py/llguidance/rust/py.rs
@@ -147,6 +147,12 @@ impl LLTokenizer {
             .getattr("bos_token_id")?
             .extract::<Option<u32>>()?;
 
+        // we want decode_bytes([EOS]) etc to be empty
+        tokens[tok_eos as usize] = vec![];
+        if let Some(t) = tok_bos {
+            tokens[t as usize] = vec![];
+        }
+
         let info = TokRxInfo {
             vocab_size: tokens.len() as u32,
             tok_eos,