Merge pull request #406 from clarkmcc/upgrade-tokenizers-crate

Updated tokenizers crate to 0.13.4
rustformers · Aug 17, 2023 · 129b84a · 129b84a
2 parents bca64b0 + e982ff8
commit 129b84a
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 6 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/llm-base/Cargo.toml b/crates/llm-base/Cargo.toml
@@ -22,7 +22,7 @@ partial_sort = "0.2.0"
 serde_bytes = "0.11"
 memmap2 = { workspace = true }
 half = "2"
-tokenizers = {version="0.13.3", default-features=false, features=["onig"]}
+tokenizers = {version="0.13.4", default-features=false, features=["onig"]}
 regex = "1.8"
 tracing = { workspace = true }
 

diff --git a/crates/llm-base/src/tokenizer/huggingface.rs b/crates/llm-base/src/tokenizer/huggingface.rs
@@ -22,7 +22,7 @@ impl HuggingFaceTokenizer {
     /// Converts a token index to the token it represents in this tokenizer.
     pub(crate) fn token(&self, idx: usize) -> Vec<u8> {
         self.tokenizer
-            .decode(vec![idx as u32], true)
+            .decode(&[idx as u32], true)
             .expect("Cannot decode token from tokenizer tokenizer.")
             .as_bytes()
             .to_vec()
@@ -67,7 +67,7 @@ impl HuggingFaceTokenizer {
     /// Decode a list `tokens` with this tokenizer.
     pub(crate) fn decode(&self, tokens: Vec<TokenId>, skip_special_tokens: bool) -> Vec<u8> {
         self.tokenizer
-            .decode(tokens, skip_special_tokens)
+            .decode(&tokens, skip_special_tokens)
             .expect("Cannot decode token from tokenizer.")
             .as_bytes()
             .to_vec()