Fixing tokenizers dep.

wonkyDD · Jun 22, 2023 · 9ea220f · 9ea220f
1 parent 86e4cbb
commit 9ea220f
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -8,6 +8,8 @@ repos:
         name: "Rust (clippy)"
         args:
           [
+            "--tests",
+            "--examples",
             "--",
             "-Dwarnings",
           ]
diff --git a/Cargo.toml b/Cargo.toml
@@ -26,7 +26,7 @@ candle-kernels = { path = "kernels", optional = true }
 anyhow = "1"
 clap = { version = "4.2.4", features = ["derive"] }
 rand = "0.8.5"
-tokenizers = "0.13.3"
+tokenizers = { version = "0.13.3", default-features=false, features=["onig"] }
 
 [features]
 default = []