chore: use tiktoken to calcualte token length

AppFlowy-IO · Nov 15, 2024 · 40ce4a8 · 40ce4a8
1 parent 6402015
commit 40ce4a8
Show file tree

Hide file tree

Showing 4 changed files with 162 additions and 139 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/services/appflowy-collaborate/Cargo.toml b/services/appflowy-collaborate/Cargo.toml
@@ -88,9 +88,8 @@ itertools = "0.12.0"
 validator = "0.16.1"
 rayon.workspace = true
 tiktoken-rs = "0.6.0"
-unicode-segmentation = "1.9.0"
-unicode-normalization = "0.1.24"
 
 [dev-dependencies]
 rand = "0.8.5"
 workspace-template.workspace = true
+unicode-normalization = "0.1.24"