bunch of changes

huggingface · Nov 26, 2024 · 78be9e0 · 78be9e0
1 parent 03c1a34
commit 78be9e0
Show file tree

Hide file tree

Showing 2 changed files with 189 additions and 153 deletions.
diff --git a/src/datatrove/pipeline/dedup/fast_mh3/Cargo.toml b/src/datatrove/pipeline/dedup/fast_mh3/Cargo.toml
@@ -4,8 +4,6 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-indicatif = "0.17.7"
-tokio-retry = "0.3"
 # AWS SDK
 aws-config = { version = "1.1.1", features = ["behavior-version-latest"] }
 aws-sdk-s3 = "1.1.1"
@@ -19,9 +17,11 @@ anyhow = "1.0.75"
 # Byte reading/writing
 byteorder = "1.5.0"
 
+# Progress bars
+indicatif = "0.17.7"
+
 # Async runtime and utilities
 tokio = { version = "1.33.0", features = ["full"] }
-futures = "0.3"
 
-# Sorting
-itertools = "0.11.0"
+# Retries
+tokio-retry = "0.3"