add local version

huggingface · Nov 27, 2024 · 6412f31 · 6412f31
1 parent 0b5591a
commit 6412f31
Show file tree

Hide file tree

Showing 3 changed files with 378 additions and 414 deletions.
diff --git a/src/datatrove/pipeline/dedup/fast_mh3/Cargo.toml b/src/datatrove/pipeline/dedup/fast_mh3/Cargo.toml
@@ -27,5 +27,5 @@ tokio = { version = "1.33.0", features = ["full"] }
 tokio-retry = "0.3"
 
 [[bin]]
-name = "check"
-path = "src/check.rs"
+name = "local"
+path = "src/local.rs"