fix for requeueing code and change minhash default

swiss-ai · May 6, 2024 · 22c739e · 22c739e
1 parent d56d3c5
commit 22c739e
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/src/datatrove/executor/slurm.py b/src/datatrove/executor/slurm.py
@@ -25,7 +25,7 @@
 def requeue_handler(signum, _frame):
     signame = signal.Signals(signum).name
     logger.warning(f"Received signal {signum} ({signame}). Requeueing and exiting...")
-    subprocess.run(["scontrol", "requeue", "${SLURM_JOB_ID}"])
+    subprocess.run(["scontrol", "requeue", os.environ.get("SLURM_JOB_ID")])
     sys.exit(15)
 
 

diff --git a/src/datatrove/pipeline/dedup/minhash.py b/src/datatrove/pipeline/dedup/minhash.py
@@ -50,7 +50,7 @@ class MinhashConfig:
     num_buckets: int = 14
     hashes_per_bucket: int = 8
 
-    use_64bit_hashes: bool = False
+    use_64bit_hashes: bool = True
     seed: int = 1
 
     norm_config: TextNormConfig = field(default_factory=TextNormConfig)