fix default compression

huggingface · Dec 26, 2024 · 47379fd · 47379fd
1 parent 61d4f86
commit 47379fd
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/src/datatrove/pipeline/writers/huggingface.py b/src/datatrove/pipeline/writers/huggingface.py
@@ -2,7 +2,7 @@
 import random
 import tempfile
 import time
-from typing import Callable
+from typing import Callable, Literal
 
 from huggingface_hub import (
     CommitOperationAdd,
@@ -31,7 +31,7 @@ def __init__(
         private: bool = True,
         local_working_dir: DataFolderLike | None = None,
         output_filename: str = None,
-        compression: str | None = None,
+        compression: Literal["snappy", "gzip", "brotli", "lz4", "zstd"] | None = "snappy",
         adapter: Callable = None,
         cleanup: bool = True,
         expand_metadata: bool = True,

diff --git a/src/datatrove/pipeline/writers/parquet.py b/src/datatrove/pipeline/writers/parquet.py
@@ -14,7 +14,7 @@ def __init__(
         self,
         output_folder: DataFolderLike,
         output_filename: str = None,
-        compression: Literal["snappy", "gzip", "brotli", "lz4", "zstd"] | None = None,
+        compression: Literal["snappy", "gzip", "brotli", "lz4", "zstd"] | None = "snappy",
         adapter: Callable = None,
         batch_size: int = 1000,
         expand_metadata: bool = False,