add back comments

Lightning-AI · Mar 11, 2024 · 25d4654 · 25d4654
1 parent a7ab349
commit 25d4654
Showing 1 changed file with 13 additions and 13 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -27,19 +27,19 @@ test = [
     "protobuf",
 ]
 all = [
-    "jsonargparse[signatures]",
-    "bitsandbytes==0.41.0",
-    "scipy",
-    "sentencepiece",
-    "tokenizers",
-    "datasets",
-    "requests",
-    "litdata",
-    "zstandard",
-    "pandas",
-    "pyarrow",
-    "tensorboard",
-    "torchmetrics",
+    "jsonargparse[signatures]",  # CLI
+    "bitsandbytes==0.41.0",      # quantization
+    "scipy",                     # required by bitsandbytes
+    "sentencepiece",             # llama-based models
+    "tokenizers",                # pythia, falcon, redpajama
+    "datasets",                  # eval
+    "requests",                  # litgpt.data
+    "litdata",                   # litgpt.data
+    "zstandard",                 # litgpt.data.prepare_slimpajama.py
+    "pandas",                    # litgpt.data.prepare_starcoder.py
+    "pyarrow",                   # litgpt.data.prepare_starcoder.py
+    "tensorboard",               # litgpt.pretrain
+    "torchmetrics",              # litgpt.pretrain
     "lm_eval @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@115206dc89dad67b8b",
 ]