pyproject.toml (#1074)

.github/workflows/cpu-tests.yml

@@ -41,13 +41,11 @@ jobs:
         python-version: ${{ matrix.python-version }}
         cache: 'pip'
         cache-dependency-path: |
-          requirements.txt
-          requirements-all.txt
-          setup.py
+          pyproject.toml
 
     - name: Install minimal dependencies
       run: |
-        pip install -r requirements.txt
+        pip install .
         pip list
         # make sure all modules are still importable with only the minimal dependencies available
         modules=$(

pyproject.toml

@@ -0,0 +1,64 @@
+[project]
+name = "litgpt"
+version = "0.1.0"
+description = "Open source large language model implementation"
+authors = [
+    { name = "Lightning AI", email = "[email protected]" },
+]
+readme = "README.md"
+license = { file = "LICENSE" }
+
+dependencies = [
+    "torch>=2.2.0",
+    "lightning @ git+https://github.com/Lightning-AI/lightning@f23b3b1e7fdab1d325f79f69a28706d33144f27e",
+]
+
+[project.urls]
+homepage = "https://github.com/lightning-AI/litgpt"
+documentation = "https://github.com/lightning-AI/litgpt/tutorials"
+
+[project.optional-dependencies]
+test = [
+    "pytest",
+    "pytest-rerunfailures",
+    "pytest-timeout",
+    "transformers>=4.38.0",
+    "einops",
+    "protobuf",
+]
+all = [
+    "jsonargparse[signatures]",  # CLI
+    "bitsandbytes==0.41.0",      # quantization
+    "scipy",                     # required by bitsandbytes
+    "sentencepiece",             # llama-based models
+    "tokenizers",                # pythia, falcon, redpajama
+    "datasets",                  # eval
+    "requests",                  # litgpt.data
+    "litdata",                   # litgpt.data
+    "zstandard",                 # litgpt.data.prepare_slimpajama.py
+    "pandas",                    # litgpt.data.prepare_starcoder.py
+    "pyarrow",                   # litgpt.data.prepare_starcoder.py
+    "tensorboard",               # litgpt.pretrain
+    "torchmetrics",              # litgpt.pretrain
+    "lm_eval @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@115206dc89dad67b8b",
+]
+
+[build-system]
+requires = [
+    "setuptools",
+    "wheel",
+]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+include = [
+    "litgpt",
+    "litgpt.*",
+]
+exclude = []
+
+[tool.setuptools.package-data]
+litgpt = [
+    "LICENSE",
+    "README.md",
+]