diff --git a/.github/workflows/cpu-tests.yml b/.github/workflows/cpu-tests.yml
index 20a04310d8..1d896e2617 100644
--- a/.github/workflows/cpu-tests.yml
+++ b/.github/workflows/cpu-tests.yml
@@ -41,13 +41,11 @@ jobs:
         python-version: ${{ matrix.python-version }}
         cache: 'pip'
         cache-dependency-path: |
-          requirements.txt
-          requirements-all.txt
-          setup.py
+          pyproject.toml
 
     - name: Install minimal dependencies
       run: |
-        pip install -r requirements.txt
+        pip install .
         pip list
         # make sure all modules are still importable with only the minimal dependencies available
         modules=$(
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000000..1909359913
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,64 @@
+[project]
+name = "litgpt"
+version = "0.1.0"
+description = "Open source large language model implementation"
+authors = [
+    { name = "Lightning AI", email = "contact@lightning.ai" },
+]
+readme = "README.md"
+license = { file = "LICENSE" }
+
+dependencies = [
+    "torch>=2.2.0",
+    "lightning @ git+https://github.com/Lightning-AI/lightning@f23b3b1e7fdab1d325f79f69a28706d33144f27e",
+]
+
+[project.urls]
+homepage = "https://github.com/lightning-AI/litgpt"
+documentation = "https://github.com/lightning-AI/litgpt/tutorials"
+
+[project.optional-dependencies]
+test = [
+    "pytest",
+    "pytest-rerunfailures",
+    "pytest-timeout",
+    "transformers>=4.38.0",
+    "einops",
+    "protobuf",
+]
+all = [
+    "jsonargparse[signatures]",  # CLI
+    "bitsandbytes==0.41.0",      # quantization
+    "scipy",                     # required by bitsandbytes
+    "sentencepiece",             # llama-based models
+    "tokenizers",                # pythia, falcon, redpajama
+    "datasets",                  # eval
+    "requests",                  # litgpt.data
+    "litdata",                   # litgpt.data
+    "zstandard",                 # litgpt.data.prepare_slimpajama.py
+    "pandas",                    # litgpt.data.prepare_starcoder.py
+    "pyarrow",                   # litgpt.data.prepare_starcoder.py
+    "tensorboard",               # litgpt.pretrain
+    "torchmetrics",              # litgpt.pretrain
+    "lm_eval @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@115206dc89dad67b8b",
+]
+
+[build-system]
+requires = [
+    "setuptools",
+    "wheel",
+]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+include = [
+    "litgpt",
+    "litgpt.*",
+]
+exclude = []
+
+[tool.setuptools.package-data]
+litgpt = [
+    "LICENSE",
+    "README.md",
+]
\ No newline at end of file
diff --git a/requirements-all.txt b/requirements-all.txt
deleted file mode 100644
index 2ffff4ef89..0000000000
--- a/requirements-all.txt
+++ /dev/null
@@ -1,15 +0,0 @@
--r requirements.txt
-jsonargparse[signatures]  # CLI
-bitsandbytes==0.41.0      # quantization
-scipy                     # required by bitsandbytes
-sentencepiece             # llama-based models
-tokenizers                # pythia, falcon, redpajama
-datasets                  # eval
-requests                  # litgpt.data
-litdata                   # litgpt.data
-zstandard                 # litgpt.data.prepare_slimpajama.py
-pandas                    # litgpt.data.prepare_starcoder.py
-pyarrow                   # litgpt.data.prepare_starcoder.py
-tensorboard               # litgpt.pretrain
-torchmetrics              # litgpt.pretrain
-lm_eval @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@115206dc89dad67b8b
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index 5922368c0a..0000000000
--- a/requirements.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-torch>=2.2.0
-lightning @ git+https://github.com/Lightning-AI/lightning@f23b3b1e7fdab1d325f79f69a28706d33144f27e
diff --git a/setup.py b/setup.py
deleted file mode 100644
index 5c3c01d98d..0000000000
--- a/setup.py
+++ /dev/null
@@ -1,27 +0,0 @@
-# Copyright Lightning AI. Licensed under the Apache License 2.0, see LICENSE file.
-
-from pathlib import Path
-
-from setuptools import find_packages, setup
-
-root = Path(__file__).parent
-readme = (root / "README.md").read_text()
-requirements = (root / "requirements.txt").read_text().split("\n")
-requirements_all = (root / "requirements-all.txt").read_text().split("\n")
-requirements_all = [r for r in requirements_all if r and not r.strip().startswith("-r")]
-
-setup(
-    name="litgpt",
-    version="0.1.0",
-    description="Open source large language model implementation",
-    author="Lightning AI",
-    url="https://github.com/lightning-AI/litgpt",
-    install_requires=requirements,
-    extras_require={
-        "all": requirements_all,
-        "test": ["pytest", "pytest-rerunfailures", "pytest-timeout", "transformers>=4.38.0", "einops", "protobuf"],
-    },
-    packages=find_packages(),
-    long_description=readme,
-    long_description_content_type="text/markdown",
-)