consolidate Python configs into pyproject.toml, other clean up (#353)

Co-authored-by: Akshita Bhagia <[email protected]>

CONTRIBUTING.md → .github/CONTRIBUTING.md

@@ -129,9 +129,9 @@ When you're ready to contribute code to address an open issue, please follow the
         isort .
         black .
 
-    Our CI also uses [`flake8`](https://github.com/allenai/LLM/tree/main/tests) to lint the code base and [`mypy`](http://mypy-lang.org/) for type-checking. You should run both of these next with
+    Our CI also uses [`ruff`](https://docs.astral.sh/ruff/) to lint the code base and [`mypy`](http://mypy-lang.org/) for type-checking. You should run both of these next with
 
-        flake8 .
+        ruff check .
 
     and
 

.github/actions/setup-venv/action.yml

@@ -19,7 +19,7 @@ runs:
     - shell: bash
       run: |
         # Install prerequisites.
-        pip install --upgrade pip setuptools wheel virtualenv
+        pip install --upgrade pip setuptools build wheel virtualenv
 
     - shell: bash
       run: |

.github/workflows/main.yml

@@ -32,7 +32,7 @@ jobs:
         task:
           - name: Lint
             run: |
-              flake8 .
+              ruff check .
 
         include:
           - python: '3.10'
@@ -49,8 +49,7 @@ jobs:
             task:
               name: Build
               run: |
-                python setup.py check
-                python setup.py bdist_wheel sdist
+                python -m build
 
           - python: '3.10'
             task:
@@ -175,7 +174,7 @@ jobs:
 
       - name: Install requirements
         run: |
-          pip install --upgrade pip setuptools wheel
+          pip install --upgrade pip setuptools wheel build
           pip install -r dev-requirements.txt
 
       - name: Prepare environment

Makefile

@@ -11,7 +11,7 @@ TEST_IMAGE =  $(shell beaker workspace images $(BEAKER_WORKSPACE) --format=json
 run-checks :
 	isort --check .
 	black --check .
-	flake8 .
+	ruff check .
 	mypy .
 	CUDA_VISIBLE_DEVICES='' pytest -v --color=yes tests/
 

dev-requirements.txt

@@ -1,18 +1,10 @@
-# Checks style, syntax, and other useful errors.
-flake8
-
-# Static type checking
+ruff
 mypy>=1.0,<1.4
-
-# Automatic code formatting
 black>=23.1,<24.0
 isort>=5.12,<5.13
-
-# Running tests
 pytest
 pytest-sphinx
-
-# Needed for packaging and uploading to PyPi
 twine>=1.11.0
 setuptools
 wheel
+build

pyproject.toml

@@ -1,3 +1,51 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "olmo"
+dynamic = ["version", "dependencies", "optional-dependencies"]
+readme = "README.md"
+description = "Open Language Model (OLMo)"
+authors = [
+    { name = "Allen Institute for Artificial Intelligence", email = "[email protected]" }
+]
+requires-python = ">=3.8"
+license = {file = "LICENSE"}
+
+[project.urls]
+Homepage = "https://github.com/allenai/LLM"
+Repository = "https://github.com/allenai/LLM"
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.package-data]
+olmo = ["py.typed"]
+
+[tool.setuptools.dynamic]
+version = { attr = "olmo.version.VERSION" }
+dependencies = { file = ["requirements.txt"] }
+optional-dependencies = { dev = { file = ["dev-requirements.txt"] } }
+
+[tool.setuptools.packages.find]
+include = ["olmo*", "hf_olmo*"]
+exclude = [
+    "*.tests",
+    "*.tests.*",
+    "tests.*",
+    "tests",
+    "test_fixtures",
+    "test_fixtures.*",
+    "docs*",
+    "scripts*",
+    "olmo_tokenizer.*",
+    "evaluation.*",
+    "pretrain_data.*",
+    "tmp_*",
+    "inference.*",
+]
+
 [tool.black]
 line-length = 115
 include = '\.pyi?$'
@@ -12,6 +60,7 @@ exclude = '''
     | \bdist\b
     | \bdoc\b
     | pretrain_data/
+    | inference/
 )
 '''
 
@@ -20,14 +69,42 @@ profile = "black"
 multi_line_output = 3
 extend_skip = ["pretrain_data", "tokenizer"]
 
+[tool.ruff]
+line-length = 115
+ignore = ["F403", "F405", "E501"]
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".venv",
+    "venv",
+    ".mypy_cache",
+    "__pycache__",
+    ".nox",
+    ".pants.d",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "doc",
+    "pretrain_data",
+    "inference",
+]
+
+[tool.ruff.lint.per-file-ignores]
+"**/__init__.py" = ["F401"]
+
 [tool.pyright]
 reportPrivateImportUsage = false
 exclude = ["pretrain_data/", "tokenizer/"]
 
-[build-system]
-requires = ["setuptools", "wheel"]
-build-backend = "setuptools.build_meta"
-
 [tool.mypy]
 ignore_missing_imports = true
 no_site_packages = true

requirements.txt

@@ -1,5 +1,3 @@
-# NOTE: when upgrading requirements here you may have to rebuild and push some
-# Docker images. See each Dockerfile for details on how to do that.
 numpy
 torch>=2.0
 torchmetrics