From 02177fb1e9ce88f97c7b84bdde102935cc4cf98f Mon Sep 17 00:00:00 2001 From: epwalsh Date: Tue, 31 Oct 2023 17:08:27 -0700 Subject: [PATCH 1/4] consolidate Python configs into pyproject.toml --- .flake8 | 27 ------- CONTRIBUTING.md => .github/CONTRIBUTING.md | 4 +- .github/actions/setup-venv/action.yml | 2 +- .github/workflows/main.yml | 5 +- Makefile | 2 +- pyproject.toml | 85 +++++++++++++++++++++- setup.py | 46 ------------ 7 files changed, 87 insertions(+), 84 deletions(-) delete mode 100644 .flake8 rename CONTRIBUTING.md => .github/CONTRIBUTING.md (97%) delete mode 100644 setup.py diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 14bb35e6a..000000000 --- a/.flake8 +++ /dev/null @@ -1,27 +0,0 @@ -[flake8] -max-line-length = 115 - -ignore = - # these rules don't play well with black - # whitespace before : - E203 - # line break before binary operator - W503 - # line too long, who cares? - E501 - -exclude = - .venv - .git - __pycache__ - docs/build - dist - .mypy_cache - pretrain_data - inference/efficiency/dependencies - inference/compression/dependencies - -per-file-ignores = - # __init__.py files are allowed to have unused imports and lines-too-long - */__init__.py:F401,F403 - */**/**/__init__.py:F401,E501 diff --git a/CONTRIBUTING.md b/.github/CONTRIBUTING.md similarity index 97% rename from CONTRIBUTING.md rename to .github/CONTRIBUTING.md index f790ef6ac..af21aa624 100644 --- a/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -129,9 +129,9 @@ When you're ready to contribute code to address an open issue, please follow the isort . black . - Our CI also uses [`flake8`](https://github.com/allenai/LLM/tree/main/tests) to lint the code base and [`mypy`](http://mypy-lang.org/) for type-checking. You should run both of these next with + Our CI also uses [`ruff`](https://docs.astral.sh/ruff/) to lint the code base and [`mypy`](http://mypy-lang.org/) for type-checking. You should run both of these next with - flake8 . + ruff check . and diff --git a/.github/actions/setup-venv/action.yml b/.github/actions/setup-venv/action.yml index e15f0fff5..9d227cc25 100644 --- a/.github/actions/setup-venv/action.yml +++ b/.github/actions/setup-venv/action.yml @@ -19,7 +19,7 @@ runs: - shell: bash run: | # Install prerequisites. - pip install --upgrade pip setuptools wheel virtualenv + pip install --upgrade pip setuptools build wheel virtualenv - shell: bash run: | diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a2f4a0497..2fbc101d6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -32,7 +32,7 @@ jobs: task: - name: Lint run: | - flake8 . + ruff check . include: - python: '3.10' @@ -49,8 +49,7 @@ jobs: task: name: Build run: | - python setup.py check - python setup.py bdist_wheel sdist + python -m build - python: '3.10' task: diff --git a/Makefile b/Makefile index d41bb7360..5eb0e6943 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ TEST_IMAGE = $(shell beaker workspace images $(BEAKER_WORKSPACE) --format=json run-checks : isort --check . black --check . - flake8 . + ruff check . mypy . CUDA_VISIBLE_DEVICES='' pytest -v --color=yes tests/ diff --git a/pyproject.toml b/pyproject.toml index fc6c7a97e..3ec3d7687 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,51 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "olmo" +dynamic = ["version", "dependencies", "optional-dependencies"] +readme = "README.md" +description = "Open Language Model (OLMo)" +authors = [ + { name = "Allen Institute for Artificial Intelligence", email = "contact@allenai.org" } +] +requires-python = ">=3.8" +license = {file = "LICENSE"} + +[project.urls] +Homepage = "https://github.com/allenai/LLM" +Repository = "https://github.com/allenai/LLM" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.package-data] +olmo = ["py.typed"] + +[tool.setuptools.dynamic] +version = { attr = "olmo.version.VERSION" } +dependencies = { file = ["requirements.txt"] } +optional-dependencies = { dev = { file = ["dev-requirements.txt"] } } + +[tool.setuptools.packages.find] +include = ["olmo*", "hf_olmo*"] +exclude = [ + "*.tests", + "*.tests.*", + "tests.*", + "tests", + "test_fixtures", + "test_fixtures.*", + "docs*", + "scripts*", + "olmo_tokenizer.*", + "evaluation.*", + "pretrain_data.*", + "tmp_*", + "inference.*", +] + [tool.black] line-length = 115 include = '\.pyi?$' @@ -12,6 +60,7 @@ exclude = ''' | \bdist\b | \bdoc\b | pretrain_data/ + | inference/ ) ''' @@ -20,14 +69,42 @@ profile = "black" multi_line_output = 3 extend_skip = ["pretrain_data", "tokenizer"] +[tool.ruff] +line-length = 115 +ignore = ["F403", "F405", "E501"] +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".venv", + "venv", + ".mypy_cache", + "__pycache__", + ".nox", + ".pants.d", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "doc", + "pretrain_data", + "inference", +] + +[tool.ruff.lint.per-file-ignores] +"**/__init__.py" = ["F401"] + [tool.pyright] reportPrivateImportUsage = false exclude = ["pretrain_data/", "tokenizer/"] -[build-system] -requires = ["setuptools", "wheel"] -build-backend = "setuptools.build_meta" - [tool.mypy] ignore_missing_imports = true no_site_packages = true diff --git a/setup.py b/setup.py deleted file mode 100644 index 7951268c2..000000000 --- a/setup.py +++ /dev/null @@ -1,46 +0,0 @@ -from setuptools import find_packages, setup - - -def read_requirements(filename: str): - with open(filename) as requirements_file: - requirements = [] - for line in requirements_file: - line = line.strip() - if line.startswith("#") or len(line) <= 0: - continue - requirements.append(line) - return requirements - - -# version.py defines the VERSION and VERSION_SHORT variables. -# We use exec here so we don't import cached_path whilst setting up. -VERSION = {} # type: ignore -with open("olmo/version.py", "r") as version_file: - exec(version_file.read(), VERSION) - -setup( - name="olmo", - version=VERSION["VERSION"], - description="", - long_description=open("README.md").read(), - long_description_content_type="text/markdown", - classifiers=[ - "Intended Audience :: Science/Research", - "Development Status :: 3 - Alpha", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - ], - keywords="", - url="https://github.com/allenai/LLM", - author="Allen Institute for Artificial Intelligence", - author_email="contact@allenai.org", - license="Apache", - packages=find_packages( - exclude=["*.tests", "*.tests.*", "tests.*", "tests"], - ), - package_data={"olmo": ["py.typed"]}, - install_requires=read_requirements("requirements.txt"), - extras_require={"dev": read_requirements("dev-requirements.txt")}, - python_requires=">=3.8", -) From 406e5aa9ae095f14507b54c22d384d862f33d3c8 Mon Sep 17 00:00:00 2001 From: epwalsh Date: Tue, 31 Oct 2023 17:11:12 -0700 Subject: [PATCH 2/4] clean up requirements files --- dev-requirements.txt | 11 +---------- requirements.txt | 2 -- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index fd9ce794d..be1bc5866 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,18 +1,9 @@ -# Checks style, syntax, and other useful errors. -flake8 - -# Static type checking +ruff mypy>=1.0,<1.4 - -# Automatic code formatting black>=23.1,<24.0 isort>=5.12,<5.13 - -# Running tests pytest pytest-sphinx - -# Needed for packaging and uploading to PyPi twine>=1.11.0 setuptools wheel diff --git a/requirements.txt b/requirements.txt index a93b3ff9d..b5800e93a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,3 @@ -# NOTE: when upgrading requirements here you may have to rebuild and push some -# Docker images. See each Dockerfile for details on how to do that. numpy torch>=2.0 torchmetrics From c62ad01cddcb852763f4d59b5198b1da8173c083 Mon Sep 17 00:00:00 2001 From: epwalsh Date: Tue, 31 Oct 2023 17:53:54 -0700 Subject: [PATCH 3/4] add missing 'build' dep --- .github/workflows/main.yml | 2 +- dev-requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2fbc101d6..7c543625a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -174,7 +174,7 @@ jobs: - name: Install requirements run: | - pip install --upgrade pip setuptools wheel + pip install --upgrade pip setuptools wheel build pip install -r dev-requirements.txt - name: Prepare environment diff --git a/dev-requirements.txt b/dev-requirements.txt index be1bc5866..3c17627e5 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -7,3 +7,4 @@ pytest-sphinx twine>=1.11.0 setuptools wheel +build From b43b0de9bbdeb7850749e786037a1a62f8a84ca3 Mon Sep 17 00:00:00 2001 From: Pete Date: Thu, 2 Nov 2023 11:47:42 -0700 Subject: [PATCH 4/4] Change contact email --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3ec3d7687..1f887ebe3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ dynamic = ["version", "dependencies", "optional-dependencies"] readme = "README.md" description = "Open Language Model (OLMo)" authors = [ - { name = "Allen Institute for Artificial Intelligence", email = "contact@allenai.org" } + { name = "Allen Institute for Artificial Intelligence", email = "olmo@allenai.org" } ] requires-python = ">=3.8" license = {file = "LICENSE"}