diff --git a/.cruft.json b/.cruft.json index fd3bff2..67c7a5e 100644 --- a/.cruft.json +++ b/.cruft.json @@ -1,6 +1,6 @@ { "template": "https://github.com/iterative/cookiecutter-dvc-plugin/", - "commit": "91159828cdce86290b97bf4985732651805523c4", + "commit": "da6f5faa767006bd66ed6164e9c573bc098cb346", "checkout": null, "context": { "cookiecutter": { diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e7589f6..be21aaf 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -20,7 +20,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@v3 with: - python-version: 3.8 + python-version: 3.9 - name: Install run: | pip install --upgrade pip wheel diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 129af0e..4e9568b 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -23,7 +23,9 @@ jobs: fail-fast: false matrix: os: [ubuntu-20.04, windows-latest, macos-latest] - pyv: ["3.8", "3.9", "3.10", "3.11"] + # NOTE: 3.12 is temporarily disabled waiting for + # https://github.com/fsspec/universal_pathlib/pull/152 + pyv: ["3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6b5fd81..2fe02b6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,11 +9,12 @@ repos: language: fail files: \.rej$ repo: local - - hooks: - - id: black - language_version: python3 - repo: https://github.com/ambv/black - rev: 22.3.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: 'v0.2.0' + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.0.1 hooks: @@ -26,21 +27,6 @@ repos: - ba,datas,fo,uptodate repo: https://github.com/codespell-project/codespell rev: v2.1.0 - - hooks: - - id: isort - language_version: python3 - repo: https://github.com/timothycrosley/isort - rev: 5.12.0 - - hooks: - - id: flake8 - language_version: python3 - additional_dependencies: - - flake8-bugbear - - flake8-comprehensions - - flake8-debugger - - flake8-string-format - repo: https://github.com/pycqa/flake8 - rev: 3.9.2 - repo: local hooks: - id: mypy @@ -49,11 +35,6 @@ repos: files: ^dvc_s3/ language: system types: [python] - - id: pylint - name: pylint - entry: pylint - language: system - types: [python] - hooks: - args: - -i diff --git a/dvc_s3/__init__.py b/dvc_s3/__init__.py index b9f1c84..16aedff 100644 --- a/dvc_s3/__init__.py +++ b/dvc_s3/__init__.py @@ -1,13 +1,14 @@ import os import threading from collections import defaultdict -from typing import Any, Dict, Optional, Tuple +from typing import Any, ClassVar, Optional from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit +from funcy import first, wrap_prop + from dvc.utils.objects import cached_property from dvc_objects.fs.base import ObjectFileSystem from dvc_objects.fs.errors import ConfigError -from funcy import first, wrap_prop _AWS_CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".aws", "config") @@ -40,19 +41,19 @@ def human_readable_to_bytes(value: str) -> int: # pylint:disable=abstract-method class S3FileSystem(ObjectFileSystem): protocol = "s3" - REQUIRES = {"s3fs": "s3fs", "boto3": "boto3"} + REQUIRES: ClassVar[dict[str, str]] = {"s3fs": "s3fs", "boto3": "boto3"} PARAM_CHECKSUM = "etag" VERSION_ID_KEY = "versionId" - _GRANTS = { + _GRANTS: ClassVar[dict[str, str]] = { "grant_full_control": "GrantFullControl", "grant_read": "GrantRead", "grant_read_acp": "GrantReadACP", "grant_write_acp": "GrantWriteACP", } - _TRANSFER_CONFIG_ALIASES = { + _TRANSFER_CONFIG_ALIASES: ClassVar[dict[str, str]] = { "max_queue_size": "max_io_queue", "max_concurrent_requests": "max_concurrency", "multipart_threshold": "multipart_threshold", @@ -63,7 +64,7 @@ def getcwd(self): return self.fs.root_marker @classmethod - def split_version(cls, path: str) -> Tuple[str, Optional[str]]: + def split_version(cls, path: str) -> tuple[str, Optional[str]]: parts = list(urlsplit(path)) query = parse_qs(parts[3]) if cls.VERSION_ID_KEY in query: @@ -91,7 +92,7 @@ def version_path(cls, path: str, version_id: Optional[str]) -> str: @classmethod def coalesce_version( cls, path: str, version_id: Optional[str] - ) -> Tuple[str, Optional[str]]: + ) -> tuple[str, Optional[str]]: path, path_version_id = cls.split_version(path) versions = {ver for ver in (version_id, path_version_id) if ver} if len(versions) > 1: @@ -99,7 +100,7 @@ def coalesce_version( return path, (versions.pop() if versions else None) @classmethod - def _get_kwargs_from_urls(cls, urlpath: str) -> Dict[str, Any]: + def _get_kwargs_from_urls(cls, urlpath: str) -> dict[str, Any]: ret = super()._get_kwargs_from_urls(urlpath) url_query = ret.get("url_query") if url_query is not None: @@ -210,9 +211,7 @@ def _prepare_credentials(self, **config): # config kwargs session_config = login_info["config_kwargs"] - session_config["s3"] = self._load_aws_config_file( - login_info["profile"] - ) + session_config["s3"] = self._load_aws_config_file(login_info["profile"]) shared_creds = config.get("credentialpath") if shared_creds: diff --git a/dvc_s3/tests/benchmarks.py b/dvc_s3/tests/benchmarks.py index 41e97bd..12b417a 100644 --- a/dvc_s3/tests/benchmarks.py +++ b/dvc_s3/tests/benchmarks.py @@ -1,5 +1 @@ # pylint: disable=unused-import -# noqa -from dvc.testing.benchmarks.cli.stories.use_cases.test_sharing import ( # noqa - test_sharing as test_sharing_s3, -) diff --git a/dvc_s3/tests/cloud.py b/dvc_s3/tests/cloud.py index 123a781..62bac47 100644 --- a/dvc_s3/tests/cloud.py +++ b/dvc_s3/tests/cloud.py @@ -2,9 +2,10 @@ import os import uuid +from funcy import cached_property + from dvc.testing.cloud import Cloud from dvc.testing.path_info import CloudURLInfo -from funcy import cached_property class S3(Cloud, CloudURLInfo): @@ -58,7 +59,7 @@ def is_dir(self): def exists(self): return self.is_file() or self.is_dir() - def mkdir(self, mode=0o777, parents=False, exist_ok=False): + def mkdir(self, mode=0o777, parents=False, exist_ok=False): # noqa: ARG002 assert mode == 0o777 assert parents diff --git a/dvc_s3/tests/conftest.py b/dvc_s3/tests/conftest.py index edf692f..cd0195f 100644 --- a/dvc_s3/tests/conftest.py +++ b/dvc_s3/tests/conftest.py @@ -1,3 +1,3 @@ -from dvc.testing.fixtures import * # noqa, pylint: disable=wildcard-import,unused-import +from dvc.testing.fixtures import * # noqa: F403 -from .fixtures import * # noqa, pylint: disable=wildcard-import,unused-import +from .fixtures import * # noqa: F403 diff --git a/dvc_s3/tests/fixtures.py b/dvc_s3/tests/fixtures.py index 0d89bfe..aa4b4a8 100644 --- a/dvc_s3/tests/fixtures.py +++ b/dvc_s3/tests/fixtures.py @@ -20,7 +20,7 @@ def _make_s3(): @pytest.fixture # pylint: disable-next=redefined-outer-name,unused-argument -def make_s3_version_aware(versioning, tmp_s3_path, s3_server): +def make_s3_version_aware(versioning, tmp_s3_path, s3_server): # noqa: ARG001 def _make_s3(): return FakeS3(str(tmp_s3_path).rstrip("/"), config=s3_server) @@ -29,29 +29,29 @@ def _make_s3(): @pytest.fixture def s3(make_s3): # pylint: disable=redefined-outer-name - yield make_s3() + return make_s3() @pytest.fixture def cloud(make_cloud): - yield make_cloud(typ="s3") + return make_cloud(typ="s3") @pytest.fixture def remote(make_remote): - yield make_remote(name="upstream", typ="s3") + return make_remote(name="upstream", typ="s3") @pytest.fixture def remote_version_aware(make_remote_version_aware): - yield make_remote_version_aware(name="upstream", typ="s3") + return make_remote_version_aware(name="upstream", typ="s3") @pytest.fixture def remote_worktree(make_remote_worktree): - yield make_remote_worktree(name="upstream", typ="s3") + return make_remote_worktree(name="upstream", typ="s3") @pytest.fixture def workspace(make_workspace): - yield make_workspace(name="workspace", typ="s3") + return make_workspace(name="workspace", typ="s3") diff --git a/dvc_s3/tests/test_dvc.py b/dvc_s3/tests/test_dvc.py index 496f3c1..65b997c 100644 --- a/dvc_s3/tests/test_dvc.py +++ b/dvc_s3/tests/test_dvc.py @@ -1,16 +1,7 @@ import pytest -from dvc.testing.api_tests import ( # noqa, pylint: disable=unused-import - TestAPI, -) -from dvc.testing.remote_tests import ( # noqa, pylint: disable=unused-import - TestRemote, - TestRemoteVersionAware, -) + from dvc.testing.workspace_tests import TestGetUrl as _TestGetUrl from dvc.testing.workspace_tests import TestImport as _TestImport -from dvc.testing.workspace_tests import ( # noqa, pylint: disable=unused-import - TestImportURLVersionAware, -) from dvc.testing.workspace_tests import TestLsUrl as _TestLsUrl from dvc.testing.workspace_tests import TestToRemote as _TestToRemote diff --git a/dvc_s3/tests/test_s3.py b/dvc_s3/tests/test_s3.py index 710d85f..e988c20 100644 --- a/dvc_s3/tests/test_s3.py +++ b/dvc_s3/tests/test_s3.py @@ -1,16 +1,16 @@ import os import pytest -from dvc.fs import ConfigError +from dvc.fs import ConfigError from dvc_s3 import S3FileSystem bucket_name = "bucket-name" prefix = "some/prefix" url = f"s3://{bucket_name}/{prefix}" key_id = "key-id" -key_secret = "key-secret" -session_token = "session-token" +key_secret = "key-secret" # noqa: S105 +session_token = "session-token" # noqa: S105 @pytest.fixture(autouse=True, name="grants") @@ -43,12 +43,12 @@ def test_s3_config_credentialpath(monkeypatch): monkeypatch.setattr(os, "environ", environment) config = {"url": url, "credentialpath": "somewhere"} - S3FileSystem(**config).fs_args # pylint: disable=W0106 + S3FileSystem(**config).fs_args # noqa: B018 assert environment["AWS_SHARED_CREDENTIALS_FILE"] == "somewhere" environment.clear() config = {"url": url, "configpath": "somewhere"} - S3FileSystem(**config).fs_args # pylint: disable=W0106 + S3FileSystem(**config).fs_args # noqa: B018 assert environment["AWS_CONFIG_FILE"] == "somewhere" environment.clear() @@ -57,7 +57,7 @@ def test_s3_config_credentialpath(monkeypatch): "credentialpath": "somewhere", "configpath": "elsewhere", } - S3FileSystem(**config).fs_args # pylint: disable=W0106 + S3FileSystem(**config).fs_args # noqa: B018 assert environment["AWS_SHARED_CREDENTIALS_FILE"] == "somewhere" assert environment["AWS_CONFIG_FILE"] == "elsewhere" environment.clear() @@ -105,8 +105,7 @@ def test_grants(): extra_args = fs.fs_args["s3_additional_kwargs"] assert ( - extra_args["GrantRead"] - == "id=read-permission-id,id=other-read-permission-id" + extra_args["GrantRead"] == "id=read-permission-id,id=other-read-permission-id" ) assert extra_args["GrantReadACP"] == "id=read-acp-permission-id" assert extra_args["GrantWriteACP"] == "id=write-acp-permission-id" @@ -119,7 +118,7 @@ def test_grants_mutually_exclusive_acl_error(grants): fs = S3FileSystem(**config) with pytest.raises(ConfigError): - fs.fs_args # pylint: disable=W0104 + fs.fs_args # noqa: B018 def test_sse_kms_key_id(): diff --git a/dvc_s3/tests/test_utils.py b/dvc_s3/tests/test_utils.py index 4ce23bd..7648537 100644 --- a/dvc_s3/tests/test_utils.py +++ b/dvc_s3/tests/test_utils.py @@ -26,5 +26,5 @@ def test_conversions_human_readable_to_bytes(test_input, expected): @pytest.mark.parametrize("invalid_input", ["foo", "10XB", "1000Pb", "fooMiB"]) def test_conversions_human_readable_to_bytes_invalid(invalid_input): - with pytest.raises(ValueError): + with pytest.raises(ValueError): # noqa: PT011 human_readable_to_bytes(invalid_input) diff --git a/pyproject.toml b/pyproject.toml index dee913d..4214687 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,28 +5,67 @@ build-backend = "setuptools.build_meta" [tool.setuptools_scm] write_to = "dvc_s3/_dvc_s3_version.py" -[tool.black] -line-length = 79 -include = '\.pyi?$' -exclude = ''' -/( - \.eggs - | \.git - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | _build - | buck-out - | build - | dist -)/ -''' - -[tool.isort] -profile = "black" -known_first_party = ["dvc_s3"] -line_length = 79 +[project] +name = "dvc-s3" +description = "s3 plugin for dvc" +readme = "README.rst" +keywords = [ + "dvc", + "s3", +] +license = { text = "Apache License 2.0" } +maintainers = [{ name = "Iterative", email = "support@dvc.org" }] +authors = [{ name = "Iterative", email = "support@dvc.org" }] +requires-python = ">=3.9" +classifiers = [ + "Development Status :: 4 - Beta", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +dynamic = ["version"] +dependencies = [ + "dvc", + "s3fs>=2023.6.0", + "aiobotocore[boto3]>=2.5.0", + "flatten_dict>=0.4.1,<1", +] + +[project.optional-dependencies] +tests = [ + "wheel==0.37.0", + "dvc[testing]", + # Test requirements + "pytest==6.2.5", + "pytest-cov==3.0.0", + "pytest-xdist==2.4.0", + "pytest-mock==3.6.1", + "pytest-lazy-fixture==0.6.3", + "pytest-servers[s3]>=0.4.0", + "pytest-docker>=1,<2", + "flaky==3.7.0", + "mock==4.0.3", + "wget==3.2", + "filelock==3.3.2", + "xmltodict==0.12.0", + # required by collective.checkdocs + "Pygments==2.10.0", + "collective.checkdocs==0.2", + "pydocstyle==6.1.1", + # type-checking + "mypy==0.981", + "types-requests==2.25.11", + "types-tabulate==0.8.3", + "types-toml==0.10.1", + # optional dependencies + 'pywin32>=225; sys_platform == "win32"', +] + +[project.urls] +Documentation = "https://dvc.org/doc" +Source = "https://github.com/iterative/dvc-s3" [tool.pytest.ini_options] log_level = "debug" @@ -51,28 +90,46 @@ warn_redundant_casts = true warn_unreachable = true files = ["dvc_s3"] -[tool.pylint.master] -extension-pkg-whitelist = ["pygit2"] -init-hook = "import sys; sys.path.append(str('tests'))" - -[tool.pylint.message_control] -disable = [ - "format", "refactoring", "spelling", "design", - "invalid-name", "duplicate-code", "fixme", - "unused-wildcard-import", "cyclic-import", "wrong-import-order", - "wrong-import-position", "ungrouped-imports", "multiple-imports", - "logging-format-interpolation", "logging-fstring-interpolation", - "missing-function-docstring", "missing-module-docstring", - "missing-class-docstring", "raise-missing-from", "import-outside-toplevel", +[tool.ruff] +output-format = "full" +show-fixes = true + +[tool.ruff.lint] +ignore = [ + "N818", "S101", "ISC001", "PT004", "PT007", "RET502", "RET503", "SIM105", "SIM108", "SIM117", + "TRY003", "TRY300", "PLR2004", "PLW2901", "LOG007", ] -enable = ["c-extension-no-member", "no-else-return"] +select = [ + "F", "E", "W", "C90", "I", "N", "UP", "YTT", "ASYNC", "S", "BLE", "B", "A", "C4", "T10", + "EXE", "ISC", "ICN", "G", "INP", "PIE", "T20", "PYI", "PT", "Q", "RSE", "RET", + "SLOT", "SIM", "TID", "TCH", "ARG", "PGH", "PLC", "PLE", "PLR", "PLW", "TRY", + "FLY", "PERF101", "LOG", "RUF", "RUF022", "RUF023", "RUF024", "RUF025", "RUF026", +] +preview = true +explicit-preview-rules = true + +[tool.ruff.lint.flake8-pytest-style] +fixture-parentheses = false +mark-parentheses = false +parametrize-names-type = "csv" +raises-extend-require-match-for = ["dvc.exceptions.DvcException"] + +[tool.ruff.lint.flake8-tidy-imports] + +[tool.ruff.lint.flake8-type-checking] +strict = true + +[tool.ruff.lint.flake8-unused-arguments] +ignore-variadic-names = true + +[tool.ruff.lint.isort] +known-first-party = ["dvc", "dvc_data", "dvc_objects"] + +[tool.ruff.lint.pep8-naming] +extend-ignore-names = ["M", "SCM"] -[tool.pylint.typecheck] -generated-members = ["pytest.lazy_fixture", "logging.TRACE", "logger.trace", "sys.getwindowsversion", "argparse.Namespace"] -ignored-classes = ["Dvcfile"] -ignored-modules = ["azure"] -signature-mutators = ["funcy.decorators.decorator"] +[tool.ruff.lint.pylint] +max-args = 10 -[tool.pylint.variables] -dummy-variables-rgx = "_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_" -ignored-argument-names = "_.*|^ignored_|^unused_|args|kwargs" +[tool.ruff.lint.per-file-ignores] +"dvc-s3/tests/**" = ["S", "ARG001", "ARG002", "TRY002", "TRY301"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 4257472..0000000 --- a/setup.cfg +++ /dev/null @@ -1,85 +0,0 @@ -[metadata] -description = s3 plugin for dvc -name = dvc-s3 -long_description = file: README.rst -long_description_content_type = text/x-rst -license = Apache License 2.0 -license_file = LICENSE -url = http://dvc.org -project_urls = - Documentation = https://dvc.org/doc - Source = https://github.com/iterative/dvc-s3 -download_url = https://github.com/iterative/dvc-s3 -keywords = dvc, s3 -classifiers = - Development Status :: 4 - Beta - Programming Language :: Python :: 3 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 - Programming Language :: Python :: 3.11 - -[options] -setup_requires = - setuptools>=48 - setuptools_scm[toml]>=6.3.1 -python_requires = >=3.8 -zip_safe = False -packages = find: -include_package_data = True -install_requires = - dvc - s3fs>=2023.6.0 - aiobotocore[boto3]>=2.5.0 - flatten_dict>=0.4.1,<1 - -[options.extras_require] -tests = - wheel==0.37.0 - dvc[testing] - # Test requirements - pytest==6.2.5 - pytest-cov==3.0.0 - pytest-xdist==2.4.0 - pytest-mock==3.6.1 - pytest-lazy-fixture==0.6.3 - pytest-servers[s3]>=0.4.0 - pytest-docker>=1,<2 - flaky==3.7.0 - mock==4.0.3 - wget==3.2 - filelock==3.3.2 - xmltodict==0.12.0 - # required by collective.checkdocs - Pygments==2.10.0 - collective.checkdocs==0.2 - pydocstyle==6.1.1 - # pylint requirements - pylint==2.16.2 - # we use this to suppress some messages in tests, eg: foo/bar naming, - # and, protected method calls in our tests - pylint-plugin-utils==0.6 - # type-checking - mypy==0.981 - types-requests==2.25.11 - types-tabulate==0.8.3 - types-toml==0.10.1 - # optional dependencies - pywin32>=225; sys_platform == 'win32' - -[flake8] -ignore= - # Whitespace before ':' - E203, - # Too many leading '#' for block comment - E266, - # Line break occurred before a binary operator - W503, - # unindexed parameters in the str.format, see: - # https://pypi.org/project/flake8-string-format/ - P1, -max_line_length = 79 -max-complexity = 15 -select = B,C,E,F,W,T4,B902,T,P -show_source = true -count = true