From 54e3c738bc02c57fa4ca2b852128f965d50dd5f7 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 11 Jan 2024 11:36:54 +0100 Subject: [PATCH 01/14] Use pyproject.toml and upgrade to Python 3.10 --- .github/workflows/cd_release.yml | 8 +- .github/workflows/ci_automerge_dependabot.yml | 4 +- .github/workflows/ci_cd_updated_main.yml | 4 +- .github/workflows/ci_dependabot.yml | 4 +- .github/workflows/ci_tests.yml | 20 ++-- .pre-commit-hooks.yaml | 2 +- MANIFEST.in | 2 - README.md | 6 +- docs/index.md | 6 +- pyproject.toml | 85 ++++++++++++++-- requirements.txt | 1 - requirements_dev.txt | 4 - requirements_docs.txt | 7 -- setup.py | 97 ------------------- 14 files changed, 106 insertions(+), 144 deletions(-) delete mode 100644 MANIFEST.in delete mode 100644 requirements.txt delete mode 100644 requirements_dev.txt delete mode 100644 requirements_docs.txt delete mode 100644 setup.py diff --git a/.github/workflows/cd_release.yml b/.github/workflows/cd_release.yml index 5387325..0b7e0e3 100644 --- a/.github/workflows/cd_release.yml +++ b/.github/workflows/cd_release.yml @@ -23,10 +23,10 @@ jobs: with: fetch-depth: 0 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.10" - name: Install Python dependencies run: | @@ -99,10 +99,10 @@ jobs: ref: ${{ env.PUBLISH_UPDATE_BRANCH }} token: ${{ secrets.RELEASE_PAT }} - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.10" - name: Install Python dependencies run: | diff --git a/.github/workflows/ci_automerge_dependabot.yml b/.github/workflows/ci_automerge_dependabot.yml index 08a35c6..ca0e7c0 100644 --- a/.github/workflows/ci_automerge_dependabot.yml +++ b/.github/workflows/ci_automerge_dependabot.yml @@ -23,10 +23,10 @@ jobs: ref: ${{ github.event.pull_request.head.ref }} persist-credentials: false - - name: Setup Python 3.9 + - name: Setup Python 3.10 uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.10" - name: Set up git user info run: | diff --git a/.github/workflows/ci_cd_updated_main.yml b/.github/workflows/ci_cd_updated_main.yml index 64d50c3..c629f8c 100644 --- a/.github/workflows/ci_cd_updated_main.yml +++ b/.github/workflows/ci_cd_updated_main.yml @@ -82,11 +82,11 @@ jobs: fetch-depth: 0 token: ${{ secrets.RELEASE_PAT }} - - name: Set up Python 3.9 + - name: Set up Python 3.10 if: env.RELEASE_RUN == 'false' uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.10" - name: Install dependencies if: env.RELEASE_RUN == 'false' diff --git a/.github/workflows/ci_dependabot.yml b/.github/workflows/ci_dependabot.yml index 91fecf5..369f58b 100644 --- a/.github/workflows/ci_dependabot.yml +++ b/.github/workflows/ci_dependabot.yml @@ -25,10 +25,10 @@ jobs: with: ref: ${{ env.DEFAULT_REPO_BRANCH }} - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.10" - name: Install `pre-commit` and dependencies run: | diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 5563bbb..92d3d1d 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -15,10 +15,10 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.10" - name: Install dependencies run: | @@ -49,10 +49,10 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.10" - name: Install dependencies run: | @@ -78,10 +78,10 @@ jobs: with: fetch-depth: 2 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.10" - name: Install python dependencies run: | @@ -121,10 +121,10 @@ jobs: with: fetch-depth: 2 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.10" - name: Install dependencies run: | @@ -146,10 +146,10 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.10" - name: Install dependencies run: | diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index 2daa0e3..b14148b 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -3,7 +3,7 @@ description: "Turtle Canon: A tool for canonizing Turtle (`.ttl`) ontology files." entry: turtle-canon language: python - language_version: python3.9 + language_version: python3.10 minimum_pre_commit_version: 1.13.0 require_serial: true files: \.ttl$ diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index a57a4e1..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -include LICENSE README.md -include requirements*.txt diff --git a/README.md b/README.md index 358c599..32e0c19 100644 --- a/README.md +++ b/README.md @@ -16,19 +16,19 @@ The main use case for developing this tool is when developing ontologies utilizi ## Install -The tool is written for Python 3.9, so one needs at minimum Python 3.9 installed to run it at this stage. +The tool is written for Python 3.10, so one needs at minimum Python 3.10 installed to run it at this stage. The plan is to make a stand-alone executable for each of the major OS'. Install via PyPI (stable version, recommended): ```shell -python3.9 -m pip install turtle-canon +python3.10 -m pip install turtle-canon ``` Install via GitHub (development version): ```shell -python3.9 -m pip install git+https://github.com/CasperWA/turtle-canon#egg=turtle-canon +python3.10 -m pip install git+https://github.com/CasperWA/turtle-canon#egg=turtle-canon ``` ## Usage diff --git a/docs/index.md b/docs/index.md index 75c6676..a762ed4 100644 --- a/docs/index.md +++ b/docs/index.md @@ -16,19 +16,19 @@ The main use case for developing this tool is when developing ontologies utilizi ## Install -The tool is written for Python 3.9, so one needs at minimum Python 3.9 installed to run it at this stage. +The tool is written for Python 3.10, so one needs at minimum Python 3.10 installed to run it at this stage. The plan is to make a stand-alone executable for each of the major OS'. Install via PyPI (stable version, recommended): ```shell -python3.9 -m pip install turtle-canon +python3.10 -m pip install turtle-canon ``` Install via GitHub (development version): ```shell -python3.9 -m pip install git+https://github.com/CasperWA/turtle-canon#egg=turtle-canon +python3.10 -m pip install git+https://github.com/CasperWA/turtle-canon#egg=turtle-canon ``` ## Usage diff --git a/pyproject.toml b/pyproject.toml index 3a3e770..6f8a682 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,18 +1,91 @@ +[build-system] +requires = ["hatchling~=1.21"] +build-backend = "hatchling.build" + +[project] +name = "turtle-canon" +authors = [ + {name = "Casper Welzel Andersen", email = "casper.w.andersen@sintef.no"}, +] +maintainers = [ + {name = "Casper Welzel Andersen", email = "casper.w.andersen@sintef.no"}, + {name = "Tor S. Haugland", email = "tor.haugland@sintef.no"}, +] +description = "A tool for canonizing Turtle (`.ttl`) ontology files." +readme = "README.md" +license = {file = "LICENSE"} +keywords = ["ontology", "turtle"] +classifiers = [ + "Development Status :: 2 - Pre-Alpha", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Topic :: Software Development", + "Topic :: Software Development :: Pre-processors", + "Topic :: Software Development :: Testing", + "Topic :: Software Development :: Version Control", + "Topic :: Text Processing", + "Topic :: Utilities", +] +requires-python = "~=3.10" +dynamic = ["version"] + +dependencies = [ + "rdflib>=6.0.1,<8", +] + +[project.optional-dependencies] +docs = [ + "invoke~=2.2", + "mike~=2.0", + "mkdocs~=1.5", + "mkdocs-awesome-pages-plugin~=2.9", + "mkdocs-material~=9.5", + "mkdocs-minify-plugin~=0.7.2", + "mkdocstrings[python]~=0.24.0", +] +dev = [ + "pre-commit~=3.6", + "pylint~=3.0", + "pytest~=7.4", + "pytest-cov~=4.1", + "turtle-canon[docs]", +] + +[project.urls] +Home = "https://CasperWA.github.io/turtle-canon" +Documentation = "https://CasperWA.github.io/turtle-canon" +Source = "https://github.com/CasperWA/turtle-canon" +"Issue Tracker" = "https://github.com/CasperWA/turtle-canon/issues" +Changelog = "https://github.com/CasperWA/turtle-canon/blob/main/CHANGELOG.md" +Package = "https://pypi.org/project/turtle-canon" + +[project.scripts] +turtle-canon = "turtle_canon.cli.cmd_turtle_canon:main" + +[tool.hatch.version] +path = "turtle_canon/__init__.py" + [tool.black] line-length = 88 -target-version = ['py39'] +target-version = ['py310'] [tool.pytest.ini_options] -minversion = "6.0" +minversion = "7.4" required_plugins = "pytest-cov>=4.1" -addopts = "-rs --cov=./turtle_canon/ --cov-report=term --durations=10" +addopts = "-rs --cov=turtle_canon --cov-report=term-missing" filterwarnings = [ - "ignore:.*imp module.*:DeprecationWarning", - "ignore:.*invalid escape sequence.*:DeprecationWarning", + "error", + # "ignore:.*imp module.*:DeprecationWarning", + # "ignore:.*invalid escape sequence.*:DeprecationWarning", ] [tool.mypy] -python_version = "3.9" +python_version = "3.10" ignore_missing_imports = true scripts_are_modules = true warn_unused_configs = true diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 5238970..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -rdflib>=6.0.1,<8 diff --git a/requirements_dev.txt b/requirements_dev.txt deleted file mode 100644 index 3c2b22d..0000000 --- a/requirements_dev.txt +++ /dev/null @@ -1,4 +0,0 @@ -pre-commit~=3.6 -pylint~=3.0 -pytest~=7.4 -pytest-cov~=4.1 diff --git a/requirements_docs.txt b/requirements_docs.txt deleted file mode 100644 index 475bb86..0000000 --- a/requirements_docs.txt +++ /dev/null @@ -1,7 +0,0 @@ -invoke~=2.2 -mike~=2.0 -mkdocs~=1.5 -mkdocs-awesome-pages-plugin~=2.9 -mkdocs-material~=9.5 -mkdocs-minify-plugin~=0.7.2 -mkdocstrings[python]~=0.24.0 diff --git a/setup.py b/setup.py deleted file mode 100644 index 1b3d4d5..0000000 --- a/setup.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Setup file for installing the `turtle-canon` package.""" -from pathlib import Path -import re - -from setuptools import setup, find_packages - - -TOP_DIR = Path(__file__).parent.resolve() - -with open(TOP_DIR / "turtle_canon/__init__.py", "r", encoding="utf8") as handle: - VERSION = AUTHOR = AUTHOR_EMAIL = None - for line in handle.readlines(): - VERSION_match = re.match(r'__version__ = (\'|")(?P.+)(\'|")', line) - AUTHOR_match = re.match(r'__author__ = (\'|")(?P.+)(\'|")', line) - AUTHOR_EMAIL_match = re.match( - r'__author_email__ = (\'|")(?P.+)(\'|")', line - ) - - if VERSION_match is not None: - VERSION = VERSION_match - if AUTHOR_match is not None: - AUTHOR = AUTHOR_match - if AUTHOR_EMAIL_match is not None: - AUTHOR_EMAIL = AUTHOR_EMAIL_match - - for info, value in { - "version": VERSION, - "author": AUTHOR, - "author email": AUTHOR_EMAIL, - }.items(): - if value is None: - raise RuntimeError( - f"Could not determine {info} from " - f"{TOP_DIR / 'turtle_canon/__init__.py'} !" - ) - VERSION = VERSION.group("version") # type: ignore[union-attr] - AUTHOR = AUTHOR.group("author") # type: ignore[union-attr] - AUTHOR_EMAIL = AUTHOR_EMAIL.group("email") # type: ignore[union-attr] - -with open(TOP_DIR / "requirements.txt", "r", encoding="utf8") as handle: - BASE = [ - f"{_.strip()}" - for _ in handle.readlines() - if not _.startswith("#") and "git+" not in _ - ] - -with open(TOP_DIR / "requirements_docs.txt", "r", encoding="utf8") as handle: - DOCS = [ - f"{_.strip()}" - for _ in handle.readlines() - if not _.startswith("#") and "git+" not in _ - ] - -with open(TOP_DIR / "requirements_dev.txt", "r", encoding="utf8") as handle: - DEV = [ - f"{_.strip()}" - for _ in handle.readlines() - if not _.startswith("#") and "git+" not in _ - ] + DOCS - -setup( - name="turtle-canon", - version=VERSION, - author=AUTHOR, - author_email=AUTHOR_EMAIL, - url="https://github.com/CasperWA/turtle-canon", - description="A tool for canonizing Turtle (`.ttl`) ontology files.", - long_description=(TOP_DIR / "README.md").read_text(), - long_description_content_type="text/markdown", - packages=find_packages(), - include_package_data=True, - python_requires=">=3.9", - install_requires=BASE, - extras_require={"dev": DEV, "docs": DOCS}, - keywords="ontology turtle", - entry_points={ - "console_scripts": [ - "turtle-canon = turtle_canon.cli.cmd_turtle_canon:main", - ], - }, - classifiers=[ - "Development Status :: 2 - Pre-Alpha", - "Environment :: Console", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", - "Natural Language :: English", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.9", - "Topic :: Software Development", - "Topic :: Software Development :: Pre-processors", - "Topic :: Software Development :: Testing", - "Topic :: Software Development :: Version Control", - "Topic :: Text Processing", - "Topic :: Utilities", - ], -) From 43d2f00a9ba1bd8913affbddfb33026f83e0b06a Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 11 Jan 2024 14:39:22 +0100 Subject: [PATCH 02/14] Use SINTEF/ci-cd throughout where possible Still use dependabot to update dependencies though. --- .github/dependabot.yml | 7 +- .github/utils/release_tag_msg.txt | 6 - .github/utils/single_dependency_pr_body.txt | 5 - .github/utils/update_docs.sh | 32 -- .github/workflows/cd_release.yml | 142 ++------ .github/workflows/ci_automerge_dependabot.yml | 110 +++--- .github/workflows/ci_cd_updated_main.yml | 137 ++----- .github/workflows/ci_dependabot.yml | 115 ++---- .github/workflows/ci_tests.yml | 155 +++----- .pre-commit-config.yaml | 24 +- pyproject.toml | 2 - tasks.py | 334 ------------------ 12 files changed, 177 insertions(+), 892 deletions(-) delete mode 100644 .github/utils/release_tag_msg.txt delete mode 100644 .github/utils/single_dependency_pr_body.txt delete mode 100755 .github/utils/update_docs.sh delete mode 100644 tasks.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a3f8f0b..343fe07 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,11 +6,13 @@ updates: interval: weekly day: monday time: "05:00" - # Should be bigger than or equal to the total number of dependencies (currently 18) - open-pull-requests-limit: 30 + # Should be bigger than or equal to the total number of dependencies (currently 11) + open-pull-requests-limit: 15 target-branch: ci/dependabot-updates labels: - dependencies + - "CI/CD" + - skip_changelog - package-ecosystem: github-actions directory: "/" schedule: @@ -19,3 +21,4 @@ updates: target-branch: ci/dependabot-updates labels: - "CI/CD" + - skip_changelog diff --git a/.github/utils/release_tag_msg.txt b/.github/utils/release_tag_msg.txt deleted file mode 100644 index f896261..0000000 --- a/.github/utils/release_tag_msg.txt +++ /dev/null @@ -1,6 +0,0 @@ -TAG_NAME - -The full release changelog can be seen in the -[online docs](https://CasperWA.github.io/turtle-canon/CHANGELOG/) -and in the -[repository source file](https://github.com/CasperWA/turtle-canon/blob/TAG_NAME/CHANGELOG.md). diff --git a/.github/utils/single_dependency_pr_body.txt b/.github/utils/single_dependency_pr_body.txt deleted file mode 100644 index 93c1c74..0000000 --- a/.github/utils/single_dependency_pr_body.txt +++ /dev/null @@ -1,5 +0,0 @@ -### Update dependencies - -Automatically created PR from [`ci/dependabot-updates`](https://github.com/CasperWA/turtle-canon/tree/ci/dependabot-updates). - -For more information see the ["Dependabot updates" workflow](https://github.com/CasperWA/turtle-canon/blob/main/.github/workflows/ci_dependabot.yml). diff --git a/.github/utils/update_docs.sh b/.github/utils/update_docs.sh deleted file mode 100755 index 11539d5..0000000 --- a/.github/utils/update_docs.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -set -e - -echo -e "\n-o- Setting commit user -o-" -git config --local user.email "${GIT_USER_EMAIL}" -git config --local user.name "${GIT_USER_NAME}" - -echo -e "\n-o- Update 'API Reference' docs -o-" -invoke create-api-reference-docs --pre-clean - -echo -e "\n-o- Update 'index.md' landing page -o-" -invoke create-docs-index - -echo -e "\n-o- Commit update - Documentation -o-" -git add docs/api_reference docs/index.md -if [ -n "$(git status --porcelain --untracked-files=no --ignored=no docs)" ]; then - # Only commit if there's something to commit (git will return non-zero otherwise) - git commit -m "Release ${GITHUB_REF#refs/tags/} - Documentation" -fi - -echo -e "\n-o- Update version -o-" -invoke setver --ver="${GITHUB_REF#refs/tags/}" - -echo -e "\n-o- Commit updates - Version & Changelog -o-" -git add turtle_canon/__init__.py -git add CHANGELOG.md -git commit -m "Release ${GITHUB_REF#refs/tags/} - Changelog" - -echo -e "\n-o- Update version tag -o-" -TAG_MSG=.github/utils/release_tag_msg.txt -sed -i "s|TAG_NAME|${GITHUB_REF#refs/tags/}|" "${TAG_MSG}" -git tag -af -F "${TAG_MSG}" ${GITHUB_REF#refs/tags/} diff --git a/.github/workflows/cd_release.yml b/.github/workflows/cd_release.yml index 0b7e0e3..82803f0 100644 --- a/.github/workflows/cd_release.yml +++ b/.github/workflows/cd_release.yml @@ -5,117 +5,35 @@ on: types: - published -env: - PUBLISH_UPDATE_BRANCH: main - GIT_USER_NAME: CasperWA - GIT_USER_EMAIL: "casper+github@welzel.nu" - jobs: - - update-and-publish: - name: Update CHANGELOG and documentation - runs-on: ubuntu-latest + publish-docs: + name: External + uses: SINTEF/ci-cd/.github/workflows/cd_release.yml@v2.7.1 if: github.repository == 'CasperWA/turtle-canon' && startsWith(github.ref, 'refs/tags/v') - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install Python dependencies - run: | - python -m pip install -U pip - pip install -U setuptools wheel - pip install -U -e .[dev] - - - name: Update changelog - uses: CharMixer/auto-changelog-action@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - release_branch: ${{ env.PUBLISH_UPDATE_BRANCH }} - - - name: Update documentation and version - Commit changes and update tag - run: .github/utils/update_docs.sh - - - name: Push updates to '${{ env.PUBLISH_UPDATE_BRANCH }}' - uses: CasperWA/push-protected@v2 - with: - token: ${{ secrets.RELEASE_PAT }} - branch: ${{ env.PUBLISH_UPDATE_BRANCH }} - force: true - tags: true - - - name: Get tagged versions - run: echo "PREVIOUS_VERSION=$(git tag -l --sort -version:refname | sed -n 2p)" >> $GITHUB_ENV - - - name: Create release-specific changelog - uses: CharMixer/auto-changelog-action@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - release_branch: ${{ env.PUBLISH_UPDATE_BRANCH }} - since_tag: "${{ env.PREVIOUS_VERSION }}" - output: "release_changelog.md" - - - name: Append changelog to release body - run: | - gh api /repos/${{ github.repository }}/releases/${{ github.event.release.id }} --jq '.body' > release_body.md - cat release_changelog.md >> release_body.md - gh api /repos/${{ github.repository }}/releases/${{ github.event.release.id }} -X PATCH -F body='@release_body.md' - env: - GITHUB_TOKEN: ${{ secrets.RELEASE_PAT }} - - # - name: Build source distribution - # run: python ./setup.py sdist - - # - name: Publish package to TestPyPI - # uses: pypa/gh-action-pypi-publish@master - # with: - # user: __token__ - # password: ${{ secrets.TEST_PYPI_TOKEN }} - # repository_url: "https://test.pypi.org/legacy" - - # - name: Publish package to PyPI - # uses: pypa/gh-action-pypi-publish@master - # with: - # user: __token__ - # password: ${{ secrets.PYPI_TOKEN }} - - docs: - name: Build and deploy documentation - needs: update-and-publish - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ env.PUBLISH_UPDATE_BRANCH }} - token: ${{ secrets.RELEASE_PAT }} - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install Python dependencies - run: | - python -m pip install -U pip - pip install -U setuptools wheel - pip install -U -e .[docs] - - - name: Set up git user - run: | - git config --global user.name "${{ env.GIT_USER_NAME }}" - git config --global user.email "${{ env.GIT_USER_EMAIL }}" - - - name: Deploy documentation - run: | - mike deploy --push --remote origin --branch gh-pages --update-aliases --config-file mkdocs.yml ${GITHUB_REF#refs/tags/v} stable - mike deploy --push --remote origin --branch gh-pages --update-aliases --config-file mkdocs.yml latest ${{ env.PUBLISH_UPDATE_BRANCH }} + with: + # General + git_username: CasperWA + git_email: "casper.w.andersen@sintef.no" + release_branch: main + + # Python package + python_package: true + package_dirs: turtle_canon + install_extras: "[dev]" + python_version_build: "3.10" + version_update_changes: | + "README.md,latest stable version is \*\*.*\*\*\.,latest stable version is **{version}**." + build_libs: build + build_cmd: "python -m build" + changelog_exclude_labels: "skip_changelog,duplicate,question,invalid,wontfix" + publish_on_pypi: false + + # Documentation + update_docs: true + python_version_docs: "3.10" + doc_extras: "[docs]" + docs_framework: "mkdocs" + mkdocs_update_latest: true + + secrets: + PAT: ${{ secrets.RELEASE_PAT }} diff --git a/.github/workflows/ci_automerge_dependabot.yml b/.github/workflows/ci_automerge_dependabot.yml index ca0e7c0..1174354 100644 --- a/.github/workflows/ci_automerge_dependabot.yml +++ b/.github/workflows/ci_automerge_dependabot.yml @@ -1,79 +1,51 @@ -name: CI - Activate auto-merging for Dependabot PRs +name: CI - Activate auto-merging for dependencies PRs on: pull_request_target: - branches: [ci/dependabot-updates] + branches: [ci/dependency-updates] jobs: - update-dependabot-branch: - name: Update permanent dependabot branch + update-dependencies-branch: + name: External + uses: SINTEF/ci-cd/.github/workflows/ci_automerge_prs.yml@v2.7.1 if: github.repository_owner == 'CasperWA' && startsWith(github.event.pull_request.head.ref, 'dependabot/') && github.actor == 'dependabot[bot]' - runs-on: ubuntu-latest - - env: - DEPENDABOT_BRANCH: ci/dependabot-updates - GIT_USER_NAME: CasperWA - GIT_USER_EMAIL: "casper+github@welzel.nu" - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.ref }} - persist-credentials: false - - - name: Setup Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Set up git user info - run: | - git config --global user.name "${{ env.GIT_USER_NAME }}" - git config --global user.email "${{ env.GIT_USER_EMAIL }}" - - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip + with: + perform_changes: true + git_username: CasperWA + git_email: "casper.w.andersen@sintef.no" + changes: | + pip install --upgrade pip pip install -U setuptools wheel - - while IFS="" read -r line || [ -n "${line}" ]; do - if [[ "${line}" =~ ^invoke.*$ ]]; then - invoke="${line}" + pip install -U . + + RDFLIB_VERSION="$(python -c 'import rdflib; print(rdflib.__version__)')" + CANONIZED_FILENAME="turtle_canon_tests_canonized_${RDFLIB_VERSION}.ttl" + CANONIZED_FILEPATH="${{ github.workspace }}/tests/static/rdflib_canonized/${CANONIZED_FILENAME}" + CORE_TEST_ONTOLOGY_FILE="${{ github.workspace }}/tests/static/rdflib_canon_tests.ttl" + + if [ -f "${CANONIZED_FILEPATH}" ]; then + echo "Canonized test file for RDFlib version ${RDFLIB_VERSION} already exists, checking contents..." + + # Copy core test ontology file into a temporary file and canonize it + TEMP_FILEPATH="/tmp/${CANONIZED_FILENAME}.ttl" + cp "${CORE_TEST_ONTOLOGY_FILE}" "${TEMP_FILEPATH}" + turtle-canon "${TEMP_FILEPATH}" + + if [ "$(diff "${TEMP_FILEPATH}" "${CANONIZED_FILEPATH}")" ]; then + echo "The existing canonized test file for RDFlib version ${RDFLIB_VERSION} differs from the currently generated one using the same version !" + rm -f "${TEMP_FILEPATH}" + exit 1 + else + echo "Canonized test file for RDFlib version ${RDFLIB_VERSION} is up-to-date, nothing to do." + rm -f "${TEMP_FILEPATH}" + exit 0 fi - done < requirements_docs.txt - - pip install ${invoke} - pip install -r requirements.txt - pip install . - - - name: Run tasks (and commit) - run: | - invoke update-pytest-reqs - invoke create-canonized-test-file - - git add pyproject.toml tests/static/rdflib_canonized - if [ -n "$(git status --porcelain pyproject.toml tests/static/rdflib_canonized)" ]; then - # Only commit if there's something to commit (git will return non-zero otherwise) - echo "Committing update to pytest dependency config and/or RDFlib version-specific canonized test file !" - git commit -m "Update version-specific parts\n\nUpdate pytest dependency config and/or\nRDFlib version-specific canonized test file" - echo "PUSH_BACK_TO_BRANCH=true" >> $GITHUB_ENV else - echo "No changes to pytest dependency config and/or RDFlib version-specific canonized test file." - echo "PUSH_BACK_TO_BRANCH=false" >> $GITHUB_ENV - fi - - - name: Update Dependabot branch - if: env.PUSH_BACK_TO_BRANCH == 'true' - uses: ad-m/github-push-action@master - with: - github_token: ${{ secrets.RELEASE_PAT }} - branch: ${{ github.event.pull_request.head.ref }} + echo "No canonized test file for RDFlib version ${RDFLIB_VERSION} found, creating it..." - - name: Activate auto-merge - run: | - PR_ID="$(gh api graphql -F owner='{owner}' -F name='{repo}' -f query='query($owner: String!, $name: String!) {repository(owner: $owner, name: $name) {pullRequest(number: ${{ github.event.pull_request.number }}) {id}}}' --jq '.data.repository.pullRequest.id')" - gh api graphql -f pr_id="$PR_ID" -f query='mutation($pr_id: ID!) {enablePullRequestAutoMerge(input:{mergeMethod:SQUASH,pullRequestId:$pr_id }) {pullRequest {number}}}' - env: - GITHUB_TOKEN: ${{ secrets.RELEASE_PAT }} + cp "${CORE_TEST_ONTOLOGY_FILE}" "${CANONIZED_FILEPATH}" + turtle-canon "${CANONIZED_FILEPATH}" + exit 0 + fi + secrets: + PAT: ${{ secrets.RELEASE_PAT }} diff --git a/.github/workflows/ci_cd_updated_main.yml b/.github/workflows/ci_cd_updated_main.yml index c629f8c..e0ec512 100644 --- a/.github/workflows/ci_cd_updated_main.yml +++ b/.github/workflows/ci_cd_updated_main.yml @@ -4,115 +4,30 @@ on: push: branches: [main] -env: - DEPENDABOT_BRANCH: ci/dependabot-updates - GIT_USER_NAME: CasperWA - GIT_USER_EMAIL: "casper+github@welzel.nu" - DEFAULT_REPO_BRANCH: main - jobs: - update-dependabot-branch: - name: Update permanent dependabot branch - if: github.repository_owner == 'CasperWA' - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ env.DEPENDABOT_BRANCH }} - fetch-depth: 0 - - - name: Set up git config - run: | - git config --global user.name "${{ env.GIT_USER_NAME }}" - git config --global user.email "${{ env.GIT_USER_EMAIL }}" - - - name: Update '${{ env.DEPENDABOT_BRANCH }}' - run: | - git fetch origin - - LATEST_PR_BODY="$(gh api /repos/${{ github.repository}}/pulls -X GET -f state=closed -f per_page=1 -f sort=updated -f direction=desc --jq '.[].body')" - if [ "${LATEST_PR_BODY}" == "$(cat .github/utils/single_dependency_pr_body.txt)" ]; then - # The dependency branch has just been merged into ${DEFAULT_REPO_BRANCH} - # The dependency branch should be reset to ${DEFAULT_REPO_BRANCH} - echo "The dependencies have just been updated! Reset to ${DEFAULT_REPO_BRANCH}." - git reset --hard origin/${DEFAULT_REPO_BRANCH} - echo "FORCE_PUSH=yes" >> $GITHUB_ENV - else - # Normal procedure: Merge `${DEFAULT_REPO_BRANCH}` into `${DEPENDABOT_BRANCH}` - echo "Merge new updates to ${DEFAULT_REPO_BRANCH} into ${DEPENDABOT_BRANCH}" - git merge -m "Keep '${{ env.DEPENDABOT_BRANCH }}' up-to-date with '${{ env.DEFAULT_REPO_BRANCH }}'" origin/${{ env.DEFAULT_REPO_BRANCH }} - echo "FORCE_PUSH=no" >> $GITHUB_ENV - fi - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Push to '${{ env.DEPENDABOT_BRANCH }}' - uses: CasperWA/push-protected@v2 - with: - token: ${{ secrets.RELEASE_PAT }} - branch: ${{ env.DEPENDABOT_BRANCH }} - sleep: 15 - force: ${{ env.FORCE_PUSH }} - - deploy-docs: - name: Deploy `latest` documentation - if: github.repository_owner == 'CasperWA' - runs-on: ubuntu-latest - - steps: - - name: Release check - run: | - COMMIT_MSG="$(gh api /repos/${{ github.repository}}/commits/main --jq '.commit.message')" - if [[ "${COMMIT_MSG}" =~ ^Release\ v.*\ -\ Changelog$ ]]; then - echo "In a release - do not run this job !" - echo "RELEASE_RUN=true" >> $GITHUB_ENV - else - echo "Not a release - update docs" - echo "RELEASE_RUN=false" >> $GITHUB_ENV - fi - env: - GITHUB_TOKEN: ${{ secrets.RELEASE_PAT }} - - - name: Checkout repository - if: env.RELEASE_RUN == 'false' - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.RELEASE_PAT }} - - - name: Set up Python 3.10 - if: env.RELEASE_RUN == 'false' - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install dependencies - if: env.RELEASE_RUN == 'false' - run: | - python -m pip install --upgrade pip - pip install -U setuptools wheel - pip install -U -e .[docs] - - - name: Set up git user - if: env.RELEASE_RUN == 'false' - run: | - git config --global user.name "${GIT_USER_NAME}" - git config --global user.email "${GIT_USER_EMAIL}" - - - name: Check API Reference and landing page - if: env.RELEASE_RUN == 'false' - run: | - invoke create-api-reference-docs --pre-clean - invoke create-docs-index - - if [ -n "$(git status --porcelain docs/api_reference docs/index.md)" ]; then - echo "The following file(s) in the documentation have not been committed:" - git status --porcelain docs/api_reference docs/index.md - exit 1 - fi - - - name: Deploy documentation - if: env.RELEASE_RUN == 'false' - run: mike deploy --push --remote origin --branch gh-pages --update-aliases --config-file mkdocs.yml latest main + update-deps-branch-and-docs: + name: External + uses: SINTEF/ci-cd/.github/workflows/ci_cd_updated_default_branch.yml@v2.7.1 + if: github.repository_owner == 'SINTEF' + with: + # General + git_username: CasperWA + git_email: "casper.w.andersen@sintef.no" + default_repo_branch: main + + # Update dependency branch + update_dependencies_branch: true + permanent_dependencies_branch: "ci/dependabot-updates" + + # Update documentation + update_docs: true + update_python_api_ref: true + update_docs_landing_page: true + package_dirs: turtle_canon + python_version: "3.10" + doc_extras: "[docs]" + changelog_exclude_labels: "skip_changelog,duplicate,question,invalid,wontfix" + docs_framework: mkdocs + + secrets: + PAT: ${{ secrets.RELEASE_PAT }} diff --git a/.github/workflows/ci_dependabot.yml b/.github/workflows/ci_dependabot.yml index 369f58b..275c6d6 100644 --- a/.github/workflows/ci_dependabot.yml +++ b/.github/workflows/ci_dependabot.yml @@ -1,4 +1,4 @@ -name: CI - Single Dependabot PR +name: CI - Update accummulated dependencies PR on: schedule: @@ -6,100 +6,27 @@ on: # Dependabot runs once a week (every Monday) (pip) # and every day (GH Actions) at 7:00 (5:00 UTC) - cron: "30 6 * * 3" + workflow_dispatch: jobs: - create-collected-pr: - name: Single dependabot PR + name: External + uses: SINTEF/ci-cd/.github/workflows/ci_update_dependencies.yml@v2.7.1 if: github.repository_owner == 'CasperWA' - runs-on: ubuntu-latest - env: - DEPENDABOT_BRANCH: ci/dependabot-updates - GIT_USER_NAME: CasperWA - GIT_USER_EMAIL: "casper+github@welzel.nu" - DEFAULT_REPO_BRANCH: main - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - ref: ${{ env.DEFAULT_REPO_BRANCH }} - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install `pre-commit` and dependencies - run: | - python -m pip install -U pip - pip install -U setuptools wheel - - while IFS="" read -r line || [ -n "${line}" ]; do - if [[ "${line}" =~ ^pre-commit.*$ ]]; then - pre_commit="${line}" - fi - done < requirements_dev.txt - - while IFS="" read -r line || [ -n "${line}" ]; do - if [[ "${line}" =~ ^invoke.*$ ]]; then - invoke="${line}" - fi - done < requirements_docs.txt - - pip install ${pre_commit} ${invoke} - - - name: Set up git user info - run: | - git config --global user.name "${{ env.GIT_USER_NAME }}" - git config --global user.email "${{ env.GIT_USER_EMAIL }}" - - - name: Reset to '${{ env.DEPENDABOT_BRANCH }}' - run: | - git fetch origin ${{ env.DEPENDABOT_BRANCH }}:${{ env.DEPENDABOT_BRANCH }} - git reset --hard ${{ env.DEPENDABOT_BRANCH }} - - - name: Auto-update `pre-commit` hooks - run: | - pre-commit autoupdate - - if [ -n "$(git status --porcelain .pre-commit-config.yaml)" ]; then - # Set environment variable notifying next steps that the hooks changed - echo "Pre-commit hooks have been updated !" - echo "UPDATED_PRE_COMMIT_HOOKS=true" >> $GITHUB_ENV - else - echo "No pre-commit hooks have been updated." - echo "UPDATED_PRE_COMMIT_HOOKS=false" >> $GITHUB_ENV - fi - - - name: Possibly run `pre-commit` with updated hooks - if: env.UPDATED_PRE_COMMIT_HOOKS == 'true' - continue-on-error: true # Still create the PR if this step fails - run: SKIP=pylint pre-commit run --all-files - - - name: Possibly commit changes and updates - if: env.UPDATED_PRE_COMMIT_HOOKS == 'true' - run: git commit -am "Update \`pre-commit\` hooks" - - - name: Fetch PR body - id: pr_body - uses: chuhlomin/render-template@v1.9 - with: - template: .github/utils/single_dependency_pr_body.txt - - - name: Create PR - id: cpr - uses: peter-evans/create-pull-request@v5 - with: - token: ${{ secrets.RELEASE_PAT }} - commit-message: New @dependabot-fueled updates - committer: "${{ env.GIT_USER_NAME }} <${{ env.GIT_USER_EMAIL }}>" - author: "${{ env.GIT_USER_NAME }} <${{ env.GIT_USER_EMAIL }}>" - branch: ci/update-dependencies - delete-branch: true - title: Update dependencies - body: ${{ steps.pr_body.outputs.result }} - labels: dependencies,CI/CD - - - name: Information - run: 'echo "${{ steps.cpr.outputs.pull-request-operation }} PR #${{ steps.cpr.outputs.pull-request-number }}: ${{ steps.cpr.outputs.pull-request-url }}"' + with: + # General + git_username: "CasperWA" + git_email: "casper.w.andersen@sintef.no" + permanent_dependencies_branch: "ci/dependabot-updates" + default_repo_branch: "main" + pr_labels: "CI/CD,skip_changelog" + extra_to_dos: "- [ ] Make sure that the PR is **squash** merged, with a sensible commit message." + + # Update pre-commit hooks + update_pre-commit: true + python_version: "3.10" + install_extras: "[dev]" + skip_pre-commit_hooks: "pylint" + + secrets: + PAT: ${{ secrets.RELEASE_PAT }} diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 92d3d1d..b83b874 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -8,66 +8,41 @@ on: - 'push-action/**' jobs: - pre-commit: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -U setuptools wheel - - while IFS="" read -r line || [ -n "${line}" ]; do - if [[ "${line}" =~ ^pre-commit.*$ ]]; then - pre_commit="${line}" - fi - done < requirements_dev.txt - - while IFS="" read -r line || [ -n "${line}" ]; do - if [[ "${line}" =~ ^invoke.*$ ]]; then - invoke="${line}" - fi - done < requirements_docs.txt - - pip install ${pre_commit} ${invoke} - - - name: Test with pre-commit - run: SKIP=pylint pre-commit run --all-files - - pylint-safety: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -U setuptools wheel - pip install -r requirements.txt -r requirements_docs.txt -r requirements_dev.txt - pip install safety - - - name: Run pylint - run: pylint --rcfile=pyproject.toml *.py turtle_canon - - - name: Run safety - # Remove ignoring 48547 as soon as RDFLib/rdflib#1844 has been fixed and the fix - # has been released. - run: pip freeze | safety check --stdin -i 48547 + basic-tests: + name: External + uses: SINTEF/ci-cd/.github/workflows/ci_tests.yml@v2.7.1 + with: + # General setup + install_extras: "[dev]" + + # pre-commit + run_pre-commit: true + python_version_pre-commit: "3.10" + skip_pre-commit_hooks: "pylint" + + # pylint & safety + python_version_pylint_safety: "3.10" + run_pylint: true + pylint_targets: "turtle_canon" + pylint_options: "--rcfile=pyproject.toml --recursive=yes" + + run_safety: true + + # Build dist + run_build_package: true + python_version_package: "3.10" + build_libs: "build" + build_cmd: "python -m build" + + # Build documentation + run_build_docs: true + python_version_docs: "3.10" + warnings_as_errors: true + use_mkdocs: true + + update_python_api_ref: true + update_docs_landing_page: true + package_dirs: "turtle_canon" pytest: runs-on: ubuntu-latest @@ -75,8 +50,6 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 - with: - fetch-depth: 2 - name: Set up Python 3.10 uses: actions/setup-python@v5 @@ -100,48 +73,19 @@ jobs: files: ./coverage.xml flags: turtle-canon - build-package: - name: Build source distribution - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Check build and install source distribution - uses: CasperWA/check-sdist-action@v1 - - docs: - name: Documentation - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -U setuptools wheel - pip install -e .[docs] - - - name: Build - run: | - invoke create-api-reference-docs --pre-clean - invoke create-docs-index - mkdocs build --strict --verbose - as-pre-commit-hook: - name: As a pre-commit hook + name: As a pre-commit hook (pre-commit ${{ matrix.pre-commit-version }}) runs-on: ubuntu-latest + strategy: + matrix: + pre-commit-version: + - "==1.13.0" # minimum pre-commit version for turtle-canon + # latest major versions + - "~=1.0" + - "~=2.0" + - "~=3.0" + steps: - name: Checkout repository uses: actions/checkout@v4 @@ -155,14 +99,7 @@ jobs: run: | python -m pip install --upgrade pip pip install -U setuptools wheel - - while IFS="" read -r line || [ -n "${line}" ]; do - if [[ "${line}" =~ ^pre-commit.*$ ]]; then - pre_commit="${line}" - fi - done < requirements_dev.txt - - pip install ${pre_commit} + pip install "pre-commit${{ matrix.pre-commit-version }}" - name: Set 'rev' to current commit SHA run: sed -i "s|COMMIT_SHA|${GITHUB_SHA}|" .github/utils/.pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a0aba10..b74306f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,6 +25,14 @@ repos: args: - --config=pyproject.toml +- repo: https://github.com/SINTEF/ci-cd + rev: v2.7.1 + hooks: + - id: docs-api-reference + args: + - --package-dir=turtle_canon + - id: docs-landing-page + - repo: https://github.com/PyCQA/bandit rev: '1.7.6' hooks: @@ -48,22 +56,6 @@ repos: types: [python] require_serial: true exclude: ^tests/.*$ - - id: update-docs-api-reference - name: Update API Reference in Documentation - entry: invoke - args: [create-api-reference-docs, --pre-clean, --pre-commit] - language: python - pass_filenames: false - files: ^turtle_canon/.*\.py$ - description: Update the API Reference documentation, whenever a Python file is touched in the code base. - - id: update-docs-index - name: Update Landing Page for Documentation - entry: invoke - args: [create-docs-index] - language: python - pass_filenames: false - files: ^README\.md$ - description: Update the landing page for the documentation if the source file (README.md) is changed. - id: codecov-validator name: Validate .codecov.yml description: Validate .codecov.yml using codecov's online validation tool. diff --git a/pyproject.toml b/pyproject.toml index 6f8a682..ec032b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,6 @@ dependencies = [ [project.optional-dependencies] docs = [ - "invoke~=2.2", "mike~=2.0", "mkdocs~=1.5", "mkdocs-awesome-pages-plugin~=2.9", @@ -76,7 +75,6 @@ target-version = ['py310'] [tool.pytest.ini_options] minversion = "7.4" -required_plugins = "pytest-cov>=4.1" addopts = "-rs --cov=turtle_canon --cov-report=term-missing" filterwarnings = [ "error", diff --git a/tasks.py b/tasks.py deleted file mode 100644 index 1b73a25..0000000 --- a/tasks.py +++ /dev/null @@ -1,334 +0,0 @@ -"""Invoke tasks. - -Update package version and update API reference documentation. -""" -from copy import deepcopy -import importlib -import os -import re -import shutil -import sys -from tempfile import TemporaryDirectory -from typing import TYPE_CHECKING -from pathlib import Path - -from invoke import task - -if TYPE_CHECKING: # pragma: no cover - from typing import Optional, Tuple, Union - - from invoke import Context, Result - - -TOP_DIR = Path(__file__).parent.resolve() - - -def update_file( - filename: "Union[Path, str]", - sub_line: "Tuple[str, str]", - strip: "Optional[str]" = None, -) -> None: - """Utility function for tasks to read, update, and write files""" - with open(filename, "r", encoding="utf8") as handle: - lines = [ - re.sub(sub_line[0], sub_line[1], line.rstrip(strip)) for line in handle - ] - - with open(filename, "w", encoding="utf8") as handle: - handle.write("\n".join(lines)) - handle.write("\n") - - -@task(help={"ver": "Turtle Canon version to set"}) -def setver(_, ver=""): - """Sets the Turtle Canon version""" - match = re.fullmatch( - ( - r"v?(?P[0-9]+(\.[0-9]+){2}" # Major.Minor.Patch - r"(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?" # pre-release - r"(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?)" # build metadata - ), - ver, - ) - if not match: - sys.exit( - "Error: Please specify version as 'Major.Minor.Patch(-Pre-Release+Build " - "Metadata)' or 'vMajor.Minor.Patch(-Pre-Release+Build Metadata)'" - ) - ver = match.group("version") - - update_file( - TOP_DIR / "turtle_canon/__init__.py", - (r'__version__ = (\'|").*(\'|")', f'__version__ = "{ver}"'), - ) - update_file( - TOP_DIR / "README.md", - ( - r"latest stable version is \*\*.*\*\*\.", - f"latest stable version is **{ver}**.", - ), - strip="\n", - ) - - print(f"Bumped version to {ver}") - - -@task( - help={ - "pre-clean": "Remove the 'api_reference' sub directory prior to (re)creation." - } -) -def create_api_reference_docs( - context, pre_clean=False, pre_commit=False -): # pylint: disable=too-many-locals - """Create the API Reference in the documentation""" - - def write_file(full_path: Path, content: str) -> None: - """Write file with `content` to `full_path`""" - if full_path.exists(): - with open(full_path, "r", encoding="utf8") as handle: - cached_content = handle.read() - if content == cached_content: - del cached_content - return - del cached_content - with open(full_path, "w", encoding="utf8") as handle: - handle.write(content) - - package_dir = TOP_DIR / "turtle_canon" - docs_api_ref_dir = TOP_DIR / "docs/api_reference" - - unwanted_subdirs = ("__pycache__",) - - pages_template = 'title: "{name}"\n' - md_template = "# {name}\n\n::: {py_path}\n" - models_template = ( - md_template + f"{' ' * 4}rendering:\n{' ' * 6}show_if_no_docstring: true\n" - ) - - if docs_api_ref_dir.exists() and pre_clean: - shutil.rmtree(docs_api_ref_dir, ignore_errors=True) - if docs_api_ref_dir.exists(): - sys.exit(f"{docs_api_ref_dir} should have been removed!") - docs_api_ref_dir.mkdir(exist_ok=True) - - for dirpath, dirnames, filenames in os.walk(package_dir): - for unwanted_dir in unwanted_subdirs: - if unwanted_dir in dirnames: - # Avoid walking into or through unwanted directories - dirnames.remove(unwanted_dir) - - relpath = Path(dirpath).relative_to(package_dir) - - # Create `.pages` - docs_sub_dir = docs_api_ref_dir / relpath - docs_sub_dir.mkdir(exist_ok=True) - if str(relpath) == ".": - write_file( - full_path=docs_api_ref_dir / ".pages", - content=pages_template.format(name="API Reference"), - ) - else: - write_file( - full_path=docs_sub_dir / ".pages", - content=pages_template.format( - name=str(relpath).rsplit("/", maxsplit=1)[-1] - ), - ) - - # Create markdown files - for filename in filenames: - if re.match(r".*\.py$", filename) is None or filename == "__init__.py": - # Not a Python file: We don't care about it! - # Or filename is `__init__.py`: We don't want it! - continue - - basename = filename[: -len(".py")] - py_path = ( - f"turtle_canon/{relpath}/{basename}".replace("/", ".") - if str(relpath) != "." - else f"turtle_canon/{basename}".replace("/", ".") - ) - md_filename = filename.replace(".py", ".md") - - # For models we want to include EVERYTHING, even if it doesn't have a - # doc-string - template = models_template if str(relpath) == "models" else md_template - - write_file( - full_path=docs_sub_dir / md_filename, - content=template.format(name=basename, py_path=py_path), - ) - - if pre_commit: - # Check if there have been any changes. - # List changes if yes. - if TYPE_CHECKING: # pragma: no cover - context: "Context" = context # type: ignore[no-redef] - - # NOTE: grep returns an exit code of 1 if it doesn't find anything - # (which will be good in this case). - # Concerning the weird last grep command see: - # http://manpages.ubuntu.com/manpages/precise/en/man1/git-status.1.html - result: "Result" = context.run( - "git status --porcelain docs/api_reference | " - "grep -E '^[? MARC][?MD]' || exit 0", - hide=True, - ) - if result.stdout: - sys.exit( - "The following files have been changed/added, please stage " - f"them:\n\n{result.stdout}" - ) - - -@task -def create_docs_index(_): - """Create the documentation index page from README.md""" - readme = TOP_DIR / "README.md" - docs_index = TOP_DIR / "docs/index.md" - - with open(readme, encoding="utf8") as handle: - content = handle.read() - - replacement_mapping = [ - ("docs/", ""), - ("(LICENSE)", "(LICENSE.md)"), - ] - - for old, new in replacement_mapping: - content = content.replace(old, new) - - with open(docs_index, "w", encoding="utf8") as handle: - handle.write(content) - - -@task -def update_pytest_reqs(_): - """Update the pytest plugins to be minimum the currently listed requirement - versions.""" - config = TOP_DIR / "pyproject.toml" - requirements = TOP_DIR / "requirements_dev.txt" - - # Retrieve dependencies specified in the config file - with open(config, encoding="utf8") as handle: - for line in handle.readlines(): - match = re.match(r'^required_plugins = "(?P.*)".*', line) - if match: - break - else: - raise RuntimeError( - "Couldn't find the required plugins for pytest in the config file at " - f"{config} !" - ) - - plugins = { - dependency.group("name"): dependency.group("version") - for dependency in [ - re.match(r"^(?P[a-z-]+)>=(?P[0-9]+(\.[0-9]+){1,2})$", _) - for _ in match.group("plugins").split(" ") # type: ignore[union-attr] - ] - if dependency - } - original_versions = deepcopy(plugins) - - # Update the retrieved versions with those from the requirements file - dependencies_found_counter = 0 - with open(requirements, encoding="utf8") as handle: - for line in handle.readlines(): - for plugin in plugins: - dependency = re.match( - rf"^{plugin}~=(?P[0-9]+(\.[0-9]+){{1,2}}).*", line - ) - if not dependency: - continue - dependencies_found_counter += 1 - plugins[plugin] = dependency.group("version") - - # Sanity check - if dependencies_found_counter != len(plugins): - raise RuntimeError( - f"Did not find all specified dependencies from the config file ({config}) " - f"in the development requirements file ({requirements}).\nDependencies " - f"found in the requirements file: {dependencies_found_counter}\n" - f"Dependencies found in the config file: {len(plugins)}" - ) - - # Update the config file dependency versions (if necessary) - for plugin in original_versions: - if original_versions[plugin] != plugins[plugin]: - break - else: - print("No updates detected; the config file is up-to-date.") - sys.exit() - - update_file( - config, - ( - r"^required_plugins = .*", - 'required_plugins = "' - f'{" ".join(f"{name}>={version}" for name, version in plugins.items())}"', - ), - ) - print( - f"Successfully updated pytest config plugins:\n {plugins}\n (was: " - f"{original_versions})" - ) - - -@task -def create_canonized_test_file(context): - """Canonize a standard test ontology file using the currently installed `rdflib`.""" - if TYPE_CHECKING: # pragma: no cover - context: "Context" = context # type: ignore[no-redef] - - try: - from rdflib import ( # pylint: disable=import-outside-toplevel - __version__ as rdflib_version, - ) - except ImportError: - sys.exit("RDFlib MUST be installed for this invoke task to work !") - - try: - importlib.import_module("turtle_canon") - except ImportError: - sys.exit("Turtle Canon MUST be installed for this invoke task to work !") - - canonized_filename = f"turtle_canon_tests_canonized_{rdflib_version}.ttl" - canonized_file = ( - TOP_DIR / "tests" / "static" / "rdflib_canonized" / canonized_filename - ) - - core_test_ontology_file = TOP_DIR / "tests" / "static" / "turtle_canon_tests.ttl" - - if canonized_file.exists(): - # Quickly check the contents of the file is the same as a fresh canonization - with TemporaryDirectory() as tmpdir: - newly_canonized_file = Path( - shutil.copy(core_test_ontology_file, Path(tmpdir) / canonized_filename) - ) - context.run(f"turtle-canon {newly_canonized_file}") - if newly_canonized_file.read_text( - encoding="utf8" - ) != canonized_file.read_text(encoding="utf8"): - sys.exit( - f"File for RDFlib version {rdflib_version} already exists, but" - "canonizing the core turtle file does not return the same content " - "as the existing file !" - ) - print(f"The RDFlib v{rdflib_version}-specific file already exists.") - sys.exit() - - # Generate new canonized file - shutil.copy(core_test_ontology_file, canonized_file) - if not canonized_file.exists(): - sys.exit( - "An error occured trying to copy " - f"{core_test_ontology_file.relative_to(TOP_DIR)} to " - f"{canonized_file.relative_to(TOP_DIR)} !" - ) - context.run(f"turtle-canon {canonized_file}") - print( - "Successfully created a new canonized test Turtle file for RDFlib " - f"v{rdflib_version}." - ) From 214c4946e5cb2c25d91832fd09bef9a6f2ecd9ad Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 11 Jan 2024 14:40:44 +0100 Subject: [PATCH 03/14] Use an explicit boolean value for key in .codecov.yml --- .codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.codecov.yml b/.codecov.yml index f05bcc9..925bc7f 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -10,7 +10,7 @@ coverage: status: project: - default: off + default: false turtle-canon: threshold: 1% flags: From be81b745f6e33f0e2ba535ebf04602ac097286ca Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 11 Jan 2024 15:06:52 +0100 Subject: [PATCH 04/14] Only support Python 3.10 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ec032b9..918f17b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ "Topic :: Text Processing", "Topic :: Utilities", ] -requires-python = "~=3.10" +requires-python = ">=3.10,<3.11" dynamic = ["version"] dependencies = [ From 45e756a21b8bccd594b5b0d58b8446e4bb0fb2b9 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 11 Jan 2024 15:51:10 +0100 Subject: [PATCH 05/14] Update code base according to ruff and pyupgrade Use ruff instead of pylint. --- .github/workflows/ci_dependabot.yml | 1 - .github/workflows/ci_tests.yml | 5 +- .pre-commit-config.yaml | 30 +++++++----- pyproject.toml | 41 ++++++++++++++-- tests/cli/conftest.py | 67 ++++++++++++++------------ tests/cli/test_cmd_turtle_canon.py | 71 ++++++++++++---------------- tests/conftest.py | 25 +++++----- tests/test_canon.py | 61 ++++++++++++++---------- tests/test_turtle_canon.py | 11 +++-- turtle_canon/__init__.py | 2 + turtle_canon/canon.py | 21 ++++---- turtle_canon/cli/cmd_turtle_canon.py | 12 ++--- turtle_canon/cli/utils.py | 41 ++++++++-------- turtle_canon/utils/exceptions.py | 1 + turtle_canon/utils/warnings.py | 1 + 15 files changed, 219 insertions(+), 171 deletions(-) diff --git a/.github/workflows/ci_dependabot.yml b/.github/workflows/ci_dependabot.yml index 275c6d6..9ba10b5 100644 --- a/.github/workflows/ci_dependabot.yml +++ b/.github/workflows/ci_dependabot.yml @@ -26,7 +26,6 @@ jobs: update_pre-commit: true python_version: "3.10" install_extras: "[dev]" - skip_pre-commit_hooks: "pylint" secrets: PAT: ${{ secrets.RELEASE_PAT }} diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index b83b874..1fed289 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -18,13 +18,10 @@ jobs: # pre-commit run_pre-commit: true python_version_pre-commit: "3.10" - skip_pre-commit_hooks: "pylint" # pylint & safety python_version_pylint_safety: "3.10" - run_pylint: true - pylint_targets: "turtle_canon" - pylint_options: "--rcfile=pyproject.toml --recursive=yes" + run_pylint: false run_safety: true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b74306f..8d2a122 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,17 +2,16 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: - - id: check-symlinks + - id: check-toml + name: Check TOML - id: check-yaml name: Check YAML - id: check-xml name: Check XML files: \.(xml|rdf|ttl)$ - - id: destroyed-symlinks + - id: debug-statements - id: end-of-file-fixer exclude: ^.*\.ttl$ - - id: requirements-txt-fixer - name: Fix requirements*.txt files: ^requirements.*\.txt$ - id: trailing-whitespace args: [--markdown-linebreak-ext=md] @@ -22,8 +21,6 @@ repos: rev: 23.12.1 hooks: - id: black - args: - - --config=pyproject.toml - repo: https://github.com/SINTEF/ci-cd rev: v2.7.1 @@ -33,6 +30,20 @@ repos: - --package-dir=turtle_canon - id: docs-landing-page +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.11 + hooks: + - id: ruff + args: + - --fix + - --show-fixes + +- repo: https://github.com/asottile/pyupgrade + rev: v3.15.0 + hooks: + - id: pyupgrade + args: [--py310-plus] + - repo: https://github.com/PyCQA/bandit rev: '1.7.6' hooks: @@ -49,13 +60,6 @@ repos: - repo: local hooks: - - id: pylint - name: pylint - entry: pylint - language: python - types: [python] - require_serial: true - exclude: ^tests/.*$ - id: codecov-validator name: Validate .codecov.yml description: Validate .codecov.yml using codecov's online validation tool. diff --git a/pyproject.toml b/pyproject.toml index 918f17b..b4f02aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,6 @@ docs = [ ] dev = [ "pre-commit~=3.6", - "pylint~=3.0", "pytest~=7.4", "pytest-cov~=4.1", "turtle-canon[docs]", @@ -75,7 +74,7 @@ target-version = ['py310'] [tool.pytest.ini_options] minversion = "7.4" -addopts = "-rs --cov=turtle_canon --cov-report=term-missing" +addopts = "-rs --cov=turtle_canon --cov-report=term-missing:skip-covered --no-cov-on-fail" filterwarnings = [ "error", # "ignore:.*imp module.*:DeprecationWarning", @@ -91,6 +90,38 @@ show_error_codes = true allow_redefinition = true check_untyped_defs = true -[tool.pylint.messages_control] -max-line-length = 88 -disable = [] +[tool.ruff.lint] +extend-select = [ + "E", # pycodestyle + "F", # pyflakes + "B", # flake8-bugbear + "BLE", # flake8-blind-except + "I", # isort + "ARG", # flake8-unused-arguments + "C4", # flake8-comprehensions + "ICN", # flake8-import-conventions + "G", # flake8-logging-format + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # pylint + "PT", # flake8-pytest-style + "PTH", # flake8-use-pathlib + "RET", # flake8-return + "RUF", # Ruff-specific + "SIM", # flake8-simplify + "T20", # flake8-print + "YTT", # flake8-2020 + "EXE", # flake8-executable + "PYI", # flake8-pyi +] +ignore = [ + "PLR", # Design related pylint codes +# "B008", # Performing function calls in argument defaults - done all the time in the CLI. +] +isort.required-imports = ["from __future__ import annotations"] + +[tool.ruff.lint.per-file-ignores] +"tests/**" = [ + "BLE", # flake8-blind-except + "T20", # flake8-print +] diff --git a/tests/cli/conftest.py b/tests/cli/conftest.py index b541978..1e43344 100644 --- a/tests/cli/conftest.py +++ b/tests/cli/conftest.py @@ -1,43 +1,51 @@ """PyTest fixtures for the CLI (`turtle_canon.cli`).""" -from collections import namedtuple +from __future__ import annotations + from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, NamedTuple import pytest -CLIOutput = namedtuple("CLIOutput", ["stdout", "stderr", "returncode"]) - -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: from subprocess import CalledProcessError, CompletedProcess - from typing import Callable, List, Optional, Union + from sys import _ExitCode + from typing import Protocol + + class CLIRunner(Protocol): + """Callable protocol for calling the `turtle-canon` CLI.""" + + def __call__( + self, + options: list[str] | None = None, + expected_error: str | None = None, + run_dir: Path | str | None = None, + use_subprocess: bool = True, + ) -> CalledProcessError | CLIOutput | CompletedProcess: + ... - from pytest import FixtureRequest - CLIRunner = Callable[ - [ - Optional[List[str]], - Optional[str], - Optional[Union[Path, str]], - bool, - ], - Union[CalledProcessError, CLIOutput, CompletedProcess], - ] +class CLIOutput(NamedTuple): + """Captured CLI output.""" + + stdout: str + stderr: str + returncode: _ExitCode | int @pytest.fixture(scope="session", params=[True, False]) -def clirunner(request: "FixtureRequest") -> "CLIRunner": +def clirunner(request: pytest.FixtureRequest) -> CLIRunner: """Call the `turtle-canon` CLI.""" - from contextlib import redirect_stderr, redirect_stdout import os - from subprocess import run, CalledProcessError + from contextlib import redirect_stderr, redirect_stdout + from subprocess import CalledProcessError, run from tempfile import TemporaryDirectory def _clirunner( - options: "Optional[List[str]]" = None, - expected_error: "Optional[str]" = None, - run_dir: "Optional[Union[Path, str]]" = None, + options: list[str] | None = None, + expected_error: str | None = None, + run_dir: Path | str | None = None, use_subprocess: bool = request.param, - ) -> "Union[CalledProcessError, CLIOutput, CompletedProcess]": + ) -> CalledProcessError | CLIOutput | CompletedProcess: """Call the `turtle-canon` CLI. Parameters: @@ -79,7 +87,7 @@ def _clirunner( if use_subprocess: try: output = run( - args=["turtle-canon"] + options, + args=["turtle-canon", *options], capture_output=True, check=True, cwd=run_dir.name @@ -120,18 +128,17 @@ def _clirunner( with TemporaryDirectory() as tmpdir: stdout_path = Path(tmpdir) / "out.txt" stderr_path = Path(tmpdir) / "err.txt" - original_cwd = os.getcwd() + original_cwd = Path.cwd() try: os.chdir( run_dir.name if isinstance(run_dir, TemporaryDirectory) else run_dir ) - with open(stdout_path, "w") as stdout, open( - stderr_path, "w" - ) as stderr: - with redirect_stdout(stdout), redirect_stderr(stderr): - cli(options if options else None) + with stdout_path.open("w") as stdout, stderr_path.open( + "w" + ) as stderr, redirect_stdout(stdout), redirect_stderr(stderr): + cli(options if options else None) except SystemExit as exit_: output = CLIOutput( stdout_path.read_text(), diff --git a/tests/cli/test_cmd_turtle_canon.py b/tests/cli/test_cmd_turtle_canon.py index 1711867..39d9b5d 100644 --- a/tests/cli/test_cmd_turtle_canon.py +++ b/tests/cli/test_cmd_turtle_canon.py @@ -1,27 +1,24 @@ """Test `turtle_canon.cli.cmd_turtle_canon` aka. the `turtle-canon` CLI.""" +from __future__ import annotations + from pathlib import Path from typing import TYPE_CHECKING -if TYPE_CHECKING: # pragma: no cover - from subprocess import CalledProcessError, CompletedProcess - from typing import List, Union - - from .conftest import CLIOutput, CLIRunner +if TYPE_CHECKING: + from .conftest import CLIRunner - CLIRunnerOutput = Union[CalledProcessError, CLIOutput, CompletedProcess] - -def test_version(clirunner: "CLIRunner") -> None: +def test_version(clirunner: CLIRunner) -> None: """Test `--version`.""" from turtle_canon import __version__ - output: "CLIRunnerOutput" = clirunner(["--version"]) + output = clirunner(["--version"]) assert output.stdout == f"Turtle Canon version {__version__}\n" -def test_absolute_path(clirunner: "CLIRunner", simple_turtle_file: Path) -> None: +def test_absolute_path(clirunner: CLIRunner, simple_turtle_file: Path) -> None: """Simple test run with minimalistic Turtle file.""" - output: "CLIRunnerOutput" = clirunner([str(simple_turtle_file)]) + output = clirunner([str(simple_turtle_file)]) assertion_help = ( f"STDOUT: {output.stdout}\nSTDERR: {output.stderr}\nRETURN_CODE: " @@ -34,7 +31,7 @@ def test_absolute_path(clirunner: "CLIRunner", simple_turtle_file: Path) -> None assert "Successful" in output.stdout, assertion_help -def test_relative_path(clirunner: "CLIRunner", simple_turtle_file: Path) -> None: +def test_relative_path(clirunner: CLIRunner, simple_turtle_file: Path) -> None: """Simple test run with minimalistic Turtle file.""" relative_path = simple_turtle_file.relative_to("/tmp") assert str(relative_path) == str( @@ -42,7 +39,7 @@ def test_relative_path(clirunner: "CLIRunner", simple_turtle_file: Path) -> None ) assert not relative_path.is_absolute() - output: "CLIRunnerOutput" = clirunner([str(relative_path)], run_dir="/tmp") + output = clirunner([str(relative_path)], run_dir="/tmp") assertion_help = ( f"STDOUT: {output.stdout}\nSTDERR: {output.stderr}\nRETURN_CODE: " @@ -55,7 +52,7 @@ def test_relative_path(clirunner: "CLIRunner", simple_turtle_file: Path) -> None assert "Successful" in output.stdout, assertion_help -def test_non_existant_file(clirunner: "CLIRunner") -> None: +def test_non_existant_file(clirunner: CLIRunner) -> None: """Ensure an error is printed with error code != 0 if the passed file does not exist.""" non_existant_file = Path(__file__).resolve().parent / "non-existant.ttl" @@ -65,9 +62,7 @@ def test_non_existant_file(clirunner: "CLIRunner") -> None: error_substring = f"Supplied file {non_existant_file.absolute()} not found." - output: "CLIRunnerOutput" = clirunner( - [str(non_existant_file)], expected_error=error_substring - ) + output = clirunner([str(non_existant_file)], expected_error=error_substring) assertion_help = ( f"STDOUT: {output.stdout}\nSTDERR: {output.stderr}\nRETURN_CODE: " @@ -75,15 +70,14 @@ def test_non_existant_file(clirunner: "CLIRunner") -> None: ) assert output.stderr, assertion_help - assert ( - error_substring in output.stderr and error_substring not in output.stdout - ), assertion_help + assert error_substring in output.stderr, assertion_help + assert error_substring not in output.stdout, assertion_help assert output.returncode == 1, assertion_help assert "ERROR" in output.stderr, assertion_help assert "Successful" not in output.stdout, assertion_help -def test_empty_file(clirunner: "CLIRunner", tmp_dir: Path) -> None: +def test_empty_file(clirunner: CLIRunner, tmp_dir: Path) -> None: """Ensure a warning is printed with error code != 0 if the passed file does not exist.""" empty_file = tmp_dir / "empty.ttl" @@ -97,7 +91,7 @@ def test_empty_file(clirunner: "CLIRunner", tmp_dir: Path) -> None: warning_substring = f"The Turtle file {empty_file.absolute()} is empty." - output: "CLIRunnerOutput" = clirunner([str(empty_file)]) + output = clirunner([str(empty_file)]) assertion_help = ( f"STDOUT: {output.stdout}\nSTDERR: {output.stderr}\nRETURN_CODE: " @@ -105,19 +99,18 @@ def test_empty_file(clirunner: "CLIRunner", tmp_dir: Path) -> None: ) assert output.stderr, assertion_help - assert ( - warning_substring in output.stderr and warning_substring not in output.stdout - ), assertion_help + assert warning_substring in output.stderr, assertion_help + assert warning_substring not in output.stdout, assertion_help assert output.returncode == 0 assert "WARNING" in output.stderr, assertion_help assert "Successful" not in output.stdout, assertion_help def test_multiple_files( - clirunner: "CLIRunner", single_turtle_permutations: "List[Path]" + clirunner: CLIRunner, single_turtle_permutations: list[Path] ) -> None: """Ensure passing multiple files to the CLI works.""" - output: "CLIRunnerOutput" = clirunner([str(_) for _ in single_turtle_permutations]) + output = clirunner([str(_) for _ in single_turtle_permutations]) assertion_help = ( f"STDOUT: {output.stdout}\nSTDERR: {output.stderr}\nRETURN_CODE: " @@ -133,7 +126,7 @@ def test_multiple_files( def test_fail_fast( - clirunner: "CLIRunner", single_turtle_permutations: "List[Path]", tmp_dir: Path + clirunner: CLIRunner, single_turtle_permutations: list[Path], tmp_dir: Path ) -> None: """Test `--fail-fast`.""" from copy import deepcopy @@ -152,8 +145,8 @@ def test_fail_fast( not error_file.exists() ), f"{error_file} was expected to not exist, but suprisingly it does !" - assert len(single_turtle_permutations) == 3 - original_single_turtle_permutation = deepcopy(single_turtle_permutations) + assert len(single_turtle_permutations) > 1 + original_single_turtle_permutations = deepcopy(single_turtle_permutations) single_turtle_permutations.insert(1, error_file) single_turtle_permutations.insert(-1, warning_file) @@ -161,7 +154,7 @@ def test_fail_fast( error_substring = f"Supplied file {error_file.absolute()} not found." - output: "CLIRunnerOutput" = clirunner( + output = clirunner( [str(_) for _ in single_turtle_permutations], expected_error=error_substring, ) @@ -172,22 +165,21 @@ def test_fail_fast( ) assert output.stderr, assertion_help - assert ( - error_substring in output.stderr and error_substring not in output.stdout - ), assertion_help + assert error_substring in output.stderr, assertion_help + assert error_substring not in output.stdout, assertion_help assert output.returncode == 1, assertion_help assert "ERROR" in output.stderr, assertion_help assert "*" in output.stderr, assertion_help assert output.stdout, assertion_help assert "Successful" not in output.stdout, assertion_help - for filename in original_single_turtle_permutation: + for filename in original_single_turtle_permutations: assert str(filename) in output.stdout, assertion_help for filename in set(single_turtle_permutations) - set( - original_single_turtle_permutation + original_single_turtle_permutations ): assert str(filename) not in output.stdout, assertion_help - output: "CLIRunnerOutput" = clirunner( + output = clirunner( ["--fail-fast"] + [str(_) for _ in single_turtle_permutations], expected_error=error_substring, ) @@ -198,9 +190,8 @@ def test_fail_fast( ) assert output.stderr, assertion_help - assert ( - error_substring in output.stderr and error_substring not in output.stdout - ), assertion_help + assert error_substring in output.stderr, assertion_help + assert error_substring not in output.stdout, assertion_help assert output.returncode == 1, assertion_help assert "ERROR" in output.stderr, assertion_help assert "*" not in output.stderr, assertion_help diff --git a/tests/conftest.py b/tests/conftest.py index 12094d6..602a025 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,13 @@ """Pytest fixtures and setup functions.""" +from __future__ import annotations + from pathlib import Path from typing import TYPE_CHECKING import pytest -if TYPE_CHECKING: # pragma: no cover - from typing import List +if TYPE_CHECKING: + pass @pytest.fixture(scope="session") @@ -14,7 +16,7 @@ def top_dir() -> Path: return Path(__file__).resolve().parent.parent -@pytest.fixture +@pytest.fixture() def simple_turtle_file(top_dir: Path) -> Path: """Load and return `Path` object to a simple Turtle file.""" import os @@ -30,13 +32,14 @@ def simple_turtle_file(top_dir: Path) -> Path: yield Path(copy(turtle_file, Path(tmpdir.name) / turtle_file.name)) finally: tmpdir.cleanup() - assert not Path( - tmpdir.name - ).exists(), f"Failed to remove temporary directory at {tmpdir.name}. Content:\n{os.listdir(tmpdir.name)}" + assert not Path(tmpdir.name).exists(), ( + f"Failed to remove temporary directory at {tmpdir.name}. " + f"Content:\n{os.listdir(tmpdir.name)}" + ) -@pytest.fixture -def single_turtle_permutations(top_dir: Path) -> "List[Path]": +@pytest.fixture() +def single_turtle_permutations(top_dir: Path) -> list[Path]: """Yield list of a single turtle file and permutations of it. The permutations are restructuring of the list of classes. @@ -62,7 +65,7 @@ def single_turtle_permutations(top_dir: Path) -> "List[Path]": ) -@pytest.fixture +@pytest.fixture() def tmp_dir() -> Path: """Open and yield a temporary directory.""" import os @@ -79,8 +82,8 @@ def tmp_dir() -> Path: ) -@pytest.fixture -def different_sources_ontologies(top_dir: Path) -> "List[Path]": +@pytest.fixture() +def different_sources_ontologies(top_dir: Path) -> list[Path]: """Yield list of Turtle files generated using different tools, i.e., coming from different sources.""" import os diff --git a/tests/test_canon.py b/tests/test_canon.py index 9474ab2..5ef9aa3 100644 --- a/tests/test_canon.py +++ b/tests/test_canon.py @@ -1,9 +1,11 @@ """Test `turtle_canon.canon` module functions.""" +from __future__ import annotations + from typing import TYPE_CHECKING import pytest -if TYPE_CHECKING: # pragma: no cover +if TYPE_CHECKING: from pathlib import Path @@ -23,7 +25,7 @@ def test_validate_turtle_no_file() -> None: validate_turtle(non_existant_file) -def test_validate_turtle_empty_file(tmp_dir: "Path") -> None: +def test_validate_turtle_empty_file(tmp_dir: Path) -> None: """Ensure a "warning" is raised if file is empty.""" from turtle_canon.canon import validate_turtle from turtle_canon.utils.warnings import EmptyFile @@ -37,7 +39,7 @@ def test_validate_turtle_empty_file(tmp_dir: "Path") -> None: validate_turtle(empty_file) -def test_validate_turtle(simple_turtle_file: "Path") -> None: +def test_validate_turtle(simple_turtle_file: Path) -> None: """Test `validate_turtle()` runs.""" from turtle_canon.canon import validate_turtle @@ -62,7 +64,7 @@ def test_export_ontology_not_found() -> None: export_ontology(ontology=Graph(), filename=non_existant_file) -def test_export_ontology_failed_export(tmp_dir: "Path") -> None: +def test_export_ontology_failed_export(tmp_dir: Path) -> None: """Ensure an exception is raised if the export fails, i.e., if the generated file is empty.""" from rdflib import Graph @@ -75,18 +77,22 @@ def test_export_ontology_failed_export(tmp_dir: "Path") -> None: with pytest.raises( FailedExportToFile, - match=f"Failed to properly save the loaded ontology from {empty_file.absolute()} to file.", + match=( + "Failed to properly save the loaded ontology from " + f"{empty_file.absolute()} to file." + ), ): export_ontology(ontology=Graph(), filename=empty_file) -def test_export_ontology(top_dir: "Path", tmp_dir: "Path") -> None: +def test_export_ontology(top_dir: Path, tmp_dir: Path) -> None: """Test `export_ontology()` runs.""" import os import re import shutil - from rdflib import __version__ as rdflib_version, Graph + from rdflib import Graph + from rdflib import __version__ as rdflib_version from turtle_canon.canon import export_ontology @@ -120,11 +126,11 @@ def test_export_ontology(top_dir: "Path", tmp_dir: "Path") -> None: ), f"The canonized Turtle file at {turtle_file} was unexpectadly changeable !" assert ( editable_turtle_file.exists() - ), f"The file has unexpectedly been removed after running `export_ontology()` !" + ), "The file has unexpectedly been removed after running `export_ontology()` !" assert editable_turtle_file.read_text(encoding="utf8") == original_canonized_content -def test_validate_turtle_binary_file(tmp_dir: "Path") -> None: +def test_validate_turtle_binary_file(tmp_dir: Path) -> None: """Ensure an exception is raised when the file cannot be decoded as UTF-8.""" from turtle_canon.canon import validate_turtle from turtle_canon.utils.exceptions import FailedReadingFile @@ -140,14 +146,16 @@ def test_validate_turtle_binary_file(tmp_dir: "Path") -> None: with pytest.raises( FailedReadingFile, - match=f"The Turtle file {binary_file.absolute()} could not be opened and read \(using UTF-8 encoding\).", + match=( + rf"The Turtle file {binary_file.absolute()} could not be opened and read " + r"\(using UTF-8 encoding\)." + ), ): validate_turtle(binary_file) -def test_validate_turtle_no_rw_file(tmp_dir: "Path") -> None: +def test_validate_turtle_no_rw_file(tmp_dir: Path) -> None: """Ensure an exception is raised when the file cannot be read or written to.""" - import os from turtle_canon.canon import validate_turtle from turtle_canon.utils.exceptions import FailedReadingFile @@ -163,21 +171,23 @@ def test_validate_turtle_no_rw_file(tmp_dir: "Path") -> None: ), f"No rw test file {no_rw_file} expected to be non-empty, but it was empty !" try: - os.chmod(no_rw_file, 0o000) # none + no_rw_file.chmod(0o000) # none with pytest.raises( FailedReadingFile, - match=f"The Turtle file {no_rw_file.absolute()} could not be opened and read \(using UTF-8 encoding\).", + match=( + rf"The Turtle file {no_rw_file.absolute()} could not be opened and " + r"read \(using UTF-8 encoding\)." + ), ): validate_turtle(str(no_rw_file)) finally: - os.chmod(no_rw_file, 0o644) + no_rw_file.chmod(0o644) assert no_rw_file.read_text() -def test_validate_turtle_no_w_file(tmp_dir: "Path") -> None: +def test_validate_turtle_no_w_file(tmp_dir: Path) -> None: """Ensure an exception is raised when the file cannot be read or written to.""" - import os from turtle_canon.canon import validate_turtle from turtle_canon.utils.exceptions import FailedReadingFile @@ -193,23 +203,23 @@ def test_validate_turtle_no_w_file(tmp_dir: "Path") -> None: ), f"No w test file {no_w_file} expected to be non-empty, but it was empty !" try: - os.chmod(no_w_file, 0o444) # read only + no_w_file.chmod(0o444) # read only with pytest.raises( FailedReadingFile, match=( f"The Turtle file {no_w_file.absolute()} could not be opened and " - "written to \(using UTF-8 encoding\)." + r"written to \(using UTF-8 encoding\)." ), ): validate_turtle(str(no_w_file)) finally: - os.chmod(no_w_file, 0o644) + no_w_file.chmod(0o644) assert no_w_file.read_text() assert no_w_file.write_text("test again") -def test_sort_ontology(simple_turtle_file: "Path") -> None: +def test_sort_ontology(simple_turtle_file: Path) -> None: """Test `sort_ontology()` runs.""" from rdflib import Graph @@ -219,7 +229,7 @@ def test_sort_ontology(simple_turtle_file: "Path") -> None: assert isinstance(ontology, Graph) -def test_sort_ontology_no_triples(tmp_dir: "Path") -> None: +def test_sort_ontology_no_triples(tmp_dir: Path) -> None: """Ensure a warning is raised (as an exception) if there are no triples. While the Turtle file in this test _is_ empty, this is not checked in @@ -242,10 +252,9 @@ def test_sort_ontology_no_triples(tmp_dir: "Path") -> None: sort_ontology(empty_file) -def test_sort_ontology_unparseable_file(tmp_dir: "Path") -> None: +def test_sort_ontology_unparseable_file(tmp_dir: Path) -> None: """Ensure exceptions from RDFlib during parsing an unparseable Turtle file are wrapped as Turtle Canon exceptions.""" - import os from turtle_canon.canon import sort_ontology from turtle_canon.utils.exceptions import FailedParsingFile @@ -254,7 +263,7 @@ def test_sort_ontology_unparseable_file(tmp_dir: "Path") -> None: unparseable_file.write_text("test") try: - os.chmod(unparseable_file, 0o000) # none + unparseable_file.chmod(0o000) # none with pytest.raises( FailedParsingFile, @@ -265,5 +274,5 @@ def test_sort_ontology_unparseable_file(tmp_dir: "Path") -> None: ): sort_ontology(unparseable_file) finally: - os.chmod(unparseable_file, 0o644) + unparseable_file.chmod(0o644) assert unparseable_file.read_text() diff --git a/tests/test_turtle_canon.py b/tests/test_turtle_canon.py index f923968..59e6624 100644 --- a/tests/test_turtle_canon.py +++ b/tests/test_turtle_canon.py @@ -1,11 +1,13 @@ """General tests for Turtle Canon.""" +from __future__ import annotations + from pathlib import Path from typing import TYPE_CHECKING import pytest -if TYPE_CHECKING: # pragma: no cover - from typing import List +if TYPE_CHECKING: + pass def test_repetitivity(simple_turtle_file: Path) -> None: @@ -37,7 +39,7 @@ def test_repetitivity(simple_turtle_file: Path) -> None: ).read_text(encoding="utf8") -def test_permutated_files(single_turtle_permutations: "List[Path]") -> None: +def test_permutated_files(single_turtle_permutations: list[Path]) -> None: """Ensure firing the Turtle Canon for a single file with content permutations renders the same result.""" from turtle_canon.canon import canonize @@ -101,6 +103,7 @@ def test_extra_whitespace(simple_turtle_file: Path) -> None: """Add random valid whitespace to a turtle file and ensure canonizing it doesn't change.""" from random import randrange + from turtle_canon.canon import canonize SPACE = " " @@ -203,7 +206,7 @@ def test_extra_whitespace(simple_turtle_file: Path) -> None: ), assertion_fail_msg -def test_different_sources(different_sources_ontologies: "List[Path]") -> None: +def test_different_sources(different_sources_ontologies: list[Path]) -> None: """Test that the canonization is true only to the given set of triples.""" import re diff --git a/turtle_canon/__init__.py b/turtle_canon/__init__.py index 3403c42..371236d 100644 --- a/turtle_canon/__init__.py +++ b/turtle_canon/__init__.py @@ -4,6 +4,8 @@ A tool for canonizing Turtle (`.ttl`) ontology files. """ +from __future__ import annotations + __version__ = "0.0.1" __author__ = "Casper Welzel Andersen" __author_email__ = "casper.w.andersen@sintef.no" diff --git a/turtle_canon/canon.py b/turtle_canon/canon.py index 74ca4c9..7aacdb4 100644 --- a/turtle_canon/canon.py +++ b/turtle_canon/canon.py @@ -1,19 +1,18 @@ """The main `turtle-canon` module.""" -from pathlib import Path +from __future__ import annotations + import re +from pathlib import Path from tempfile import TemporaryDirectory -from typing import TYPE_CHECKING from rdflib import Graph +from rdflib.exceptions import Error as RDFlibError +from rdflib.exceptions import ParserError from turtle_canon.utils import exceptions, warnings -if TYPE_CHECKING: # pragma: no cover - from typing import Union - - -def canonize(turtle_file: "Union[Path, str]") -> "Union[Path, None]": +def canonize(turtle_file: Path | str) -> Path | None: """The main function for running `turtle-canon`. Workflow: @@ -42,7 +41,7 @@ def canonize(turtle_file: "Union[Path, str]") -> "Union[Path, None]": return Path(turtle_file) if changed_file else None -def validate_turtle(turtle_file: "Union[Path, str]") -> Path: +def validate_turtle(turtle_file: Path | str) -> Path: """Validate a Turtle file. Parameters: @@ -93,7 +92,7 @@ def sort_ontology(turtle_file: Path) -> Graph: """ try: ontology = Graph().parse(location=str(turtle_file), format="turtle") - except Exception as exc: + except (SyntaxError, ParserError, RDFlibError) as exc: raise exceptions.FailedParsingFile( f"Failed to properly parse the Turtle file at {turtle_file}" ) from exc @@ -111,7 +110,7 @@ def sort_ontology(turtle_file: Path) -> Graph: try: for triple in triples: sorted_ontology.add(triple) - except Exception as exc: + except (AssertionError, RDFlibError) as exc: raise exceptions.FailedCreatingOntology( "Failed to properly create a sorted ontology from the triples in the " f"Turtle file at {turtle_file}" @@ -146,7 +145,7 @@ def export_ontology(ontology: Graph, filename: Path) -> bool: tmp_turtle_file = Path(tmp_dir) / "tmp_turtle_file.ttl" try: ontology.serialize(tmp_turtle_file, format="turtle") - except Exception as exc: + except (ValueError, RDFlibError) as exc: raise exceptions.FailedExportToFile( f"Failed to properly save the loaded ontology from {filename} to file." ) from exc diff --git a/turtle_canon/cli/cmd_turtle_canon.py b/turtle_canon/cli/cmd_turtle_canon.py index d46ab35..64cebaf 100644 --- a/turtle_canon/cli/cmd_turtle_canon.py +++ b/turtle_canon/cli/cmd_turtle_canon.py @@ -1,14 +1,14 @@ """Command line interface (CLI) for running `turtle-canon`.""" -# pylint: disable=import-outside-toplevel +from __future__ import annotations + import argparse import logging -from pathlib import Path import sys +from pathlib import Path from typing import TYPE_CHECKING if TYPE_CHECKING: # pragma: no cover from dataclasses import dataclass - from typing import List, Optional @dataclass class CLIArgs: @@ -17,13 +17,13 @@ class CLIArgs: version: str log_level: str fail_fast: bool - turtle_files: List[Path] + turtle_files: list[Path] LOGGING_LEVELS = [logging.getLevelName(level).lower() for level in range(0, 51, 10)] -def main(args: "Optional[List[str]]" = None) -> None: +def main(args: list[str] | None = None) -> None: """Turtle Canon - It's turtles all the way down.""" from turtle_canon import __version__ from turtle_canon.canon import canonize @@ -70,7 +70,7 @@ def main(args: "Optional[List[str]]" = None) -> None: metavar="TURTLE_FILE", ) - args: "CLIArgs" = parser.parse_args(args) # type: ignore[assignment] + args: CLIArgs = parser.parse_args(args) # type: ignore[assignment] cache = utils.Cache() diff --git a/turtle_canon/cli/utils.py b/turtle_canon/cli/utils.py index bc42dae..a096d7c 100644 --- a/turtle_canon/cli/utils.py +++ b/turtle_canon/cli/utils.py @@ -1,48 +1,51 @@ """Utility functions for `turtle-canon` CLI.""" -from pathlib import Path +from __future__ import annotations + import sys +from pathlib import Path from typing import TYPE_CHECKING if TYPE_CHECKING: # pragma: no cover - from typing import List, Optional, Sequence, TextIO, Union + from collections.abc import Sequence + from typing import TextIO class Cache: """Small cache.""" def __init__(self) -> None: - self._errors: "List[Union[str, Exception]]" = [] - self._warnings: "List[Union[str, Exception]]" = [] - self._files: "List[Union[str, Path]]" = [] + self._errors: list[str | Exception] = [] + self._warnings: list[str | Exception] = [] + self._files: list[str | Path] = [] @property - def errors(self) -> "List[Union[str, Exception]]": + def errors(self) -> list[str | Exception]: """Get `errors` attribute.""" return self._errors @property - def warnings(self) -> "List[Union[str, Exception]]": + def warnings(self) -> list[str | Exception]: """Get `warnings` attribute.""" return self._warnings @property - def files(self) -> "List[Union[str, Path]]": + def files(self) -> list[str | Path]: """Get `files` attribute.""" return self._files - def add_error(self, error: "Union[str, Exception]") -> None: + def add_error(self, error: str | Exception) -> None: """Add an error to the cache.""" if not isinstance(error, (str, Exception)): raise TypeError("error must be either a str or Exception") self._errors.append(error) - def add_warning(self, warning: "Union[str, Exception]") -> None: + def add_warning(self, warning: str | Exception) -> None: """Add a warning to the cache.""" if not isinstance(warning, (str, Exception)): raise TypeError("warning must be either a str or Exception") self._warnings.append(warning) - def add_file(self, file: "Union[str, Path]") -> None: + def add_file(self, file: str | Path) -> None: """Add a file to the cache.""" if not isinstance(file, (str, Path)): raise TypeError("file must be either a str or Exception") @@ -51,8 +54,8 @@ def add_file(self, file: "Union[str, Path]") -> None: def _print_message( message: str, - target: "Optional[TextIO]" = None, - prefix: "Optional[str]" = None, + target: TextIO | None = None, + prefix: str | None = None, exit_after: bool = False, ) -> None: """Print a message to target. @@ -72,7 +75,7 @@ def _print_message( sys.exit(1) -def print_error(message: "Union[str, Exception]", exit_after: bool = True) -> None: +def print_error(message: str | Exception, exit_after: bool = True) -> None: """Print an error message to the console. Parameters: @@ -86,7 +89,7 @@ def print_error(message: "Union[str, Exception]", exit_after: bool = True) -> No _print_message(res, target=sys.stderr, prefix="ERROR: ", exit_after=exit_after) -def print_warning(message: "Union[str, Exception]", exit_after: bool = False) -> None: +def print_warning(message: str | Exception, exit_after: bool = False) -> None: """Print a warnings message to the console. Parameters: @@ -101,8 +104,8 @@ def print_warning(message: "Union[str, Exception]", exit_after: bool = False) -> def print_summary( - errors: "Optional[Sequence[Union[str, Exception]]]" = None, - warnings: "Optional[Sequence[Union[str, Exception]]]" = None, + errors: Sequence[str | Exception] | None = None, + warnings: Sequence[str | Exception] | None = None, exit_after: bool = False, ) -> None: """Print a summary, including of error and/or warning messages. @@ -139,9 +142,7 @@ def print_summary( _print_message(res[:-1], target=target, prefix="", exit_after=exit_after) -def print_changed_files( - files: "Sequence[Union[str, Path]]", exit_after: bool = False -) -> None: +def print_changed_files(files: Sequence[str | Path], exit_after: bool = False) -> None: """Print list of changed files. Parameters: diff --git a/turtle_canon/utils/exceptions.py b/turtle_canon/utils/exceptions.py index daa5907..a5d53e5 100644 --- a/turtle_canon/utils/exceptions.py +++ b/turtle_canon/utils/exceptions.py @@ -1,4 +1,5 @@ """Exceptions for general usage by the Turtle Canon tool.""" +from __future__ import annotations class TurtleCanonException(Exception): diff --git a/turtle_canon/utils/warnings.py b/turtle_canon/utils/warnings.py index 391b8a1..910452e 100644 --- a/turtle_canon/utils/warnings.py +++ b/turtle_canon/utils/warnings.py @@ -5,6 +5,7 @@ Instead, they are `Exception`s that will be caught and treated specially by the CLI. """ +from __future__ import annotations class TurtleCanonWarning(Exception): From d00e50e0cd0cbb03283d8ac2727da992437f4165 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 11 Jan 2024 15:53:41 +0100 Subject: [PATCH 06/14] Update codecov CI action input --- .github/workflows/ci_tests.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 1fed289..f6c6ab3 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -66,9 +66,12 @@ jobs: if: github.repository == 'CasperWA/turtle-canon' uses: codecov/codecov-action@v3 with: - name: turtle-canon - files: ./coverage.xml + fail_ci_if_error: true + env_vars: OS,PYTHON flags: turtle-canon + env: + OS: ubuntu-latest + PYTHON: "3.10" as-pre-commit-hook: name: As a pre-commit hook (pre-commit ${{ matrix.pre-commit-version }}) From 43e35157d68c65e42a92d3031d475af6e72488cc Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 11 Jan 2024 15:54:45 +0100 Subject: [PATCH 07/14] Add back the name input --- .github/workflows/ci_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index f6c6ab3..4cdddda 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -68,6 +68,7 @@ jobs: with: fail_ci_if_error: true env_vars: OS,PYTHON + name: turtle-canon flags: turtle-canon env: OS: ubuntu-latest From 68dd1bebe59027a05179070cc756233c4411d9da Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 11 Jan 2024 15:58:29 +0100 Subject: [PATCH 08/14] Add PermissionError to list of exceptions to catch --- turtle_canon/canon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turtle_canon/canon.py b/turtle_canon/canon.py index 7aacdb4..e596325 100644 --- a/turtle_canon/canon.py +++ b/turtle_canon/canon.py @@ -92,7 +92,7 @@ def sort_ontology(turtle_file: Path) -> Graph: """ try: ontology = Graph().parse(location=str(turtle_file), format="turtle") - except (SyntaxError, ParserError, RDFlibError) as exc: + except (SyntaxError, PermissionError, ParserError, RDFlibError) as exc: raise exceptions.FailedParsingFile( f"Failed to properly parse the Turtle file at {turtle_file}" ) from exc From 8c2d5b74c3a933d4ff462cb4b58fad9193fe7421 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 11 Jan 2024 15:59:18 +0100 Subject: [PATCH 09/14] Run a-pre-commit-hook job for all matrix strategies --- .github/workflows/ci_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 4cdddda..a5784d9 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -79,6 +79,7 @@ jobs: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: pre-commit-version: - "==1.13.0" # minimum pre-commit version for turtle-canon From 9a2848688f9123b31919a9a08c4a364a524d7d95 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 11 Jan 2024 16:04:51 +0100 Subject: [PATCH 10/14] Explicitly install git and write out pre-commit.log --- .github/workflows/ci_tests.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index a5784d9..62da720 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -97,6 +97,9 @@ jobs: with: python-version: "3.10" + - name: Install system dependencies + run: sudo apt-get update && sudo apt-get install -y git + - name: Install dependencies run: | python -m pip install --upgrade pip @@ -113,10 +116,16 @@ jobs: run: | if [ -z "$(git status --porcelain tests)" ]; then echo "Turtle Canon didn't canonize any files under tests/ !" + + echo "pre-commit log:" + cat "/home/runner/.cache/pre-commit/pre-commit.log" exit 1 fi if [ -n "$(git status --porcelain tests/static/rdflib_canonized)" ]; then echo "Turtle Canon canonized already canonized files under tests/static/rdflib_canonized !" + + echo "pre-commit log:" + cat "/home/runner/.cache/pre-commit/pre-commit.log" exit 1 fi From de7c321d53a5b64190f0b97169d454b385860277 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 11 Jan 2024 16:06:44 +0100 Subject: [PATCH 11/14] Up to minimum use pre-commit v1.14.0 --- .github/workflows/ci_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 62da720..749fe67 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -82,7 +82,7 @@ jobs: fail-fast: false matrix: pre-commit-version: - - "==1.13.0" # minimum pre-commit version for turtle-canon + - "==1.14.0" # minimum pre-commit version for turtle-canon # latest major versions - "~=1.0" - "~=2.0" From 5de44dd4ab393eecf080990988ad5b49564c0011 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 11 Jan 2024 16:08:54 +0100 Subject: [PATCH 12/14] test all other v1 versions up to latest --- .github/workflows/ci_tests.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 749fe67..bf7e224 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -82,7 +82,12 @@ jobs: fail-fast: false matrix: pre-commit-version: - - "==1.14.0" # minimum pre-commit version for turtle-canon + - "==1.15.0" # minimum pre-commit version for turtle-canon + - "==1.16.0" # minimum pre-commit version for turtle-canon + - "==1.17.0" # minimum pre-commit version for turtle-canon + - "==1.18.0" # minimum pre-commit version for turtle-canon + - "==1.19.0" # minimum pre-commit version for turtle-canon + - "==1.20.0" # minimum pre-commit version for turtle-canon # latest major versions - "~=1.0" - "~=2.0" From 1fca4fca08f18437875b8d61b8b03565064345d8 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 11 Jan 2024 16:10:35 +0100 Subject: [PATCH 13/14] Check v1.14 patch versions --- .github/workflows/ci_tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index bf7e224..b6ef1d7 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -82,12 +82,12 @@ jobs: fail-fast: false matrix: pre-commit-version: + - "==1.14.0" # minimum pre-commit version for turtle-canon + - "==1.14.1" # minimum pre-commit version for turtle-canon + - "==1.14.2" # minimum pre-commit version for turtle-canon + - "==1.14.3" # minimum pre-commit version for turtle-canon + - "==1.14.4" # minimum pre-commit version for turtle-canon - "==1.15.0" # minimum pre-commit version for turtle-canon - - "==1.16.0" # minimum pre-commit version for turtle-canon - - "==1.17.0" # minimum pre-commit version for turtle-canon - - "==1.18.0" # minimum pre-commit version for turtle-canon - - "==1.19.0" # minimum pre-commit version for turtle-canon - - "==1.20.0" # minimum pre-commit version for turtle-canon # latest major versions - "~=1.0" - "~=2.0" From df9473d07980c8d49d960f175d21e883a912aa23 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 11 Jan 2024 16:12:06 +0100 Subject: [PATCH 14/14] Update minimum pre-commit version in hook Update to a version that can be properly tested. --- .github/workflows/ci_tests.yml | 8 -------- .pre-commit-hooks.yaml | 2 +- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index b6ef1d7..e07d6ab 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -82,11 +82,6 @@ jobs: fail-fast: false matrix: pre-commit-version: - - "==1.14.0" # minimum pre-commit version for turtle-canon - - "==1.14.1" # minimum pre-commit version for turtle-canon - - "==1.14.2" # minimum pre-commit version for turtle-canon - - "==1.14.3" # minimum pre-commit version for turtle-canon - - "==1.14.4" # minimum pre-commit version for turtle-canon - "==1.15.0" # minimum pre-commit version for turtle-canon # latest major versions - "~=1.0" @@ -102,9 +97,6 @@ jobs: with: python-version: "3.10" - - name: Install system dependencies - run: sudo apt-get update && sudo apt-get install -y git - - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index b14148b..09720be 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -4,7 +4,7 @@ entry: turtle-canon language: python language_version: python3.10 - minimum_pre_commit_version: 1.13.0 + minimum_pre_commit_version: "1.15.0" require_serial: true files: \.ttl$ types: [text]