From 72ddca50f110e3bd3dbf13a73073f765202e7664 Mon Sep 17 00:00:00 2001 From: misohu Date: Fri, 9 Dec 2022 12:18:19 +0100 Subject: [PATCH] feat: use pip compile for all the parts of the CI pieline --- .github/workflows/branch_automation.yaml | 2 +- .github/workflows/on_pull_request.yaml | 6 +- .../publish.yaml | 6 +- .../publish.yaml | 2 +- pyproject.toml | 2 +- requirements-fmt.txt | 2 +- requirements-lint.txt | 2 +- requirements-test_batch_release_charms.in | 3 + requirements-test_batch_release_charms.txt | 149 ++++++++++++++++++ requirements-test_branch_creation copy.in | 6 + requirements-test_branch_creation copy.txt | 48 ++++++ scripts/release-charms/requirements.in | 3 + scripts/release-charms/requirements.txt | 82 +++++++++- tests/1dot4/pipelines/mnist.py | 1 - tox.ini | 14 +- 15 files changed, 307 insertions(+), 21 deletions(-) create mode 100644 requirements-test_batch_release_charms.in create mode 100644 requirements-test_batch_release_charms.txt create mode 100644 requirements-test_branch_creation copy.in create mode 100644 requirements-test_branch_creation copy.txt create mode 100644 scripts/release-charms/requirements.in diff --git a/.github/workflows/branch_automation.yaml b/.github/workflows/branch_automation.yaml index 8743808..3e9d1ec 100644 --- a/.github/workflows/branch_automation.yaml +++ b/.github/workflows/branch_automation.yaml @@ -12,7 +12,7 @@ on: jobs: branch-track-creation: name: Branch creation - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 env: KUBEFLOW_BOT_TOKEN: ${{ secrets.KUBEFLOW_BOT_TOKEN }} steps: diff --git a/.github/workflows/on_pull_request.yaml b/.github/workflows/on_pull_request.yaml index b51a4f9..a6ce6c0 100644 --- a/.github/workflows/on_pull_request.yaml +++ b/.github/workflows/on_pull_request.yaml @@ -6,7 +6,7 @@ on: jobs: lint: name: Lint Code - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - run: sudo apt update && sudo apt install tox @@ -14,7 +14,7 @@ jobs: branch_creation_tests: name: Branch Creation Script Tests - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - run: sudo apt update && sudo apt install tox @@ -22,7 +22,7 @@ jobs: batch_release_charms_tests: name: Batch Release Charms Script Tests - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - run: sudo apt update && sudo apt install tox diff --git a/cannon_runs/2021-12-06_publish_workflow_bundles/publish.yaml b/cannon_runs/2021-12-06_publish_workflow_bundles/publish.yaml index beaeebf..e3517e8 100644 --- a/cannon_runs/2021-12-06_publish_workflow_bundles/publish.yaml +++ b/cannon_runs/2021-12-06_publish_workflow_bundles/publish.yaml @@ -15,7 +15,7 @@ on: jobs: get-charm-matrix: name: Generate the Charm Matrix - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 outputs: charm_paths_json: ${{ steps.get-charm-matrix-step.outputs.CHARM_PATHS_JSON }} steps: @@ -26,7 +26,7 @@ jobs: publish-charm: name: Publish Charm - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 # Only publish to charmhub if we are pushing to a special branch or running PRs from something named `branch/*` if: (github.event_name == 'push') || (startsWith( github.head_ref, 'branch/' )) needs: get-charm-matrix @@ -44,7 +44,7 @@ jobs: publish-bundle: # TODO: This publishes always, even if nothing changed. Decouple bundle publishing from charm publishing in future? name: Publish Bundle - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 env: CHARMCRAFT_AUTH: ${{ secrets.CHARMCRAFT_CREDENTIALS }} steps: diff --git a/cannon_runs/2021-12-06_publish_workflow_charms/publish.yaml b/cannon_runs/2021-12-06_publish_workflow_charms/publish.yaml index a1cb819..5151da6 100644 --- a/cannon_runs/2021-12-06_publish_workflow_charms/publish.yaml +++ b/cannon_runs/2021-12-06_publish_workflow_charms/publish.yaml @@ -15,7 +15,7 @@ on: jobs: publish-charm: name: Publish Charm - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 # Only publish to charmhub if we are pushing to a special branch or running PRs from something named `branch/*` if: (github.event_name == 'push') || (startsWith( github.head_ref, 'branch/' )) steps: diff --git a/pyproject.toml b/pyproject.toml index 985afc7..bd95981 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ exclude = [".git", "__pycache__", ".tox", "build", "dist", "*.egg_info", "venv", select = ["E", "W", "F", "C", "N", "R", "D", "H"] # Ignore W503, E501 because using black creates errors with this # Ignore D107 Missing docstring in __init__ -ignore = ["W503", "E501", "D107", "D415"] +ignore = ["W503", "E501", "D107", "D415", "C801", "F541", "N812", ""] # D100, D101, D102, D103: Ignore missing docstrings in tests per-file-ignores = ["tests/*:D100,D101,D102,D103,D104"] docstring-convention = "google" diff --git a/requirements-fmt.txt b/requirements-fmt.txt index 53094cd..168c798 100644 --- a/requirements-fmt.txt +++ b/requirements-fmt.txt @@ -14,7 +14,7 @@ mypy-extensions==0.4.3 # via black pathspec==0.10.2 # via black -platformdirs==2.5.4 +platformdirs==2.6.0 # via black tomli==2.0.1 # via black diff --git a/requirements-lint.txt b/requirements-lint.txt index 5612ab9..4143e8f 100644 --- a/requirements-lint.txt +++ b/requirements-lint.txt @@ -36,7 +36,7 @@ pathspec==0.10.2 # black pep8-naming==0.13.2 # via -r ./requirements-lint.in -platformdirs==2.5.4 +platformdirs==2.6.0 # via # -r ./requirements-fmt.txt # black diff --git a/requirements-test_batch_release_charms.in b/requirements-test_batch_release_charms.in new file mode 100644 index 0000000..523506b --- /dev/null +++ b/requirements-test_batch_release_charms.in @@ -0,0 +1,3 @@ +pytest +pytest-mock +-r ./scripts/release-charms/requirements.txt \ No newline at end of file diff --git a/requirements-test_batch_release_charms.txt b/requirements-test_batch_release_charms.txt new file mode 100644 index 0000000..3319ea7 --- /dev/null +++ b/requirements-test_batch_release_charms.txt @@ -0,0 +1,149 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile --resolver=backtracking ./requirements-test_batch_release_charms.in +# +anyio==3.6.2 + # via + # -r ././scripts/release-charms/requirements.txt + # httpcore +attrs==22.1.0 + # via pytest +certifi==2022.12.7 + # via + # -r ././scripts/release-charms/requirements.txt + # httpcore + # httpx + # requests +cffi==1.15.1 + # via + # -r ././scripts/release-charms/requirements.txt + # pynacl +charmed-kubeflow-chisme==0.0.5 + # via -r ././scripts/release-charms/requirements.txt +charset-normalizer==2.1.1 + # via + # -r ././scripts/release-charms/requirements.txt + # requests +click==8.1.3 + # via + # -r ././scripts/release-charms/requirements.txt + # typer +deepdiff==6.2.1 + # via + # -r ././scripts/release-charms/requirements.txt + # charmed-kubeflow-chisme +deprecated==1.2.13 + # via + # -r ././scripts/release-charms/requirements.txt + # pygithub +exceptiongroup==1.0.4 + # via pytest +h11==0.14.0 + # via + # -r ././scripts/release-charms/requirements.txt + # httpcore +httpcore==0.16.2 + # via + # -r ././scripts/release-charms/requirements.txt + # httpx +httpx==0.23.1 + # via + # -r ././scripts/release-charms/requirements.txt + # lightkube +idna==3.4 + # via + # -r ././scripts/release-charms/requirements.txt + # anyio + # requests + # rfc3986 +iniconfig==1.1.1 + # via pytest +jinja2==3.1.2 + # via + # -r ././scripts/release-charms/requirements.txt + # charmed-kubeflow-chisme +lightkube==0.11.0 + # via + # -r ././scripts/release-charms/requirements.txt + # charmed-kubeflow-chisme +lightkube-models==1.25.4.4 + # via + # -r ././scripts/release-charms/requirements.txt + # lightkube +markupsafe==2.1.1 + # via + # -r ././scripts/release-charms/requirements.txt + # jinja2 +ops==1.5.4 + # via + # -r ././scripts/release-charms/requirements.txt + # charmed-kubeflow-chisme +ordered-set==4.1.0 + # via + # -r ././scripts/release-charms/requirements.txt + # deepdiff +packaging==22.0 + # via pytest +pluggy==1.0.0 + # via pytest +pycparser==2.21 + # via + # -r ././scripts/release-charms/requirements.txt + # cffi +pygithub==1.57 + # via -r ././scripts/release-charms/requirements.txt +pyjwt==2.6.0 + # via + # -r ././scripts/release-charms/requirements.txt + # pygithub +pynacl==1.5.0 + # via + # -r ././scripts/release-charms/requirements.txt + # pygithub +pytest==7.2.0 + # via + # -r ./requirements-test_batch_release_charms.in + # pytest-mock +pytest-mock==3.10.0 + # via -r ./requirements-test_batch_release_charms.in +pyyaml==6.0 + # via + # -r ././scripts/release-charms/requirements.txt + # lightkube + # ops +requests==2.28.1 + # via + # -r ././scripts/release-charms/requirements.txt + # pygithub +rfc3986[idna2008]==1.5.0 + # via + # -r ././scripts/release-charms/requirements.txt + # httpx +ruamel-yaml==0.17.21 + # via + # -r ././scripts/release-charms/requirements.txt + # charmed-kubeflow-chisme +ruamel-yaml-clib==0.2.7 + # via + # -r ././scripts/release-charms/requirements.txt + # ruamel-yaml +sniffio==1.3.0 + # via + # -r ././scripts/release-charms/requirements.txt + # anyio + # httpcore + # httpx +tomli==2.0.1 + # via pytest +typer==0.7.0 + # via -r ././scripts/release-charms/requirements.txt +urllib3==1.26.13 + # via + # -r ././scripts/release-charms/requirements.txt + # requests +wrapt==1.14.1 + # via + # -r ././scripts/release-charms/requirements.txt + # deprecated diff --git a/requirements-test_branch_creation copy.in b/requirements-test_branch_creation copy.in new file mode 100644 index 0000000..b835faa --- /dev/null +++ b/requirements-test_branch_creation copy.in @@ -0,0 +1,6 @@ +pyyaml +requests +GitPython +pytest +requests-mock +pytest-mock \ No newline at end of file diff --git a/requirements-test_branch_creation copy.txt b/requirements-test_branch_creation copy.txt new file mode 100644 index 0000000..51ee0c0 --- /dev/null +++ b/requirements-test_branch_creation copy.txt @@ -0,0 +1,48 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile --resolver=backtracking './requirements-test_branch_creation copy.in' +# +attrs==22.1.0 + # via pytest +certifi==2022.12.7 + # via requests +charset-normalizer==2.1.1 + # via requests +exceptiongroup==1.0.4 + # via pytest +gitdb==4.0.10 + # via gitpython +gitpython==3.1.29 + # via -r ./requirements-test_branch_creation copy.in +idna==3.4 + # via requests +iniconfig==1.1.1 + # via pytest +packaging==22.0 + # via pytest +pluggy==1.0.0 + # via pytest +pytest==7.2.0 + # via + # -r ./requirements-test_branch_creation copy.in + # pytest-mock +pytest-mock==3.10.0 + # via -r ./requirements-test_branch_creation copy.in +pyyaml==6.0 + # via -r ./requirements-test_branch_creation copy.in +requests==2.28.1 + # via + # -r ./requirements-test_branch_creation copy.in + # requests-mock +requests-mock==1.10.0 + # via -r ./requirements-test_branch_creation copy.in +six==1.16.0 + # via requests-mock +smmap==5.0.0 + # via gitdb +tomli==2.0.1 + # via pytest +urllib3==1.26.13 + # via requests diff --git a/scripts/release-charms/requirements.in b/scripts/release-charms/requirements.in new file mode 100644 index 0000000..c4b6e3c --- /dev/null +++ b/scripts/release-charms/requirements.in @@ -0,0 +1,3 @@ +charmed-kubeflow-chisme +PyGithub +typer diff --git a/scripts/release-charms/requirements.txt b/scripts/release-charms/requirements.txt index c4b6e3c..04c8f19 100644 --- a/scripts/release-charms/requirements.txt +++ b/scripts/release-charms/requirements.txt @@ -1,3 +1,79 @@ -charmed-kubeflow-chisme -PyGithub -typer +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile --resolver=backtracking ./scripts/release-charms/requirements.in +# +anyio==3.6.2 + # via httpcore +certifi==2022.12.7 + # via + # httpcore + # httpx + # requests +cffi==1.15.1 + # via pynacl +charmed-kubeflow-chisme==0.0.5 + # via -r ./scripts/release-charms/requirements.in +charset-normalizer==2.1.1 + # via requests +click==8.1.3 + # via typer +deepdiff==6.2.1 + # via charmed-kubeflow-chisme +deprecated==1.2.13 + # via pygithub +h11==0.14.0 + # via httpcore +httpcore==0.16.2 + # via httpx +httpx==0.23.1 + # via lightkube +idna==3.4 + # via + # anyio + # requests + # rfc3986 +jinja2==3.1.2 + # via charmed-kubeflow-chisme +lightkube==0.11.0 + # via charmed-kubeflow-chisme +lightkube-models==1.25.4.4 + # via lightkube +markupsafe==2.1.1 + # via jinja2 +ops==1.5.4 + # via charmed-kubeflow-chisme +ordered-set==4.1.0 + # via deepdiff +pycparser==2.21 + # via cffi +pygithub==1.57 + # via -r ./scripts/release-charms/requirements.in +pyjwt==2.6.0 + # via pygithub +pynacl==1.5.0 + # via pygithub +pyyaml==6.0 + # via + # lightkube + # ops +requests==2.28.1 + # via pygithub +rfc3986[idna2008]==1.5.0 + # via httpx +ruamel-yaml==0.17.21 + # via charmed-kubeflow-chisme +ruamel-yaml-clib==0.2.7 + # via ruamel-yaml +sniffio==1.3.0 + # via + # anyio + # httpcore + # httpx +typer==0.7.0 + # via -r ./scripts/release-charms/requirements.in +urllib3==1.26.13 + # via requests +wrapt==1.14.1 + # via deprecated diff --git a/tests/1dot4/pipelines/mnist.py b/tests/1dot4/pipelines/mnist.py index d0b6ccc..c41cc39 100644 --- a/tests/1dot4/pipelines/mnist.py +++ b/tests/1dot4/pipelines/mnist.py @@ -195,7 +195,6 @@ def test_task( ): """Connects to served model and tests example MNIST images.""" - import json import time import numpy as np diff --git a/tox.ini b/tox.ini index 475da33..ec80cd0 100644 --- a/tox.ini +++ b/tox.ini @@ -28,7 +28,9 @@ allowlist_externals = xargs commands = ; uses 'bash -c' because piping didn't work in regular tox commands - bash -c \'find . -type f -name "requirements*.in" | xargs --replace=\{\} pip-compile --resolver=backtracking \{\}' + pip-compile requirements-fmt.in + pip-compile {toxinidir}/scripts/release-charms/requirements.in + bash -c 'find . -type f -name "requirements*.in" | xargs --replace=\{\} pip-compile --resolver=backtracking \{\}' deps = pip-tools description = Update requirements files by executing pip-compile on all requirements*.in files, including those in subdirs. @@ -45,11 +47,11 @@ description = Apply coding style standards to code commands = # uncomment the following line if this charm owns a lib # codespell {[vars]lib_path} - codespell {toxinidir}/. --skip {toxinidir}/.git --skip {toxinidir}/.tox \ - --skip {toxinidir}/build --skip {toxinidir}/lib --skip {toxinidir}/venv \ - --skip {toxinidir}/.mypy_cache \ - --skip {toxinidir}/icon.svg --skip *.json.tmpl \ - --skip {toxinidir}/cannon_runs + codespell {toxinidir}/. --skip {toxinidir}/./.git --skip {toxinidir}/./.tox \ + --skip {toxinidir}/./build --skip {toxinidir}/./lib --skip {toxinidir}/./venv \ + --skip {toxinidir}/./.mypy_cache \ + --skip {toxinidir}/./icon.svg --skip *.json.tmpl \ + --skip {toxinidir}/./cannon_runs # pflake8 wrapper supports config from pyproject.toml pflake8 {[vars]all_path} isort --check-only --diff {[vars]all_path}