From abaafb5e1a255d25579acb2dbb1584c44c217938 Mon Sep 17 00:00:00 2001 From: Sorin Sbarnea Date: Mon, 5 Aug 2024 11:00:30 +0100 Subject: [PATCH] Unify testing jobs into a single workflows Related: #219 --- .github/workflows/integration-tests.yml | 46 ----- .github/workflows/tests.yml | 47 ----- .github/workflows/tox.yml | 245 +++++++++++++++++++++++- CONTRIBUTING.md | 35 ++-- test_requirements.txt | 2 + tox.ini | 25 ++- 6 files changed, 275 insertions(+), 125 deletions(-) delete mode 100644 .github/workflows/integration-tests.yml delete mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml deleted file mode 100644 index 05053994..00000000 --- a/.github/workflows/integration-tests.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Run integration tests -on: - push: - branches: ["main"] - pull_request: - branches: ["main"] - # Run the integration tests every 8 hours. - # This will help to identify faster if - # there is a CI failure related to a - # change in any dependency. - schedule: - - cron: '0 */8 * * *' -jobs: - integration: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Install Java - uses: actions/setup-java@v3 - with: - distribution: "zulu" - java-version: "17" - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - - name: Install package dependencies - run: | - sudo apt-get update - sudo apt-get --assume-yes --no-install-recommends install libsystemd0 libsystemd-dev pkg-config - - - name: Install test requirements - run: | - pip install -U pip - pip install wheel - pip install -r test_requirements.txt - - - name: Install collection - run: ansible-galaxy collection install . - - - name: run integration tests - run: pytest tests/integration -vvv -s diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 40d34876..00000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Run Ansible tests -on: - push: - branches: ["main"] - pull_request: - branches: ["main"] - # Run the unit tests every 8 hours. - # This will help to identify faster if - # there is a CI failure related to a - # change in any dependency. - schedule: - - cron: '0 */8 * * *' -jobs: - tests: - runs-on: ubuntu-latest - - strategy: - matrix: - python-version: - - "3.10" - - "3.9" - - defaults: - run: - working-directory: ansible_collections/ - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - path: ansible_collections/ansible/eda - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install ansible - run: python -m pip install ansible - - - name: Run sanity tests - run: ansible-test sanity - working-directory: ansible_collections/ansible/eda - - - name: Run unit tests - run: ansible-test units --venv -v --num-workers 1 - working-directory: ansible_collections/ansible/eda diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 4bf8fa01..123943fe 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -1,5 +1,5 @@ --- -name: Tox +name: tox on: push: branches: ["main"] @@ -14,19 +14,248 @@ on: # Run on demand workflow_dispatch: jobs: - tox: - runs-on: ubuntu-latest + prepare: + name: prepare + runs-on: ubuntu-22.04 + outputs: + matrix: ${{ steps.generate_matrix.outputs.matrix }} + steps: + - name: Determine matrix + id: generate_matrix + uses: coactions/dynamic-matrix@v1 + with: + min_python: "3.9" + max_python: "3.10" + default_python: "3.9" + other_names: | + lint + darglint + integration + platforms: linux + build: + name: ${{ matrix.name }} + runs-on: ${{ matrix.os || 'ubuntu-22.04' }} + needs: + - prepare + defaults: + run: + shell: ${{ matrix.shell || 'bash'}} + working-directory: ansible_collections/ansible/eda + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.prepare.outputs.matrix) }} + # max-parallel: 5 + # The matrix testing goal is to cover the *most likely* environments + # which are expected to be used by users in production. Avoid adding a + # combination unless there are good reasons to test it, like having + # proof that we failed to catch a bug by not running it. Using + # distribution should be preferred instead of custom builds. steps: - - uses: actions/checkout@v3 + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # needed by setuptools-scm + path: ansible_collections/ansible/eda + submodules: true - name: Install package dependencies run: | sudo apt-get update sudo apt-get --assume-yes --no-install-recommends install libsystemd0 libsystemd-dev pkg-config - - name: Install deps - run: python -m pip install tox + - name: Set pre-commit cache + uses: actions/cache@v4 + if: ${{ matrix.passed_name == 'lint' }} + with: + path: | + ~/.cache/pre-commit + key: pre-commit-${{ matrix.name || matrix.passed_name }}-${{ hashFiles('.pre-commit-config.yaml') }} - - name: Run tox + - name: Set up Python ${{ matrix.python_version || '3.10' }} + uses: actions/setup-python@v5 + with: + cache: pip + python-version: ${{ matrix.python_version || '3.10' }} + + - name: Install tox run: | - python -m tox + python3 -m pip install --upgrade pip + python3 -m pip install --upgrade "tox>=4.0.0" + + - name: Initialize tox envs ${{ matrix.passed_name }} + run: python3 -m tox -n --skip-missing-interpreters false -vv -e ${{ matrix.passed_name }} + timeout-minutes: 5 # average is under 1, but macos can be over 3 + + # sequential run improves browsing experience (almost no speed impact) + - name: tox -e ${{ matrix.passed_name }} + run: python3 -m tox -e ${{ matrix.passed_name }} + + - name: Archive logs + uses: actions/upload-artifact@v4 + with: + name: logs-${{ matrix.name }}.zip + path: | + .tox/**/log/ + .tox/**/.coverage* + .tox/**/coverage.xml + + - name: Report failure if git reports dirty status + run: | + if [[ -n $(git status -s) ]]; then + # shellcheck disable=SC2016 + echo -n '::error file=git-status::' + printf '### Failed as git reported modified and/or untracked files\n```\n%s\n```\n' "$(git status -s)" | tee -a "$GITHUB_STEP_SUMMARY" + exit 99 + fi + # https://github.com/actions/toolkit/issues/193 + check: + if: always() + permissions: + id-token: write + checks: read + + needs: + - build + + runs-on: ubuntu-latest + + steps: + # checkout needed for codecov action which needs codecov.yml file + - uses: actions/checkout@v4 + + - name: Set up Python # likely needed for coverage + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - run: pip3 install 'coverage>=7.5.1' + + - name: Merge logs into a single archive + uses: actions/upload-artifact/merge@v4 + with: + name: logs.zip + pattern: logs-*.zip + # artifacts like py312.zip and py312-macos do have overlapping files + separate-directories: true + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: logs.zip + path: . + + - name: Check for expected number of coverage.xml reports + run: | + JOBS_PRODUCING_COVERAGE=8 + if [ "$(find . -name coverage.xml | wc -l | bc)" -ne "${JOBS_PRODUCING_COVERAGE}" ]; then + echo "::error::Number of coverage.xml files was not the expected one (${JOBS_PRODUCING_COVERAGE}): $(find . -name coverage.xml |xargs echo)" + exit 1 + fi + + - name: Upload coverage data + uses: codecov/codecov-action@v4 + with: + name: ${{ matrix.passed_name }} + # verbose: true # optional (default = false) + fail_ci_if_error: true + use_oidc: true # cspell:ignore oidc + + - name: Check codecov.io status + if: github.event_name == 'pull_request' + uses: coactions/codecov-status@main + + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} + + - name: Delete Merged Artifacts + uses: actions/upload-artifact/merge@v4 + with: + delete-merged: true + + # lint: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + + # - name: Install package dependencies + # run: | + # sudo apt-get update + # sudo apt-get --assume-yes --no-install-recommends install libsystemd0 libsystemd-dev pkg-config + + # - name: Install deps + # run: python -m pip install tox + + # - name: Run tox + # run: | + # python -m tox -e lint,darglint + # tests: + # runs-on: ubuntu-latest + + # strategy: + # matrix: + # python-version: + # - "3.10" + # - "3.9" + + # defaults: + # run: + # working-directory: ansible_collections/ + + # steps: + # - name: Checkout + # uses: actions/checkout@v3 + # with: + # path: ansible_collections/ansible/eda + + # - name: Set up Python ${{ matrix.python-version }} + # uses: actions/setup-python@v4 + # with: + # python-version: ${{ matrix.python-version }} + + # # - name: Install ansible + # # run: python -m pip install ansible + + # # - name: Run sanity tests + # # run: ansible-test sanity + # # working-directory: ansible_collections/ansible/eda + + # - name: Run unit tests + # run: tox -e py + # # ansible-test units --venv -v --num-workers 1 + # working-directory: ansible_collections/ansible/eda + # integration: + # runs-on: ubuntu-latest + # steps: + # - name: Checkout + # uses: actions/checkout@v3 + + # - name: Install Java + # uses: actions/setup-java@v3 + # with: + # distribution: "zulu" + # java-version: "17" + + # - name: Set up Python + # uses: actions/setup-python@v4 + # with: + # python-version: "3.9" + + # - name: Install package dependencies + # run: | + # sudo apt-get update + # sudo apt-get --assume-yes --no-install-recommends install libsystemd0 libsystemd-dev pkg-config + + # - name: Install test requirements + # run: | + # pip install -U pip tox + # # pip install wheel + # # pip install -r test_requirements.txt + + # # - name: Install collection + # # run: ansible-galaxy collection install . + + # - name: run integration tests + # run: tox -e integration + # # pytest tests/integration -vvv -s diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6501a8c0..0d719aa2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,6 +4,12 @@ Contributions are welcome, and they are greatly appreciated! Every little bit he Every new feature should be tested and documented. New source plugins or source filters will be evaluated for inclusion in the collection and might be rejected. Please consider the option of creating a new collection for your plugins if it is not a good fit for this collection. +## Cloning + +Due to ansible-test own requirements, you must clone the repository into +a directory structure such `ansible_collections/ansible/eda`. This will allow +ansible-test to test your code without having to install the collection. + ## Pre-commit We recommend running pre-commit prior to submitting pull requests. A [pre-commit config](.pre-commit-config.yaml) file is included in this repository and the following steps will get you up and running with pre-commit quickly: @@ -18,7 +24,7 @@ We recommend running pre-commit prior to submitting pull requests. A [pre-commit Pre-commit is now set up to run each time you create a new commit. If you wish to run pre-commit against all tracked files in the repository without performing a commit, you can run: -``` +```shell pre-commit run --all ``` @@ -41,30 +47,21 @@ We recommend setting up a Python virtual environment to install the test depende pip install -r test_requirements.txt -### Integration tests +### Sanity and Unit tests -Integration tests require the addition of [docker](https://docs.docker.com/engine/install/) or [podman](https://podman.io/getting-started/installation) and [docker-compose](https://docs.docker.com/compose/install/). -We recommend installing the Python implementation of `docker-compose` via pip: +Sanity and unity tests can easily be run via tox: +```shell +tox -e py ``` -pip install docker-compose -``` - -Then install the collection directly from your local repo and execute the tests: -``` -ansible-galaxy collection install . -pytest tests/integration -``` +### Integration tests -### Sanity tests +Integration tests require the addition of [docker](https://docs.docker.com/engine/install/) or [podman](https://podman.io/getting-started/installation) and [docker-compose](https://docs.docker.com/compose/install/). -```sh -ansible-test sanity -``` -### Units tests +Then install the collection directly from your local repo and execute the tests: -```sh -ansible-test units +```shell +tox -e integration ``` diff --git a/test_requirements.txt b/test_requirements.txt index 2f3d8daf..0970a0ad 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -7,3 +7,5 @@ ansible requests ansible-rulebook tox +docker-compose +ansible-core diff --git a/tox.ini b/tox.ini index d129be9a..26745c8e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,16 +1,25 @@ # Recommended usage of this file is detailed in https://github.com/ansible/eda-partner-testing/blob/main/README.md. # The linter paths can be changed, but may result in false passes. # {posargs} in this case would be the path to collection root relative from the .github/workflows dir (`../..`) +# cspell: ignore TOXPYTHON [tox] -envlist = lint -requires = - lint - darglint +envlist = lint, darglint, py, integration skipsdist = true # this repo is not a python package [testenv] -basepython = python3.9, python3.10 +description = Run sanity and unit tests +basepython = + py39: {env:TOXPYTHON:python3.9} + py310: {env:TOXPYTHON:python3.10} +deps = -r test_requirements.txt +commands_pre = + bash -c 'test "$(basename $(cd ../.. && pwd))" == ansible_collections || { echo "Repository must be cloned inside a directory structure like ansible_collections/ansible/eda in order allow ansible-test to run."; exit 3;}' +commands = + ansible-test sanity + ansible-test units --venv -v --num-workers 1 +allowlist_externals = + bash [testenv:lint] deps = pre-commit @@ -20,3 +29,9 @@ commands = pre-commit run -a [testenv:darglint] deps = darglint commands = darglint -s numpy -z full extensions/eda/plugins + +[testenv:integration] +deps = -r test_requirements.txt +description = Run integration tests +commands = + pytest tests/integration -vvv -s