From d383d04ade375f4c20ce7164cbe7ebec884500d7 Mon Sep 17 00:00:00 2001 From: Sorin Sbarnea Date: Mon, 5 Aug 2024 11:00:30 +0100 Subject: [PATCH] Unify testing jobs into a single workflows Related: #219 --- .github/workflows/integration-tests.yml | 46 ------- .github/workflows/tests.yml | 47 ------- .github/workflows/tox.yml | 166 ++++++++++++++++++++++-- CONTRIBUTING.md | 35 +++-- requirements.txt | 2 +- test_requirements.txt | 1 + tox.ini | 52 +++++++- 7 files changed, 223 insertions(+), 126 deletions(-) delete mode 100644 .github/workflows/integration-tests.yml delete mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml deleted file mode 100644 index 05053994..00000000 --- a/.github/workflows/integration-tests.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Run integration tests -on: - push: - branches: ["main"] - pull_request: - branches: ["main"] - # Run the integration tests every 8 hours. - # This will help to identify faster if - # there is a CI failure related to a - # change in any dependency. - schedule: - - cron: '0 */8 * * *' -jobs: - integration: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Install Java - uses: actions/setup-java@v3 - with: - distribution: "zulu" - java-version: "17" - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - - name: Install package dependencies - run: | - sudo apt-get update - sudo apt-get --assume-yes --no-install-recommends install libsystemd0 libsystemd-dev pkg-config - - - name: Install test requirements - run: | - pip install -U pip - pip install wheel - pip install -r test_requirements.txt - - - name: Install collection - run: ansible-galaxy collection install . - - - name: run integration tests - run: pytest tests/integration -vvv -s diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 40d34876..00000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Run Ansible tests -on: - push: - branches: ["main"] - pull_request: - branches: ["main"] - # Run the unit tests every 8 hours. - # This will help to identify faster if - # there is a CI failure related to a - # change in any dependency. - schedule: - - cron: '0 */8 * * *' -jobs: - tests: - runs-on: ubuntu-latest - - strategy: - matrix: - python-version: - - "3.10" - - "3.9" - - defaults: - run: - working-directory: ansible_collections/ - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - path: ansible_collections/ansible/eda - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install ansible - run: python -m pip install ansible - - - name: Run sanity tests - run: ansible-test sanity - working-directory: ansible_collections/ansible/eda - - - name: Run unit tests - run: ansible-test units --venv -v --num-workers 1 - working-directory: ansible_collections/ansible/eda diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 4bf8fa01..43f9c0bf 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -1,5 +1,5 @@ --- -name: Tox +name: tox on: push: branches: ["main"] @@ -14,19 +14,169 @@ on: # Run on demand workflow_dispatch: jobs: - tox: - runs-on: ubuntu-latest + prepare: + name: prepare + runs-on: ubuntu-22.04 + outputs: + matrix: ${{ steps.generate_matrix.outputs.matrix }} + steps: + - name: Determine matrix + id: generate_matrix + uses: coactions/dynamic-matrix@v1 + with: + min_python: "3.9" + max_python: "3.10" + default_python: "3.9" + other_names: | + lint + darglint + integration + platforms: linux + build: + name: ${{ matrix.name }} + runs-on: ${{ matrix.os || 'ubuntu-22.04' }} + continue-on-error: ${{ contains(matrix.name, 'integration') && true || false }} + needs: + - prepare + defaults: + run: + shell: ${{ matrix.shell || 'bash'}} + working-directory: ansible_collections/ansible/eda + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.prepare.outputs.matrix) }} + # max-parallel: 5 + # The matrix testing goal is to cover the *most likely* environments + # which are expected to be used by users in production. Avoid adding a + # combination unless there are good reasons to test it, like having + # proof that we failed to catch a bug by not running it. Using + # distribution should be preferred instead of custom builds. steps: - - uses: actions/checkout@v3 + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # needed by setuptools-scm + path: ansible_collections/ansible/eda + submodules: true - name: Install package dependencies run: | sudo apt-get update sudo apt-get --assume-yes --no-install-recommends install libsystemd0 libsystemd-dev pkg-config - - name: Install deps - run: python -m pip install tox + - name: Install Java + if: ${{ matrix.passed_name == 'integration' }} + uses: actions/setup-java@v4 + with: + distribution: "zulu" + java-version: "17" + + - name: Set pre-commit cache + uses: actions/cache@v4 + if: ${{ matrix.passed_name == 'lint' }} + with: + path: | + ~/.cache/pre-commit + key: pre-commit-${{ matrix.name || matrix.passed_name }}-${{ hashFiles('.pre-commit-config.yaml') }} - - name: Run tox + - name: Set up Python ${{ matrix.python_version || '3.10' }} + uses: actions/setup-python@v5 + with: + cache: pip + python-version: ${{ matrix.python_version || '3.10' }} + + - name: Install tox run: | - python -m tox + python3 -m pip install --upgrade pip wheel tox + + - name: Initialize tox envs ${{ matrix.passed_name }} + run: python3 -m tox -n --skip-missing-interpreters false -vv -e ${{ matrix.passed_name }} + timeout-minutes: 5 # average is under 1, but macos can be over 3 + + # sequential run improves browsing experience (almost no speed impact) + - name: tox -e ${{ matrix.passed_name }} + run: python3 -m tox -e ${{ matrix.passed_name }} + + - name: Archive logs + uses: actions/upload-artifact@v4 + with: + name: logs-${{ matrix.name }}.zip + path: | + .tox/**/log/ + .tox/**/.coverage* + .tox/**/coverage.xml + + - name: Report failure if git reports dirty status + run: | + if [[ -n $(git status -s) ]]; then + # shellcheck disable=SC2016 + echo -n '::error file=git-status::' + printf '### Failed as git reported modified and/or untracked files\n```\n%s\n```\n' "$(git status -s)" | tee -a "$GITHUB_STEP_SUMMARY" + exit 99 + fi + # https://github.com/actions/toolkit/issues/193 + check: + if: always() + permissions: + id-token: write + checks: read + + needs: + - build + + runs-on: ubuntu-latest + + steps: + # checkout needed for codecov action which needs codecov.yml file + - uses: actions/checkout@v4 + + - name: Set up Python # likely needed for coverage + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - run: pip3 install 'coverage>=7.5.1' + + - name: Merge logs into a single archive + uses: actions/upload-artifact/merge@v4 + with: + name: logs.zip + pattern: logs-*.zip + # artifacts like py312.zip and py312-macos do have overlapping files + separate-directories: true + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: logs.zip + path: . + + - name: Check for expected number of coverage.xml reports + run: | + JOBS_PRODUCING_COVERAGE=8 + if [ "$(find . -name coverage.xml | wc -l | bc)" -ne "${JOBS_PRODUCING_COVERAGE}" ]; then + echo "::error::Number of coverage.xml files was not the expected one (${JOBS_PRODUCING_COVERAGE}): $(find . -name coverage.xml |xargs echo)" + exit 1 + fi + + - name: Upload coverage data + uses: codecov/codecov-action@v4 + with: + name: ${{ matrix.passed_name }} + # verbose: true # optional (default = false) + fail_ci_if_error: true + use_oidc: true # cspell:ignore oidc + + - name: Check codecov.io status + if: github.event_name == 'pull_request' + uses: coactions/codecov-status@main + + - name: Decide whether the needed jobs succeeded or failed + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} + + - name: Delete Merged Artifacts + uses: actions/upload-artifact/merge@v4 + with: + delete-merged: true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6501a8c0..0d719aa2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,6 +4,12 @@ Contributions are welcome, and they are greatly appreciated! Every little bit he Every new feature should be tested and documented. New source plugins or source filters will be evaluated for inclusion in the collection and might be rejected. Please consider the option of creating a new collection for your plugins if it is not a good fit for this collection. +## Cloning + +Due to ansible-test own requirements, you must clone the repository into +a directory structure such `ansible_collections/ansible/eda`. This will allow +ansible-test to test your code without having to install the collection. + ## Pre-commit We recommend running pre-commit prior to submitting pull requests. A [pre-commit config](.pre-commit-config.yaml) file is included in this repository and the following steps will get you up and running with pre-commit quickly: @@ -18,7 +24,7 @@ We recommend running pre-commit prior to submitting pull requests. A [pre-commit Pre-commit is now set up to run each time you create a new commit. If you wish to run pre-commit against all tracked files in the repository without performing a commit, you can run: -``` +```shell pre-commit run --all ``` @@ -41,30 +47,21 @@ We recommend setting up a Python virtual environment to install the test depende pip install -r test_requirements.txt -### Integration tests +### Sanity and Unit tests -Integration tests require the addition of [docker](https://docs.docker.com/engine/install/) or [podman](https://podman.io/getting-started/installation) and [docker-compose](https://docs.docker.com/compose/install/). -We recommend installing the Python implementation of `docker-compose` via pip: +Sanity and unity tests can easily be run via tox: +```shell +tox -e py ``` -pip install docker-compose -``` - -Then install the collection directly from your local repo and execute the tests: -``` -ansible-galaxy collection install . -pytest tests/integration -``` +### Integration tests -### Sanity tests +Integration tests require the addition of [docker](https://docs.docker.com/engine/install/) or [podman](https://podman.io/getting-started/installation) and [docker-compose](https://docs.docker.com/compose/install/). -```sh -ansible-test sanity -``` -### Units tests +Then install the collection directly from your local repo and execute the tests: -```sh -ansible-test units +```shell +tox -e integration ``` diff --git a/requirements.txt b/requirements.txt index f59c591b..496858fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +pyyaml>=6.0.1 aiobotocore aiohttp aiokafka @@ -5,7 +6,6 @@ azure-servicebus dpath kafka-python psycopg -pyyaml systemd-python; sys_platform != 'darwin' watchdog xxhash diff --git a/test_requirements.txt b/test_requirements.txt index 2f3d8daf..becf57e8 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -7,3 +7,4 @@ ansible requests ansible-rulebook tox +ansible-core diff --git a/tox.ini b/tox.ini index d129be9a..35228957 100644 --- a/tox.ini +++ b/tox.ini @@ -1,16 +1,52 @@ # Recommended usage of this file is detailed in https://github.com/ansible/eda-partner-testing/blob/main/README.md. # The linter paths can be changed, but may result in false passes. # {posargs} in this case would be the path to collection root relative from the .github/workflows dir (`../..`) +# cspell: ignore TOXPYTHON setenv passenv REQPASS PYTHONPYCACHEPREFIX PYTHONIOENCODING PYTHONBREAKPOINT [tox] -envlist = lint -requires = - lint - darglint +envlist = lint, darglint, py, integration skipsdist = true # this repo is not a python package +isolated_build = true +requires = + tox >= 4.6.3 + setuptools >= 65.3.0 # editable installs [testenv] -basepython = python3.9, python3.10 +description = Run sanity and unit tests +basepython = + py39: {env:TOXPYTHON:python3.9} + py310: {env:TOXPYTHON:python3.10} +deps = -r test_requirements.txt +commands_pre = + bash -c 'test "$(basename $(cd ../.. && pwd))" == ansible_collections || { echo "Repository must be cloned inside a directory structure like ansible_collections/ansible/eda in order allow ansible-test to run."; exit 3;}' +commands = + ansible-test sanity + ansible-test units --venv -v --num-workers 1 +allowlist_externals = + bash +isolated_build = true +; # See https://stackoverflow.com/a/76884052/99834 +; py310: true +setenv = + PIP_NO_BUILD_ISOLATION = 1 +passenv = + CURL_CA_BUNDLE # https proxies, https://github.com/tox-dev/tox/issues/1437 + FORCE_COLOR + HOME + NO_COLOR + PYTEST_* # allows developer to define their own preferences + PYTEST_REQPASS # needed for CI + PYTHON* # PYTHONPYCACHEPREFIX, PYTHONIOENCODING, PYTHONBREAKPOINT,... + PY_COLORS + RTD_TOKEN + REQUESTS_CA_BUNDLE # https proxies + SETUPTOOLS_SCM_DEBUG + SSL_CERT_FILE # https proxies + SSH_AUTH_SOCK # may be needed by git + LANG + LC_* +# https://tox.wiki/en/latest/upgrading.html#editable-mode +package = editable [testenv:lint] deps = pre-commit @@ -20,3 +56,9 @@ commands = pre-commit run -a [testenv:darglint] deps = darglint commands = darglint -s numpy -z full extensions/eda/plugins + +[testenv:integration] +deps = -r test_requirements.txt +description = Run integration tests +commands = + pytest tests/integration -vvv -s