Unify testing jobs into a single workflows

Related: #219
ansible · Aug 5, 2024 · d383d04 · d383d04
1 parent 1021e53
commit d383d04
Show file tree

Hide file tree

Showing 7 changed files with 223 additions and 126 deletions.
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml
@@ -1,5 +1,5 @@
 ---
-name: Tox
+name: tox
 on:
   push:
     branches: ["main"]
@@ -14,19 +14,169 @@ on:
   # Run on demand
   workflow_dispatch:
 jobs:
-  tox:
-    runs-on: ubuntu-latest
+  prepare:
+    name: prepare
+    runs-on: ubuntu-22.04
+    outputs:
+      matrix: ${{ steps.generate_matrix.outputs.matrix }}
+    steps:
+      - name: Determine matrix
+        id: generate_matrix
+        uses: coactions/dynamic-matrix@v1
+        with:
+          min_python: "3.9"
+          max_python: "3.10"
+          default_python: "3.9"
+          other_names: |
+            lint
+            darglint
+            integration
+          platforms: linux
+  build:
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.os || 'ubuntu-22.04' }}
+    continue-on-error: ${{ contains(matrix.name, 'integration') && true || false }}
+    needs:
+      - prepare
+    defaults:
+      run:
+        shell: ${{ matrix.shell || 'bash'}}
+        working-directory: ansible_collections/ansible/eda
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.prepare.outputs.matrix) }}
+      # max-parallel: 5
+      # The matrix testing goal is to cover the *most likely* environments
+      # which are expected to be used by users in production. Avoid adding a
+      # combination unless there are good reasons to test it, like having
+      # proof that we failed to catch a bug by not running it. Using
+      # distribution should be preferred instead of custom builds.
     steps:
-      - uses: actions/checkout@v3
+
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # needed by setuptools-scm
+          path: ansible_collections/ansible/eda
+          submodules: true
 
       - name: Install package dependencies
         run: |
           sudo apt-get update
           sudo apt-get --assume-yes --no-install-recommends install libsystemd0 libsystemd-dev pkg-config
 
-      - name: Install deps
-        run: python -m pip install tox
+      - name: Install Java
+        if: ${{ matrix.passed_name == 'integration' }}
+        uses: actions/setup-java@v4
+        with:
+          distribution: "zulu"
+          java-version: "17"
+
+      - name: Set pre-commit cache
+        uses: actions/cache@v4
+        if: ${{ matrix.passed_name == 'lint' }}
+        with:
+          path: |
+            ~/.cache/pre-commit
+          key: pre-commit-${{ matrix.name || matrix.passed_name }}-${{ hashFiles('.pre-commit-config.yaml') }}
 
-      - name: Run tox
+      - name: Set up Python ${{ matrix.python_version || '3.10' }}
+        uses: actions/setup-python@v5
+        with:
+          cache: pip
+          python-version: ${{ matrix.python_version || '3.10' }}
+
+      - name: Install tox
         run: |
-          python -m tox
+          python3 -m pip install --upgrade pip wheel tox
+
+      - name: Initialize tox envs ${{ matrix.passed_name }}
+        run: python3 -m tox -n --skip-missing-interpreters false -vv -e ${{ matrix.passed_name }}
+        timeout-minutes: 5 # average is under 1, but macos can be over 3
+
+      # sequential run improves browsing experience (almost no speed impact)
+      - name: tox -e ${{ matrix.passed_name }}
+        run: python3 -m tox -e ${{ matrix.passed_name }}
+
+      - name: Archive logs
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-${{ matrix.name }}.zip
+          path: |
+            .tox/**/log/
+            .tox/**/.coverage*
+            .tox/**/coverage.xml
+
+      - name: Report failure if git reports dirty status
+        run: |
+          if [[ -n $(git status -s) ]]; then
+            # shellcheck disable=SC2016
+            echo -n '::error file=git-status::'
+            printf '### Failed as git reported modified and/or untracked files\n```\n%s\n```\n' "$(git status -s)" | tee -a "$GITHUB_STEP_SUMMARY"
+            exit 99
+          fi
+        # https://github.com/actions/toolkit/issues/193
+  check:
+    if: always()
+    permissions:
+      id-token: write
+      checks: read
+
+    needs:
+      - build
+
+    runs-on: ubuntu-latest
+
+    steps:
+      # checkout needed for codecov action which needs codecov.yml file
+      - uses: actions/checkout@v4
+
+      - name: Set up Python # likely needed for coverage
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - run: pip3 install 'coverage>=7.5.1'
+
+      - name: Merge logs into a single archive
+        uses: actions/upload-artifact/merge@v4
+        with:
+          name: logs.zip
+          pattern: logs-*.zip
+          # artifacts like py312.zip and py312-macos do have overlapping files
+          separate-directories: true
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: logs.zip
+          path: .
+
+      - name: Check for expected number of coverage.xml reports
+        run: |
+          JOBS_PRODUCING_COVERAGE=8
+          if [ "$(find . -name coverage.xml | wc -l | bc)" -ne "${JOBS_PRODUCING_COVERAGE}" ]; then
+            echo "::error::Number of coverage.xml files was not the expected one (${JOBS_PRODUCING_COVERAGE}): $(find . -name coverage.xml |xargs echo)"
+            exit 1
+          fi
+
+      - name: Upload coverage data
+        uses: codecov/codecov-action@v4
+        with:
+          name: ${{ matrix.passed_name }}
+          # verbose: true # optional (default = false)
+          fail_ci_if_error: true
+          use_oidc: true # cspell:ignore oidc
+
+      - name: Check codecov.io status
+        if: github.event_name == 'pull_request'
+        uses: coactions/codecov-status@main
+
+      - name: Decide whether the needed jobs succeeded or failed
+        uses: re-actors/alls-green@release/v1
+        with:
+          jobs: ${{ toJSON(needs) }}
+
+      - name: Delete Merged Artifacts
+        uses: actions/upload-artifact/merge@v4
+        with:
+          delete-merged: true
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -4,6 +4,12 @@ Contributions are welcome, and they are greatly appreciated! Every little bit he
 Every new feature should be tested and documented.
 New source plugins or source filters will be evaluated for inclusion in the collection and might be rejected. Please consider the option of creating a new collection for your plugins if it is not a good fit for this collection.
 
+## Cloning
+
+Due to ansible-test own requirements, you must clone the repository into
+a directory structure such `ansible_collections/ansible/eda`. This will allow
+ansible-test to test your code without having to install the collection.
+
 ## Pre-commit
 
 We recommend running pre-commit prior to submitting pull requests. A [pre-commit config](.pre-commit-config.yaml) file is included in this repository and the following steps will get you up and running with pre-commit quickly:
@@ -18,7 +24,7 @@ We recommend running pre-commit prior to submitting pull requests. A [pre-commit
 
 Pre-commit is now set up to run each time you create a new commit. If you wish to run pre-commit against all tracked files in the repository without performing a commit, you can run:
 
-```
+```shell
 pre-commit run --all
 ```
 
@@ -41,30 +47,21 @@ We recommend setting up a Python virtual environment to install the test depende
 
         pip install -r test_requirements.txt
 
-### Integration tests
+### Sanity and Unit tests
 
-Integration tests require the addition of [docker](https://docs.docker.com/engine/install/) or [podman](https://podman.io/getting-started/installation) and [docker-compose](https://docs.docker.com/compose/install/).
-We recommend installing the Python implementation of `docker-compose` via pip:
+Sanity and unity tests can easily be run via tox:
 
+```shell
+tox -e py
 ```
-pip install docker-compose
-```
-
-Then install the collection directly from your local repo and execute the tests:
 
-```
-ansible-galaxy collection install .
-pytest tests/integration
-```
+### Integration tests
 
-### Sanity tests
+Integration tests require the addition of [docker](https://docs.docker.com/engine/install/) or [podman](https://podman.io/getting-started/installation) and [docker-compose](https://docs.docker.com/compose/install/).
 
-```sh
-ansible-test sanity
-```
 
-### Units tests
+Then install the collection directly from your local repo and execute the tests:
 
-```sh
-ansible-test units
+```shell
+tox -e integration
 ```
diff --git a/requirements.txt b/requirements.txt
@@ -1,11 +1,11 @@
+pyyaml>=6.0.1
 aiobotocore
 aiohttp
 aiokafka
 azure-servicebus
 dpath
 kafka-python
 psycopg
-pyyaml
 systemd-python; sys_platform != 'darwin'
 watchdog
 xxhash
diff --git a/test_requirements.txt b/test_requirements.txt
@@ -7,3 +7,4 @@ ansible
 requests
 ansible-rulebook
 tox
+ansible-core