Unify testing jobs into a single workflows

Related: #219

.github/workflows/tox.yml

@@ -14,19 +14,260 @@ on:
   # Run on demand
   workflow_dispatch:
 jobs:
-  tox:
-    runs-on: ubuntu-latest
+  prepare:
+    name: prepare
+    runs-on: ubuntu-22.04
+    outputs:
+      matrix: ${{ steps.generate_matrix.outputs.matrix }}
+    steps:
+      - name: Determine matrix
+        id: generate_matrix
+        uses: coactions/dynamic-matrix@v1
+        with:
+          min_python: "3.9"
+          max_python: "3.10"
+          default_python: "3.9"
+          other_names: |
+            lint,darglint
+            integration
+          platforms: linux
+  build:
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.os || 'ubuntu-22.04' }}
+    needs:
+      - prepare
+    defaults:
+      run:
+        shell: ${{ matrix.shell || 'bash'}}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.prepare.outputs.matrix) }}
+      # max-parallel: 5
+      # The matrix testing goal is to cover the *most likely* environments
+      # which are expected to be used by users in production. Avoid adding a
+      # combination unless there are good reasons to test it, like having
+      # proof that we failed to catch a bug by not running it. Using
+      # distribution should be preferred instead of custom builds.
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # needed by setuptools-scm
+          submodules: true
+
+      - name: Set pre-commit cache
+        uses: actions/cache@v4
+        if: ${{ matrix.passed_name == 'lint' }}
+        with:
+          path: |
+            ~/.cache/pre-commit
+          key: pre-commit-${{ matrix.name || matrix.passed_name }}-${{ hashFiles('.pre-commit-config.yaml') }}
+
+      - name: Set ansible cache(s)
+        uses: actions/cache@v4
+        with:
+          path: |
+            .cache/eco
+            examples/playbooks/collections/ansible_collections
+            ~/.cache/ansible-compat
+            ~/.ansible/collections
+            ~/.ansible/roles
+          key: ${{ matrix.name || matrix.passed_name }}-${{ hashFiles('tools/test-eco.sh', 'requirements.yml', 'examples/playbooks/collections/requirements.yml') }}
+
+      - name: Set up Python ${{ matrix.python_version || '3.10' }}
+        if: "!contains(matrix.shell, 'wsl')"
+        uses: actions/setup-python@v5
+        with:
+          cache: pip
+          python-version: ${{ matrix.python_version || '3.10' }}
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: "npm"
+          cache-dependency-path: test/schemas/package-lock.json
 
-      - name: Install package dependencies
+      - name: Run ./tools/test-setup.sh
+        run: ./tools/test-setup.sh
+
+      - name: Install tox
         run: |
-          sudo apt-get update
-          sudo apt-get --assume-yes --no-install-recommends install libsystemd0 libsystemd-dev pkg-config
+          python3 -m pip install --upgrade pip
+          python3 -m pip install --upgrade "tox>=4.0.0"
+
+      - name: Initialize tox envs ${{ matrix.passed_name }}
+        run: python3 -m tox -n --skip-missing-interpreters false -vv -e ${{ matrix.passed_name }}
+        timeout-minutes: 5 # average is under 1, but macos can be over 3
+
+      # sequential run improves browsing experience (almost no speed impact)
+      - name: tox -e ${{ matrix.passed_name }}
+        run: python3 -m tox -e ${{ matrix.passed_name }}
 
-      - name: Install deps
-        run: python -m pip install tox
+      - name: Archive logs
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-${{ matrix.name }}.zip
+          path: |
+            .tox/**/log/
+            .tox/**/.coverage*
+            .tox/**/coverage.xml
 
-      - name: Run tox
+      - name: Report failure if git reports dirty status
         run: |
-          python -m tox
+          if [[ -n $(git status -s) ]]; then
+            # shellcheck disable=SC2016
+            echo -n '::error file=git-status::'
+            printf '### Failed as git reported modified and/or untracked files\n```\n%s\n```\n' "$(git status -s)" | tee -a "$GITHUB_STEP_SUMMARY"
+            exit 99
+          fi
+        # https://github.com/actions/toolkit/issues/193
+  check:
+    if: always()
+    permissions:
+      id-token: write
+      checks: read
+
+    needs:
+      - build
+
+    runs-on: ubuntu-latest
+
+    steps:
+      # checkout needed for codecov action which needs codecov.yml file
+      - uses: actions/checkout@v4
+
+      - name: Set up Python # likely needed for coverage
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - run: pip3 install 'coverage>=7.5.1'
+
+      - name: Merge logs into a single archive
+        uses: actions/upload-artifact/merge@v4
+        with:
+          name: logs.zip
+          pattern: logs-*.zip
+          # artifacts like py312.zip and py312-macos do have overlapping files
+          separate-directories: true
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: logs.zip
+          path: .
+
+      - name: Check for expected number of coverage.xml reports
+        run: |
+          JOBS_PRODUCING_COVERAGE=8
+          if [ "$(find . -name coverage.xml | wc -l | bc)" -ne "${JOBS_PRODUCING_COVERAGE}" ]; then
+            echo "::error::Number of coverage.xml files was not the expected one (${JOBS_PRODUCING_COVERAGE}): $(find . -name coverage.xml |xargs echo)"
+            exit 1
+          fi
+
+      - name: Upload coverage data
+        uses: codecov/codecov-action@v4
+        with:
+          name: ${{ matrix.passed_name }}
+          # verbose: true # optional (default = false)
+          fail_ci_if_error: true
+          use_oidc: true # cspell:ignore oidc
+
+      - name: Check codecov.io status
+        if: github.event_name == 'pull_request'
+        uses: coactions/codecov-status@main
+
+      - name: Decide whether the needed jobs succeeded or failed
+        uses: re-actors/alls-green@release/v1
+        with:
+          jobs: ${{ toJSON(needs) }}
+
+      - name: Delete Merged Artifacts
+        uses: actions/upload-artifact/merge@v4
+        with:
+          delete-merged: true
+
+  # lint:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v3
+
+  #     - name: Install package dependencies
+  #       run: |
+  #         sudo apt-get update
+  #         sudo apt-get --assume-yes --no-install-recommends install libsystemd0 libsystemd-dev pkg-config
+
+  #     - name: Install deps
+  #       run: python -m pip install tox
+
+  #     - name: Run tox
+  #       run: |
+  #         python -m tox -e lint,darglint
+  # tests:
+  #   runs-on: ubuntu-latest
+
+  #   strategy:
+  #     matrix:
+  #       python-version:
+  #         - "3.10"
+  #         - "3.9"
+
+  #   defaults:
+  #     run:
+  #       working-directory: ansible_collections/
+
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v3
+  #       with:
+  #         path: ansible_collections/ansible/eda
+
+  #     - name: Set up Python ${{ matrix.python-version }}
+  #       uses: actions/setup-python@v4
+  #       with:
+  #         python-version: ${{ matrix.python-version }}
+
+  #     # - name: Install ansible
+  #     #   run: python -m pip install ansible
+
+  #     # - name: Run sanity tests
+  #     #   run: ansible-test sanity
+  #     #   working-directory: ansible_collections/ansible/eda
+
+  #     - name: Run unit tests
+  #       run: tox -e py
+  #       # ansible-test units --venv -v --num-workers 1
+  #       working-directory: ansible_collections/ansible/eda
+  # integration:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v3
+
+  #     - name: Install Java
+  #       uses: actions/setup-java@v3
+  #       with:
+  #         distribution: "zulu"
+  #         java-version: "17"
+
+  #     - name: Set up Python
+  #       uses: actions/setup-python@v4
+  #       with:
+  #         python-version: "3.9"
+
+  #     - name: Install package dependencies
+  #       run: |
+  #         sudo apt-get update
+  #         sudo apt-get --assume-yes --no-install-recommends install libsystemd0 libsystemd-dev pkg-config
+
+  #     - name: Install test requirements
+  #       run: |
+  #         pip install -U pip tox
+  #     #     pip install wheel
+  #     #     pip install -r test_requirements.txt
+
+  #     # - name: Install collection
+  #     #   run: ansible-galaxy collection install .
+
+  #     - name: run integration tests
+  #       run: tox -e integration
+  #       # pytest tests/integration -vvv -s