codecov_per_test_coverage.yml

name: Codecov - per test coverage
# This workflow generates pytest coverage with the flag --cov-context=test
# This coverage is used as input for Codecov Automated Test Selection (see .github/workflows/codecov_ats.yml)
# However there's a performance toll in running tests with this flag.
# So we will not be running the test suite on every commit

on: [workflow_dispatch, workflow_call]

jobs:
  # Same as 'backend' in .github/workflows/backed.yml
  # Except for run_backend_tests step (which includes the extra --cov-context=test flag)
  # And the coverage generation and handling
  backend-test-with-cov-context:
    if: github.ref == 'refs/heads/master'
    name: backend test
    runs-on: ubuntu-22.04
    timeout-minutes: 120
    strategy:
      # This helps not having to run multiple jobs because one fails, thus, reducing resource usage
      # and reducing the risk that one of many runs would turn red again (read: intermittent tests)
      fail-fast: false
      matrix:
        # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL.
        instance: [0, 1, 2, 3, 4, 5, 6]
        pg-version: ['14']

    env:
      # XXX: `MATRIX_INSTANCE_TOTAL` must be hardcoded to the length of `strategy.matrix.instance`.
      # If this increases, make sure to also increase `flags.backend.after_n_builds` in `codecov.yml`.
      MATRIX_INSTANCE_TOTAL: 7

    steps:
      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          # Avoid codecov error message related to SHA resolution:
          # https://github.com/codecov/codecov-bash/blob/7100762afbc822b91806a6574658129fe0d23a7d/codecov#L891
          fetch-depth: '2'

      - name: Setup sentry env
        uses: ./.github/actions/setup-sentry
        id: setup
        with:
          kafka: true
          snuba: true
          symbolicator: true
          # Right now, we run so few bigtable related tests that the
          # overhead of running bigtable in all backend tests
          # is way smaller than the time it would take to run in its own job.
          bigtable: true
          pg-version: ${{ matrix.pg-version }}

      - name: Run backend test (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }}) with --cov-context=test
        id: run_backend_tests
        run: |
          make test-python-ci COV_ARGS=--cov-context=test

      # Separate from the testing step above so that we always create the report
      # Even if some tests fail
      - name: Create coverage report in JSON format
        if: ${{ always() }}
        run: |
          coverage json --show-contexts -o .artifacts/python.coverage.json

      # Upload coverage data even if running the tests step fails since
      # it reduces large coverage fluctuations
      - name: Upload coverage - special case to test Codecov ATS
        if: ${{ always() }}
        uses: codecov/codecov-action@v4-beta
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
        with:
          files: .artifacts/python.coverage.codecov.json
          flags: smart-tests
          plugins: compress-pycoverage
        continue-on-error: true