ci: get list of tests to run and skip from codecov ats

.github/workflows/codecov_ats.yml

@@ -13,6 +13,12 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+defaults:
+  run:
+    # the default default is:
+    #      bash --noprofile --norc -eo pipefail {0}
+    shell: bash --noprofile --norc -eo pipefail -ux {0}
+
 jobs:
     files-changed:
         name: detect what files changed
@@ -35,19 +41,26 @@ jobs:
               token: ${{ github.token }}
               filters: .github/file-filters.yml
     coverage-ats:
-        # Temporary test
         if: needs.files-changed.outputs.backend == 'true'
         needs: files-changed
-        timeout-minutes: 40
         runs-on: ubuntu-latest
+        # Map a step output to a job output
+        outputs:
+          ATS_TESTS_TO_RUN: ${{ steps.codecov_automated_test_selection.outputs.ATS_TESTS_TO_RUN }}
+          ATS_TESTS_TO_SKIP: ${{ steps.codecov_automated_test_selection.outputs.ATS_TESTS_TO_SKIP }}
         steps:
         - uses: actions/checkout@v3
           with:
+            # fetch-depth: 0 - Use if the BASE_COMMIT on codecov_automated_test_selection is at unknown depth
+            #                   (i.e. git merge-base ${{ github.sha }}^ origin/main)
+            # fetch-depth: 2 - Use if the BASE_COMMIT on codecov_automated_test_selection is at known depth
+            #                   (i.e. git rev-parse ${{ github.sha }}^)
             fetch-depth: 0
         - name: Set up Python 3.10.10
           uses: actions/setup-python@v4
           with:
             python-version: "3.10.10"
+        # We need the setup to collect the list of tests properly
         - name: Setup sentry env
           uses: ./.github/actions/setup-sentry
           id: setup
@@ -60,7 +73,7 @@ jobs:
             pg-version: 14
         - name: Download Codecov CLI
           run: |
-            pip install --extra-index-url https://pypi.org/simple --no-cache-dir pytest codecov-cli
+            pip install --extra-index-url https://pypi.org/simple --no-cache-dir codecov-cli>=0.4.0
         # Creates the commit and report objects in codecov
         - name: Codecov startup
           run: |
@@ -80,13 +93,77 @@ jobs:
           env:
             CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
             CODECOV_STATIC_TOKEN: ${{ secrets.CODECOV_STATIC_TOKEN }}
-        # Run label analysis IN DRY MODE (no tests will actually run)
-        - name: Label Analysis
-          continue-on-error: true
+        # Run Automated Test Selection in dry mode to get the list of tests to run
+        # The base commit will be the parent commit (apparently commits on master don't exist in codecov)
+        - name: Codecov Automated Test Selection
+          id: codecov_automated_test_selection
           run: |
             BASE_COMMIT=$(git merge-base ${{ github.sha }}^ origin/master)
             echo $BASE_COMMIT
-            codecovcli --codecov-yml-path=codecov.yml label-analysis --dry-run --token=${CODECOV_STATIC_TOKEN} --base-sha=${BASE_COMMIT}
+            output=$(codecovcli --codecov-yml-path=codecov.yml label-analysis --dry-run --token=${CODECOV_STATIC_TOKEN} --base-sha=${BASE_COMMIT}) || true
+            if [ -n "${output}" ];
+            then
+
+              echo ATS_TESTS_TO_RUN=$(jq <<< $output '.runner_options + .ats_tests_to_run | @json | @sh' --raw-output) >> "$GITHUB_OUTPUT"
+              echo ATS_TESTS_TO_SKIP=$(jq <<< $output '.runner_options + .ats_tests_to_skip | @json | @sh' --raw-output) >> "$GITHUB_OUTPUT"
+
+              testcount() { jq <<< $output ".$1 | length"; }
+              run_count=$(testcount ats_tests_to_run)
+              skip_count=$(testcount ats_tests_to_skip)
+              tee <<< "Selected $run_count / $(($run_count + $skip_count)) tests to run" "$GITHUB_STEP_SUMMARY"
+            else
+              tee <<< "ATS failed. Can't get list of tests to run. Fallback to all tests" "$GITHUB_STEP_SUMMARY"
+              # We need not forget to add the search options in the fallback command, otherwise pytest might run more tests than expected
+              # These search options match what's defined in codecov.yml:105
+              echo 'ATS_TESTS_TO_RUN<<EOF' >> $GITHUB_OUTPUT
+              jq -c @json <<< '[
+                "--cov-context=test",
+                "tests/sentry",
+                "tests/integration",
+                "--ignore=tests/sentry/eventstream/kafka",
+                "--ignore=tests/sentry/post_process_forwarder",
+                "--ignore=tests/sentry/snuba",
+                "--ignore=tests/sentry/search/events",
+                "--ignore=tests/sentry/ingest/ingest_consumer/test_ingest_consumer_kafka.py",
+                "--ignore=tests/sentry/region_to_control/test_region_to_control_kafka.py"
+              ]' >> $GITHUB_OUTPUT
+              echo 'EOF' >> $GITHUB_OUTPUT
+              echo ATS_TESTS_TO_SKIP="'[]'" >> "$GITHUB_OUTPUT"
+              echo "::error ATS failed"
+              exit 1
+            fi
           env:
             CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
             CODECOV_STATIC_TOKEN: ${{ secrets.CODECOV_STATIC_TOKEN }}
+    # The actual running of tests would come here, after the labels are available
+    # Something like pytest <options> $ATS_TESTS_TO_RUN
+    debug:
+      runs-on: ubuntu-latest
+      needs: coverage-ats
+      if: ${{ always() }}
+      steps:
+        - name: Debug ATS_TESTS_TO_RUN
+          run: |
+            : ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_RUN }}
+            length_of_tests=$(jq <<< ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_RUN }} 'length')
+            # The 1st value doesn't count, it's '--cov-context=test' (hence -gt 1)
+            if [ $length_of_tests -gt 1 ]; then
+              echo "Running $length_of_tests tests"
+              # --raw-output0 doesn't work.
+              jq <<< ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_RUN }} 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo
+            else
+              echo "No tests to run"
+            fi
+        - name: Debug ATS_TESTS_TO_SKIP
+          run: |
+            : ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_SKIP }}
+            ATS_TESTS_TO_SKIP='${{ needs.coverage-ats.outputs.ATS_TESTS_TO_SKIP }}'
+            length_of_tests=$(jq <<< ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_SKIP }} 'length')
+            # The 1st value doesn't count, it's '--cov-context=test'
+            if [ $length_of_tests -gt 1 ]; then
+              echo "Running $length_of_tests tests"
+              # --raw-output0 doesn't work.
+              jq <<< ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_SKIP }} 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo
+            else
+              echo "No tests to run"
+            fi