test suggestions to improve ats workflow

giovanni-guidini · Nov 3, 2023 · 875d144 · 875d144
1 parent 6d136cb
commit 875d144
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 30 deletions.
diff --git a/.github/workflows/codecov_ats.yml b/.github/workflows/codecov_ats.yml
@@ -13,6 +13,12 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+defaults:
+  run:
+    # the default default is:
+    #      bash --noprofile --norc -eo pipefail {0}
+    shell: bash --noprofile --norc -eo pipefail -ux {0}
+
 jobs:
     files-changed:
         name: detect what files changed
@@ -63,7 +69,7 @@ jobs:
             pg-version: 14
         - name: Download Codecov CLI
           run: |
-            pip install --extra-index-url https://pypi.org/simple --no-cache-dir codecov-cli>=0.4.0
+            pip install --extra-index-url https://pypi.org/simple --no-cache-dir codecov-cli>=0.4.1
         # Creates the commit and report objects in codecov
         - name: Codecov startup
           run: |
@@ -94,8 +100,8 @@ jobs:
             if [ -n "${output}" ];
             then
 
-              echo $(jq <<< $output '.runner_options + .ats_tests_to_run | @json' --raw-output) > .artifacts/codecov_ats_tests_to_run.json
-              echo $(jq <<< $output '.runner_options + .ats_tests_to_skip | @json' --raw-output) > .artifacts/codecov_ats_tests_to_skip.json
+              jq <<< $output '.runner_options + .ats_tests_to_run | @json' --raw-output > .artifacts/codecov_ats/tests_to_run.json
+              jq <<< $output '.runner_options + .ats_tests_to_skip | @json' --raw-output > .artifacts/codecov_ats/tests_to_skip.json
 
               testcount() { jq <<< $output ".$1 | length"; }
               run_count=$(testcount ats_tests_to_run)
@@ -111,9 +117,8 @@ jobs:
               tee <<< \
                 "{\"ats_success\": $ats_success, \"error\": $ats_fallback_reason, \"tests_to_run\": $run_count, \"tests_analyzed\": $((run_count+skip_count))}" \
                 "$GITHUB_STEP_SUMMARY" \
-                ".artifacts/codecov_ats_result.json"
+                ".artifacts/codecov_ats/result.json"
             else
-              tee <<< "ATS failed. Can't get list of tests to run." "$GITHUB_STEP_SUMMARY"
               # We need not forget to add the search options in the fallback command, otherwise pytest might run more tests than expected
               # These search options match what's defined in codecov.yml:105
               jq '@json' --raw-output <<< '[
@@ -126,28 +131,18 @@ jobs:
                 "--ignore=tests/sentry/search/events",
                 "--ignore=tests/sentry/ingest/ingest_consumer/test_ingest_consumer_kafka.py",
                 "--ignore=tests/sentry/region_to_control/test_region_to_control_kafka.py"
-              ]' > .artifacts/codecov_ats_tests_to_skip.json
-              echo '[]' > .artifacts/codecov_ats_tests_to_run.json
+              ]' > .artifacts/codecov_ats/tests_to_skip.json
+              echo '[]' > .artifacts/codecov_ats/tests_to_run.json
               # If we reached this point it means that ATS failed with some error
-              tee <<< '{"ats_success": false, "error": "exception_raised"}' "$GITHUB_STEP_SUMMARY" ".artifacts/codecov_ats_result.json"
+              tee <<< '{"ats_success": false, "error": "exception_raised"}' "$GITHUB_STEP_SUMMARY" ".artifacts/codecov_ats/result.json"
             fi
           env:
             CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
             CODECOV_STATIC_TOKEN: ${{ secrets.CODECOV_STATIC_TOKEN }}
         - uses: actions/upload-artifact@v3
           with:
-            name: ats_tests_to_run.json
-            path: .artifacts/codecov_ats_tests_to_run.json
-            if-no-files-found: error
-        - uses: actions/upload-artifact@v3
-          with:
-            name: ats_tests_to_skip.json
-            path: .artifacts/codecov_ats_tests_to_skip.json
-            if-no-files-found: error
-        - uses: actions/upload-artifact@v3
-          with:
-            name: ats_result.json
-            path: .artifacts/codecov_ats_result.json
+            name: codecov_ats
+            path: .artifacts/codecov_ats
             if-no-files-found: error
     # The actual running of tests would come here, after the labels are available
     # Something like pytest <options> $ATS_TESTS_TO_RUN
@@ -160,31 +155,27 @@ jobs:
       steps:
         - uses: actions/download-artifact@v3
           with:
-            name: ats_tests_to_run.json
-            path: .artifacts
-        - uses: actions/download-artifact@v3
-          with:
-            name: ats_tests_to_skip.json
+            name: codecov_ats
             path: .artifacts
         - name: Debug ATS_TESTS_TO_RUN
           run: |
-            length_of_tests=$(cat .artifacts/codecov_ats_tests_to_run.json | jq 'length')
+            length_of_tests=$(cat .artifacts/codecov_ats/tests_to_run.json | jq 'length')
             # The 1st value doesn't count, it's '--cov-context=test' (hence -gt 1)
             if [ $length_of_tests -gt 1 ]; then
               echo "Running $length_of_tests tests"
               # --raw-output0 doesn't work.
-              cat .artifacts/codecov_ats_tests_to_run.json | jq 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo 'pytest'
+              cat .artifacts/codecov_ats/tests_to_run.json | jq 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo 'pytest'
             else
               echo "No tests to run"
             fi
         - name: Debug ATS_TESTS_TO_SKIP
           run: |
-            length_of_tests=$(cat .artifacts/codecov_ats_tests_to_skip.json | jq 'length')
+            length_of_tests=$(cat .artifacts/codecov_ats/tests_to_skip.json | jq 'length')
             # The 1st value doesn't count, it's '--cov-context=test'
             if [ $length_of_tests -gt 1 ]; then
               echo "Running $length_of_tests tests"
               # --raw-output0 doesn't work.
-              cat .artifacts/codecov_ats_tests_to_skip.json | jq 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo 'pytest'
+              cat .artifacts/codecov_ats/tests_to_skip.json | jq 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo 'pytest'
             else
               echo "No tests to run"
             fi
diff --git a/.github/workflows/shuffle-tests.yml b/.github/workflows/shuffle-tests.yml
@@ -21,7 +21,7 @@ env:
 
 jobs:
   per-test-coverage:
-    if: ${{ inputs.per-test-coverage == 'true' || github.event_name == 'schedule' }}
+    if: ${{ inputs.per-test-coverage == 'true' }}
     uses: ./.github/workflows/codecov_per_test_coverage.yml
     secrets: inherit
   backend-test: