Skip to content

Commit

Permalink
surface ats errors
Browse files Browse the repository at this point in the history
  • Loading branch information
giovanni-guidini committed Nov 1, 2023
1 parent fe0b1cb commit b743d3d
Showing 1 changed file with 20 additions and 12 deletions.
32 changes: 20 additions & 12 deletions .github/workflows/codecov_ats.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,6 @@ jobs:
if: needs.files-changed.outputs.backend == 'true'
needs: files-changed
runs-on: ubuntu-latest
# Map a step output to a job output
outputs:
ats_success: ${{ steps.codecov_automated_test_selection.outputs.ats_success }}
steps:
- uses: actions/checkout@v3
with:
Expand Down Expand Up @@ -90,7 +87,6 @@ jobs:
# The base commit will be the parent commit (apparently commits on master don't exist in codecov)
- name: Codecov Automated Test Selection
id: codecov_automated_test_selection
continue-on-error: true
run: |
BASE_COMMIT=$(git merge-base ${{ github.sha }}^ origin/master)
echo $BASE_COMMIT
Expand All @@ -105,9 +101,16 @@ jobs:
run_count=$(testcount ats_tests_to_run)
skip_count=$(testcount ats_tests_to_skip)
tee <<< "Selected $run_count / $(($run_count + $skip_count)) tests to run" "$GITHUB_STEP_SUMMARY"
echo "ats_success=true" >> "$GITHUB_OUTPUT"
# Parse any potential errors that made ATS fallback to running all tests
# And surface them
ats_fallback_reason=$(jq <<< "$output" '.ats_fallback_reason')
if [ "$ats_fallback_reason" == "null" ]; then
tee <<< '{"ats_success": true, "error": null}' "$GITHUB_STEP_SUMMARY" ".artifacts/codecov_ats_result.json"
else
tee <<< "{\"ats_success\": false, \"error\": $ats_fallback_reason}" "$GITHUB_STEP_SUMMARY" ".artifacts/codecov_ats_result.json"
fi
else
tee <<< "ATS failed. Can't get list of tests to run. Fallback to all tests" "$GITHUB_STEP_SUMMARY"
tee <<< "ATS failed. Can't get list of tests to run." "$GITHUB_STEP_SUMMARY"
# We need not forget to add the search options in the fallback command, otherwise pytest might run more tests than expected
# These search options match what's defined in codecov.yml:105
jq '@json' --raw-output <<< '[
Expand All @@ -120,9 +123,10 @@ jobs:
"--ignore=tests/sentry/search/events",
"--ignore=tests/sentry/ingest/ingest_consumer/test_ingest_consumer_kafka.py",
"--ignore=tests/sentry/region_to_control/test_region_to_control_kafka.py"
]' > .artifacts/codecov_ats_tests_to_run.json
echo '[]' > .artifacts/codecov_ats_tests_to_skip.json
echo "ats_success=false" >> "$GITHUB_OUTPUT"
]' > .artifacts/codecov_ats_tests_to_skip.json
echo '[]' > .artifacts/codecov_ats_tests_to_run.json
# If we reached this point it means that ATS failed with some error
tee <<< '{"ats_success": false, "error": "exception_raised"}' "$GITHUB_STEP_SUMMARY" ".artifacts/codecov_ats_result.json"
fi
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
Expand All @@ -137,6 +141,11 @@ jobs:
name: ats_tests_to_skip.json
path: .artifacts/codecov_ats_tests_to_skip.json
if-no-files-found: error
- uses: actions/upload-artifact@v3
with:
name: ats_result.json
path: .artifacts/codecov_ats_result.json
if-no-files-found: error
# The actual running of tests would come here, after the labels are available
# Something like pytest <options> $ATS_TESTS_TO_RUN
debug:
Expand All @@ -161,19 +170,18 @@ jobs:
if [ $length_of_tests -gt 1 ]; then
echo "Running $length_of_tests tests"
# --raw-output0 doesn't work.
cat .artifacts/codecov_ats_tests_to_run.json | jq 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo
cat .artifacts/codecov_ats_tests_to_run.json | jq 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo 'pytest'
else
echo "No tests to run"
fi
- name: Debug ATS_TESTS_TO_SKIP
run: |
length_of_tests=$(cat .artifacts/codecov_ats_tests_to_skip.json | jq 'length')
# The 1st value doesn't count, it's '--cov-context=test'
if [ $length_of_tests -gt 1 ]; then
echo "Running $length_of_tests tests"
# --raw-output0 doesn't work.
cat .artifacts/codecov_ats_tests_to_skip.json | jq 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo
cat .artifacts/codecov_ats_tests_to_skip.json | jq 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo 'pytest'
else
echo "No tests to run"
fi

0 comments on commit b743d3d

Please sign in to comment.