ci: get list of tests to run and skip from codecov ats #98
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: IGNORE ME codecov-ats | |
| on: | |
| pull_request: | |
| env: | |
| CLI_VERSION: v0.1.5 | |
| # Cancel in progress workflows on pull_requests. | |
| # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| # the default default is: | |
| # bash --noprofile --norc -eo pipefail {0} | |
| shell: bash --noprofile --norc -eo pipefail -ux {0} | |
| jobs: | |
| files-changed: | |
| name: detect what files changed | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 3 | |
| # Map a step output to a job output | |
| outputs: | |
| api_docs: ${{ steps.changes.outputs.api_docs }} | |
| backend: ${{ steps.changes.outputs.backend_all }} | |
| backend_dependencies: ${{ steps.changes.outputs.backend_dependencies }} | |
| backend_any_type: ${{ steps.changes.outputs.backend_any_type }} | |
| migration_lockfile: ${{ steps.changes.outputs.migration_lockfile }} | |
| steps: | |
| - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 | |
| - name: Check for backend file changes | |
| uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 | |
| id: changes | |
| with: | |
| token: ${{ github.token }} | |
| filters: .github/file-filters.yml | |
| coverage-ats: | |
| if: needs.files-changed.outputs.backend == 'true' | |
| needs: files-changed | |
| runs-on: ubuntu-latest | |
| # Map a step output to a job output | |
| outputs: | |
| ATS_TESTS_TO_RUN: ${{ steps.codecov_automated_test_selection.outputs.ATS_TESTS_TO_RUN }} | |
| ATS_TESTS_TO_SKIP: ${{ steps.codecov_automated_test_selection.outputs.ATS_TESTS_TO_SKIP }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| # fetch-depth: 0 - Use if the BASE_COMMIT on codecov_automated_test_selection is at unknown depth | |
| # (i.e. git merge-base ${{ github.sha }}^ origin/main) | |
| # fetch-depth: 2 - Use if the BASE_COMMIT on codecov_automated_test_selection is at known depth | |
| # (i.e. git rev-parse ${{ github.sha }}^) | |
| fetch-depth: 0 | |
| - name: Set up Python 3.10.10 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10.10" | |
| # We need the setup to collect the list of tests properly | |
| - name: Setup sentry env | |
| uses: ./.github/actions/setup-sentry | |
| id: setup | |
| with: | |
| snuba: true | |
| # Right now, we run so few bigtable related tests that the | |
| # overhead of running bigtable in all backend tests | |
| # is way smaller than the time it would take to run in its own job. | |
| bigtable: true | |
| pg-version: 14 | |
| - name: Download Codecov CLI | |
| run: | | |
| pip install --extra-index-url https://pypi.org/simple --no-cache-dir codecov-cli==0.4.0 | |
| # Creates the commit and report objects in codecov | |
| - name: Codecov startup | |
| run: | | |
| codecovcli create-commit | |
| codecovcli create-report | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| # Sends static analysis information to codecov | |
| - name: Static Analysis | |
| run: | | |
| codecovcli static-analysis --token=${CODECOV_STATIC_TOKEN} \ | |
| --folders-to-exclude .artifacts \ | |
| --folders-to-exclude .github \ | |
| --folders-to-exclude .venv \ | |
| --folders-to-exclude static \ | |
| --folders-to-exclude bin | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| CODECOV_STATIC_TOKEN: ${{ secrets.CODECOV_STATIC_TOKEN }} | |
| # Run Automated Test Selection in dry mode to get the list of tests to run | |
| # The base commit will be the parent commit (apparently commits on master don't exist in codecov) | |
| - name: Codecov Automated Test Selection | |
| id: codecov_automated_test_selection | |
| run: | | |
| BASE_COMMIT=$(git merge-base ${{ github.sha }}^ origin/master) | |
| echo $BASE_COMMIT | |
| output=$(codecovcli --codecov-yml-path=codecov.yml label-analysis --dry-run --token=${CODECOV_STATIC_TOKEN} --base-sha=${BASE_COMMIT}) || true | |
| if [ -n "${output}" ]; | |
| then | |
| echo ATS_TESTS_TO_RUN=$(jq <<< $output '.runner_options + .ats_tests_to_run | @json | @sh' --raw-output) >> "$GITHUB_OUTPUT" | |
| echo ATS_TESTS_TO_SKIP=$(jq <<< $output '.runner_options + .ats_tests_to_skip | @json | @sh' --raw-output) >> "$GITHUB_OUTPUT" | |
| testcount() { jq <<< $output ".$1 | length"; } | |
| run_count=$(testcount ats_tests_to_run) | |
| skip_count=$(testcount ats_tests_to_skip) | |
| tee <<< "Selected $run_count / $(($run_count + $skip_count)) tests to run" "$GITHUB_STEP_SUMMARY" | |
| else | |
| tee <<< "ATS failed. Can't get list of tests to run. Fallback to all tests" "$GITHUB_STEP_SUMMARY" | |
| # We need not forget to add the search options in the fallback command, otherwise pytest might run more tests than expected | |
| # These search options match what's defined in codecov.yml:105 | |
| echo 'ATS_TESTS_TO_RUN<<EOF' >> $GITHUB_OUTPUT | |
| jq -c @json <<< '[ | |
| "--cov-context=test", | |
| "tests/sentry", | |
| "tests/integration", | |
| "--ignore=tests/sentry/eventstream/kafka", | |
| "--ignore=tests/sentry/post_process_forwarder", | |
| "--ignore=tests/sentry/snuba", | |
| "--ignore=tests/sentry/search/events", | |
| "--ignore=tests/sentry/ingest/ingest_consumer/test_ingest_consumer_kafka.py", | |
| "--ignore=tests/sentry/region_to_control/test_region_to_control_kafka.py" | |
| ]' >> $GITHUB_OUTPUT | |
| echo 'EOF' >> $GITHUB_OUTPUT | |
| echo ATS_TESTS_TO_SKIP="'[]'" >> "$GITHUB_OUTPUT" | |
| echo "::error ATS failed" | |
| exit 1 | |
| fi | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| CODECOV_STATIC_TOKEN: ${{ secrets.CODECOV_STATIC_TOKEN }} | |
| # The actual running of tests would come here, after the labels are available | |
| # Something like pytest <options> $ATS_TESTS_TO_RUN | |
| debug: | |
| runs-on: ubuntu-latest | |
| needs: coverage-ats | |
| if: ${{ always() }} | |
| steps: | |
| - name: Debug ATS_TESTS_TO_RUN | |
| run: | | |
| : ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_RUN }} | |
| length_of_tests=$(jq <<< ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_RUN }} 'length') | |
| # The 1st value doesn't count, it's '--cov-context=test' (hence -gt 1) | |
| if [ $length_of_tests -gt 1 ]; then | |
| echo "Running $length_of_tests tests" | |
| jq <<< ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_RUN }} 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo | |
| else | |
| echo "No tests to run" | |
| fi | |
| - name: Debug ATS_TESTS_TO_SKIP | |
| run: | | |
| : ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_SKIP }} | |
| ATS_TESTS_TO_SKIP='${{ needs.coverage-ats.outputs.ATS_TESTS_TO_SKIP }}' | |
| length_of_tests=$(jq <<< ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_SKIP }} 'length') | |
| # The 1st value doesn't count, it's '--cov-context=test' | |
| if [ $length_of_tests -gt 1 ]; then | |
| echo "Running $length_of_tests tests" | |
| jq <<< ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_SKIP }} 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo | |
| else | |
| echo "No tests to run" | |
| fi |