From 5ef0acb2e689be148c0a545737dd54884e17815c Mon Sep 17 00:00:00 2001 From: John Watson Date: Sun, 8 Dec 2024 19:38:26 +0000 Subject: [PATCH] feat: dont page if no artifacts were created from the test to reduce test flake --- .github/workflows/e2e-validation.yml | 68 +++++++++++++--------- .github/workflows/run-api-test.yml | 84 ++++++++++++++++++++-------- bin/si-api-test/main.ts | 2 +- 3 files changed, 105 insertions(+), 49 deletions(-) diff --git a/.github/workflows/e2e-validation.yml b/.github/workflows/e2e-validation.yml index 034d5c04aa..777bfde128 100644 --- a/.github/workflows/e2e-validation.yml +++ b/.github/workflows/e2e-validation.yml @@ -95,25 +95,25 @@ jobs: # Run the npx task and store exit code in a variable npx cypress run --spec "cypress/e2e/${{ matrix.tests }}/**" || exit_code=$? - + # Check the exit code if [ -z "$exit_code" ]; then echo "Cypress Test task succeeded!" break fi + if [ $n -ge $max_retries ]; then + echo "All $max_retries attempts failed." + exit 1 + fi + n=$((n+1)) echo "Attempt $n/$max_retries failed with exit code $exit_code! Retrying..." done - if [ $n -ge $max_retries ]; then - echo "All $max_retries attempts failed." - exit 1 - fi - - name: 'Upload Cypress Recordings to Github' - uses: actions/upload-artifact@v4 - if: always() + uses: actions/upload-artifact@v3 + if: failure() with: name: cypress-recordings-run-${{ matrix.tests }} path: app/web/cypress/videos/**/*.mp4 @@ -126,24 +126,42 @@ jobs: on-failure: runs-on: ubuntu-latest needs: cypress-tests - if: ${{ failure() && github.ref == 'refs/heads/main' }} + environment: ${{ inputs.environment }} + if: ${{ failure() }} && github.ref == 'refs/heads/main' }} steps: - - run: | - curl --location "${{ secrets.FIREHYDRANT_WEBHOOK_URL }}" \ - --header "Content-Type: application/json" \ - --data "{ - \"summary\": \"E2E ${{ inputs.environment }} Tests Fail\", - \"body\": \"E2E Tests have failed for ${{ inputs.environment }}.\", - \"links\": [ - { - \"href\": \"https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID\", - \"text\": \"E2E Test Run ${{ inputs.environment }}\" - } - ], - \"tags\": [ - \"service:github\" - ] - }" + - name: Download all artifacts + uses: actions/download-artifact@v3 + with: + path: artifacts + - run: | + has_artifacts=false + # Check for marker files + for marker in artifacts/*/*.mp4; do + if [ -f "$marker" ]; then + echo "Artifact detected for failed test: $marker" + echo "Setting failure to true and breaking" + has_artifacts=true + break + fi + done + # If at least one valid failure marker is present, then page + if [ "$has_artifacts" = true ]; then + curl --location "${{ secrets.FIREHYDRANT_WEBHOOK_URL }}" \ + --header "Content-Type: application/json" \ + --data "{ + \"summary\": \"E2E ${{ inputs.environment }} Tests Fail\", + \"body\": \"E2E Tests have failed for ${{ inputs.environment }}.\", + \"links\": [ + { + \"href\": \"https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID\", + \"text\": \"E2E Test Run ${{ inputs.environment }}\" + } + ], + \"tags\": [ + \"service:github\" + ] + }" + fi - run: | curl -X POST \ diff --git a/.github/workflows/run-api-test.yml b/.github/workflows/run-api-test.yml index 761cab1b50..ea381ab612 100644 --- a/.github/workflows/run-api-test.yml +++ b/.github/workflows/run-api-test.yml @@ -68,6 +68,8 @@ jobs: env: SDF_API_URL: ${{ vars.SDF_API_URL }} AUTH_API_URL: ${{ vars.AUTH_API_URL }} + outputs: + last_exit_code: ${{ steps.capture-exit-code.outputs.last_exit_code }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -95,6 +97,7 @@ jobs: # Retry loop with 5 attempts n=0 max_retries=5 + exit_code=0 until [ $n -ge $max_retries ] do @@ -109,7 +112,7 @@ jobs: # Check the exit code if [ -z "$exit_code" ]; then - echo "Deno task succeeded!" + echo "Deno task succeeded [ or the orchestration failed for a totally non-valid reason ]!" break fi @@ -120,32 +123,67 @@ jobs: if [ $n -ge $max_retries ]; then echo "All $max_retries attempts failed." - exit 1 + exit_code=1 fi + echo "last_exit_code=$exit_code" >> "$GITHUB_ENV" + exit $last_exit_code + + - name: Upload artifact if exit code 53 + if: ${{ failure() && env.last_exit_code == '53' }} + run: | + echo "Uploading marker for test ${{ matrix.tests.name }}" + mkdir -p artifacts/${{ matrix.tests.name }} + echo "failure-marker" > artifacts/${{ matrix.tests.name }}/failure-marker + + - name: Upload artifacts + if: ${{ failure() && env.last_exit_code == '53' }} + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.tests.name }} + path: artifacts/${{ matrix.tests.name }} + on-failure: runs-on: ubuntu-latest needs: api-test - if: ${{ failure() && github.ref == 'refs/heads/main' }} + environment: ${{ inputs.environment }} + if: ${{ failure() }} && github.ref == 'refs/heads/main' }} steps: + - name: Download all artifacts + uses: actions/download-artifact@v3 + with: + path: artifacts + - run: | + has_artifacts=false + # Check for marker files + for marker in artifacts/*/failure-marker; do + if [ -f "$marker" ]; then + echo "Artifact detected for failed test: $marker" + echo "Setting failure to true and breaking" + has_artifacts=true + break + fi + done + # If at least one valid failure marker is present, then page + if [ "$has_artifacts" = true ]; then + curl --location "${{ secrets.FIREHYDRANT_WEBHOOK_URL }}" \ + --header "Content-Type: application/json" \ + --data "{ + \"summary\": \"API ${{ inputs.environment }} Tests Fail\", + \"body\": \"API Tests have failed for ${{ inputs.environment }}.\", + \"links\": [ + { + \"href\": \"https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID\", + \"text\": \"E2E Test Run ${{ inputs.environment }}\" + } + ], + \"tags\": [ + \"service:github\" + ] + }" + fi - run: | - curl --location "${{ secrets.FIREHYDRANT_WEBHOOK_URL }}" \ - --header "Content-Type: application/json" \ - --data "{ - \"summary\": \"API ${{ inputs.environment }} Tests Fail\", - \"body\": \"API Tests have failed for ${{ inputs.environment }}.\", - \"links\": [ - { - \"href\": \"https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID\", - \"text\": \"E2E Test Run ${{ inputs.environment }}\" - } - ], - \"tags\": [ - \"service:github\" - ] - }" - - run: | - curl -X POST \ - --header 'Content-type: application/json' \ - --data "{\"text\": \":si: Failed API Tests for ${{ inputs.environment }}: \"}" \ - ${{ secrets.SLACK_WEBHOOK_URL }} + # Always send the Internal Slack Notification if failure detected, regardless of error source + curl --location "${{ secrets.SLACK_WEBHOOK_URL }}" -X POST \ + --header 'Content-type: application/json' \ + --data "{\"text\": \":si: Failed API Tests for ${{ inputs.environment }}: \"}" diff --git a/bin/si-api-test/main.ts b/bin/si-api-test/main.ts index a1899bcfd6..c03191ecd1 100644 --- a/bin/si-api-test/main.ts +++ b/bin/si-api-test/main.ts @@ -106,7 +106,7 @@ if (import.meta.main) { clearInterval(intervalId); console.log("~~ FINAL REPORT GENERATED ~~"); await printTestReport(testReport, reportFile); - const exitCode = testsFailed(testReport) ? 1 : 0; + const exitCode = testsFailed(testReport) ? 53 : 0; Deno.exit(exitCode); }