feat: dont page if no artifacts were created from the test to reduce …

…test flake
systeminit · Dec 9, 2024 · 5ef0acb · 5ef0acb
1 parent b97e8a2
commit 5ef0acb
Show file tree

Hide file tree

Showing 3 changed files with 105 additions and 49 deletions.
diff --git a/.github/workflows/e2e-validation.yml b/.github/workflows/e2e-validation.yml
@@ -95,25 +95,25 @@ jobs:
 
             # Run the npx task and store exit code in a variable
             npx cypress run --spec "cypress/e2e/${{ matrix.tests }}/**" || exit_code=$?
-            
+
             # Check the exit code
             if [ -z "$exit_code" ]; then
               echo "Cypress Test task succeeded!"
               break
             fi
 
+            if [ $n -ge $max_retries ]; then
+              echo "All $max_retries attempts failed."
+              exit 1
+            fi
+
             n=$((n+1))
             echo "Attempt $n/$max_retries failed with exit code $exit_code! Retrying..."
           done
 
-          if [ $n -ge $max_retries ]; then
-            echo "All $max_retries attempts failed."
-            exit 1
-          fi
-
       - name: 'Upload Cypress Recordings to Github'
-        uses: actions/upload-artifact@v4
-        if: always()
+        uses: actions/upload-artifact@v3
+        if: failure()
         with:
           name: cypress-recordings-run-${{ matrix.tests }}
           path: app/web/cypress/videos/**/*.mp4
@@ -126,24 +126,42 @@ jobs:
   on-failure:
     runs-on: ubuntu-latest
     needs: cypress-tests
-    if: ${{ failure() && github.ref == 'refs/heads/main' }}
+    environment: ${{ inputs.environment }}
+    if: ${{ failure() }} && github.ref == 'refs/heads/main' }}
     steps:
-      - run: |
-          curl --location "${{ secrets.FIREHYDRANT_WEBHOOK_URL }}" \
-            --header "Content-Type: application/json" \
-            --data "{
-              \"summary\": \"E2E ${{ inputs.environment }} Tests Fail\",
-              \"body\": \"E2E Tests have failed for ${{ inputs.environment }}.\",
-              \"links\": [
-                {
-                  \"href\": \"https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID\",
-                  \"text\": \"E2E Test Run ${{ inputs.environment }}\"
-                }
-              ],
-              \"tags\": [
-                \"service:github\"
-              ]
-            }"
+      - name: Download all artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: artifacts
+      - run: |     
+          has_artifacts=false
+          # Check for marker files
+          for marker in artifacts/*/*.mp4; do
+            if [ -f "$marker" ]; then
+              echo "Artifact detected for failed test: $marker"
+              echo "Setting failure to true and breaking"
+              has_artifacts=true
+              break
+            fi
+          done
+          # If at least one valid failure marker is present, then page
+          if [ "$has_artifacts" = true ]; then
+            curl --location "${{ secrets.FIREHYDRANT_WEBHOOK_URL }}" \
+              --header "Content-Type: application/json" \
+              --data "{
+                \"summary\": \"E2E ${{ inputs.environment }} Tests Fail\",
+                \"body\": \"E2E Tests have failed for ${{ inputs.environment }}.\",
+                \"links\": [
+                  {
+                    \"href\": \"https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID\",
+                    \"text\": \"E2E Test Run ${{ inputs.environment }}\"
+                  }
+                ],
+                \"tags\": [
+                  \"service:github\"
+                ]
+              }"
+          fi
 
       - run: |
           curl -X POST \

diff --git a/.github/workflows/run-api-test.yml b/.github/workflows/run-api-test.yml
@@ -68,6 +68,8 @@ jobs:
     env:
       SDF_API_URL: ${{ vars.SDF_API_URL }}
       AUTH_API_URL: ${{ vars.AUTH_API_URL }}
+    outputs:
+      last_exit_code: ${{ steps.capture-exit-code.outputs.last_exit_code }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -95,6 +97,7 @@ jobs:
           # Retry loop with 5 attempts
           n=0
           max_retries=5
+          exit_code=0
           
           until [ $n -ge $max_retries ]
           do
@@ -109,7 +112,7 @@ jobs:
             
             # Check the exit code
             if [ -z "$exit_code" ]; then
-              echo "Deno task succeeded!"
+              echo "Deno task succeeded [ or the orchestration failed for a totally non-valid reason ]!"
               break
             fi
 
@@ -120,32 +123,67 @@ jobs:
 
           if [ $n -ge $max_retries ]; then
             echo "All $max_retries attempts failed."
-            exit 1
+            exit_code=1
           fi
 
+          echo "last_exit_code=$exit_code" >> "$GITHUB_ENV"
+          exit $last_exit_code
+
+      - name: Upload artifact if exit code 53
+        if: ${{ failure() && env.last_exit_code == '53' }}
+        run: |
+          echo "Uploading marker for test ${{ matrix.tests.name }}"
+          mkdir -p artifacts/${{ matrix.tests.name }}
+          echo "failure-marker" > artifacts/${{ matrix.tests.name }}/failure-marker
+
+      - name: Upload artifacts
+        if: ${{ failure() && env.last_exit_code == '53' }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.tests.name }}
+          path: artifacts/${{ matrix.tests.name }}
+
   on-failure:
     runs-on: ubuntu-latest
     needs: api-test
-    if: ${{ failure() && github.ref == 'refs/heads/main' }}
+    environment: ${{ inputs.environment }}
+    if: ${{ failure() }} && github.ref == 'refs/heads/main' }}
     steps:
+      - name: Download all artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: artifacts
+      - run: |     
+          has_artifacts=false
+          # Check for marker files
+          for marker in artifacts/*/failure-marker; do
+            if [ -f "$marker" ]; then
+              echo "Artifact detected for failed test: $marker"
+              echo "Setting failure to true and breaking"
+              has_artifacts=true
+              break
+            fi
+          done
+          # If at least one valid failure marker is present, then page
+          if [ "$has_artifacts" = true ]; then
+              curl --location "${{ secrets.FIREHYDRANT_WEBHOOK_URL }}" \
+              --header "Content-Type: application/json" \
+              --data "{
+                \"summary\": \"API ${{ inputs.environment }} Tests Fail\",
+                \"body\": \"API Tests have failed for ${{ inputs.environment }}.\",
+                \"links\": [
+                  {
+                    \"href\": \"https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID\",
+                    \"text\": \"E2E Test Run ${{ inputs.environment }}\"
+                  }
+                ],
+                \"tags\": [
+                  \"service:github\"
+                ]
+              }" 
+          fi
       - run: |
-            curl --location "${{ secrets.FIREHYDRANT_WEBHOOK_URL }}" \
-            --header "Content-Type: application/json" \
-            --data "{
-              \"summary\": \"API ${{ inputs.environment }} Tests Fail\",
-              \"body\": \"API Tests have failed for ${{ inputs.environment }}.\",
-              \"links\": [
-                {
-                  \"href\": \"https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID\",
-                  \"text\": \"E2E Test Run ${{ inputs.environment }}\"
-                }
-              ],
-              \"tags\": [
-                \"service:github\"
-              ]
-            }"
-      - run: |
-          curl -X POST \
-          --header 'Content-type: application/json' \
-          --data "{\"text\": \":si: Failed API Tests for ${{ inputs.environment }}: <https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID|:test_tube: Link>\"}" \
-          ${{ secrets.SLACK_WEBHOOK_URL }}
+          # Always send the Internal Slack Notification if failure detected, regardless of error source
+          curl --location "${{ secrets.SLACK_WEBHOOK_URL }}" -X POST \
+            --header 'Content-type: application/json' \
+            --data "{\"text\": \":si: Failed API Tests for ${{ inputs.environment }}: <https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID|:test_tube: Link>\"}"
diff --git a/bin/si-api-test/main.ts b/bin/si-api-test/main.ts
@@ -106,7 +106,7 @@ if (import.meta.main) {
   clearInterval(intervalId);
   console.log("~~ FINAL REPORT GENERATED ~~");
   await printTestReport(testReport, reportFile);
-  const exitCode = testsFailed(testReport) ? 1 : 0;
+  const exitCode = testsFailed(testReport) ? 53 : 0;
   Deno.exit(exitCode);
 }