test: add option to run e2e twice

zeta-chain · Dec 9, 2024 · d34a585 · d34a585
1 parent bc1d8ae
commit d34a585
Show file tree

Hide file tree

Showing 2 changed files with 32 additions and 0 deletions.
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -123,6 +123,7 @@ jobs:
       V2_TESTS: ${{ steps.matrix-conditionals.outputs.V2_TESTS }}
       V2_MIGRATION_TESTS: ${{ steps.matrix-conditionals.outputs.V2_MIGRATION_TESTS }}
       ENABLE_MONITORING: ${{ steps.matrix-conditionals.outputs.ENABLE_MONITORING }}
+      TWICE_TESTS: ${{ steps.matrix-conditionals.outputs.TWICE_TESTS }}
     steps:
       # use api rather than event context to avoid race conditions (label added after push)
       - id: matrix-conditionals
@@ -156,9 +157,11 @@ jobs:
               core.setOutput('V2_TESTS', labels.includes('V2_TESTS')); // for v2 tests, TODO: remove this once we fully migrate to v2 (https://github.com/zeta-chain/node/issues/2627)
               core.setOutput('V2_MIGRATION_TESTS', labels.includes('V2_MIGRATION_TESTS')); // for v2 tests, TODO: remove this once we fully migrate to v2 (https://github.com/zeta-chain/node/issues/2627)
               core.setOutput('ENABLE_MONITORING', labels.includes('ENABLE_MONITORING'));
+              core.setOutput('TWICE_TESTS', labels.includes('TWICE_TESTS'));
             } else if (context.eventName === 'merge_group') {
               // default mergequeue tests
               core.setOutput('DEFAULT_TESTS', true);
+              core.setOutput('TWICE_TESTS', true);
               core.setOutput('UPGRADE_LIGHT_TESTS', true);
 
               // conditional tests based on PR labels
@@ -224,6 +227,7 @@ jobs:
         include:
           - make-target: "start-e2e-test"
             runs-on: ubuntu-20.04
+            run-twice: ${{ needs.matrix-conditionals.outputs.TWICE_TESTS == 'true' }}
             run: ${{ needs.matrix-conditionals.outputs.DEFAULT_TESTS == 'true' }}
           - make-target: "start-e2e-consensus-test"
             runs-on: ubuntu-20.04
@@ -270,6 +274,7 @@ jobs:
       make-target: ${{ matrix.make-target }}
       runs-on: ${{ matrix.runs-on}}
       run: ${{ matrix.run }}
+      run-twice: ${{ matrix.run-twice || false }}
       timeout-minutes: "${{ matrix.timeout-minutes || 25 }}"
       zetanode-image: ${{ needs.build-zetanode.outputs.image }}
       enable-monitoring: ${{ needs.matrix-conditionals.outputs.ENABLE_MONITORING == 'true' }}

diff --git a/.github/workflows/reusable-e2e.yml b/.github/workflows/reusable-e2e.yml
@@ -27,6 +27,10 @@ on:
         description: 'Enable the monitoring stack for this run'
         type: boolean
         default: false
+      run-twice:
+        description: 'Run the tests a second time'
+        type: boolean
+        default: false
 
 jobs:
   e2e:
@@ -86,6 +90,29 @@ jobs:
           name: ${{ inputs.make-target }}-docker-logs
           path: /tmp/logs.txt
 
+      - name: Start Test (second run)
+        if: inputs.run-twice
+        run: make ${{ inputs.make-target }}
+
+      - name: Watch Test (second run)
+        if: inputs.run-twice
+        run: |
+          container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}")
+          docker logs -f "${container_id}" &
+          exit $(docker wait "${container_id}")
+
+      - name: Full Log Dump On Failure (second run)
+        if: inputs.run-twice && failure()
+        run: |
+          cd contrib/localnet && docker compose --profile all logs --timestamps > /tmp/logs.txt
+
+      - name: Upload Logs On Failure (second run)
+        if: inputs.run-twice && failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ inputs.make-target }}-docker-logs
+          path: /tmp/logs.txt
+
       - name: Stop Private Network
         if: always()
         run: |