test: add e2e consensus test

.github/workflows/e2e.yml

@@ -111,6 +111,7 @@ jobs:
     outputs:
       DEFAULT_TESTS: ${{ steps.matrix-conditionals.outputs.DEFAULT_TESTS }}
       UPGRADE_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_TESTS }}
+      CONSENSUS_TESTS: ${{ steps.matrix-conditionals.outputs.CONSENSUS_TESTS }}
       UPGRADE_LIGHT_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_LIGHT_TESTS }}
       UPGRADE_IMPORT_MAINNET_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_IMPORT_MAINNET_TESTS }}
       ADMIN_TESTS: ${{ steps.matrix-conditionals.outputs.ADMIN_TESTS }}
@@ -128,16 +129,22 @@ jobs:
         uses: actions/github-script@v7
         with:
           script: |
-            if (context.eventName === 'pull_request') {
+            const getPrLabels = async (pull_number) => {
               const { data: pr } = await github.rest.pulls.get({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                pull_number: context.payload.pull_request.number,
-              });
-              const labels = pr.labels.map(label => label.name);
-              console.log("labels:", labels);
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  pull_number: pull_number,
+                });
+                const labels = pr.labels.map(label => label.name);
+                console.log(`labels for ${pull_number}:`, labels);
+                return labels;
+            }
+
+            if (context.eventName === 'pull_request') {
+              const labels = await getPrLabels(context.payload.pull_request.number);
               core.setOutput('DEFAULT_TESTS', true);
               core.setOutput('UPGRADE_TESTS', labels.includes('UPGRADE_TESTS'));
+              core.setOutput('CONSENSUS_TESTS', labels.includes('CONSENSUS_TESTS'));
               core.setOutput('UPGRADE_LIGHT_TESTS', labels.includes('UPGRADE_LIGHT_TESTS'));
               core.setOutput('UPGRADE_IMPORT_MAINNET_TESTS', labels.includes('UPGRADE_IMPORT_MAINNET_TESTS'));
               core.setOutput('ADMIN_TESTS', labels.includes('ADMIN_TESTS'));
@@ -150,8 +157,20 @@ jobs:
               core.setOutput('V2_MIGRATION_TESTS', labels.includes('V2_MIGRATION_TESTS')); // for v2 tests, TODO: remove this once we fully migrate to v2 (https://github.com/zeta-chain/node/issues/2627)
               core.setOutput('ENABLE_MONITORING', labels.includes('ENABLE_MONITORING'));
             } else if (context.eventName === 'merge_group') {
+              // default mergequeue tests
               core.setOutput('DEFAULT_TESTS', true);
               core.setOutput('UPGRADE_LIGHT_TESTS', true);
+
+              // conditional tests based on PR labels
+              const commit_message = context.payload.merge_group.head_commit.message;
+              const pr_match = commit_message.split('\n')[0].match(/\(#(\d+)\)$/);
+              if (!pr_match) {
+                console.error("unable to extract PR number from mergequeue commit message");
+                return;
+              }
+              const pr_number = pr_match[1];
+              const pr_labels = await getPrLabels(pr_number);
+              core.setOutput('CONSENSUS_TESTS', !pr_labels.includes('CONSENSUS_BREAKING_ACK'));
             } else if (context.eventName === 'push' && context.ref === 'refs/heads/develop') {
               core.setOutput('DEFAULT_TESTS', true);
             } else if (context.eventName === 'push' && context.ref.startsWith('refs/heads/release/')) {
@@ -206,6 +225,9 @@ jobs:
           - make-target: "start-e2e-test"
             runs-on: ubuntu-20.04
             run: ${{ needs.matrix-conditionals.outputs.DEFAULT_TESTS == 'true' }}
+          - make-target: "start-e2e-consensus-test"
+            runs-on: ubuntu-20.04
+            run: ${{ needs.matrix-conditionals.outputs.CONSENSUS_TESTS == 'true' }}
           - make-target: "start-upgrade-test"
             runs-on: ubuntu-20.04
             run: ${{ needs.matrix-conditionals.outputs.UPGRADE_TESTS == 'true' }}

Makefile

@@ -268,7 +268,7 @@ solana:
 
 start-e2e-test: e2e-images
 	@echo "--> Starting e2e test"
-	cd contrib/localnet/ && $(DOCKER_COMPOSE) up -d 
+	cd contrib/localnet/ && $(DOCKER_COMPOSE) up -d
 
 start-e2e-admin-test: e2e-images
 	@echo "--> Starting e2e admin test"
@@ -286,6 +286,12 @@ start-e2e-import-mainnet-test: e2e-images
 	export ZETACORED_START_PERIOD=15m && \
 	cd contrib/localnet/ && ./scripts/import-data.sh mainnet && $(DOCKER_COMPOSE) up -d
 
+start-e2e-consensus-test: e2e-images
+	@echo "--> Starting e2e consensus test"
+	export ZETACORE1_IMAGE=ghcr.io/zeta-chain/zetanode:develop && \
+	export ZETACORE1_PLATFORM=linux/amd64 && \
+	cd contrib/localnet/ && $(DOCKER_COMPOSE) up -d 
+
 start-stress-test: e2e-images
 	@echo "--> Starting stress test"
 	cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile stress up -d

cmd/zetae2e/local/local.go

@@ -190,6 +190,10 @@ func localE2ETest(cmd *cobra.Command, _ []string) {
 		noError(deployerRunner.FundEmissionsPool())
 	}
 
+	// monitor block production to ensure we fail fast if there are consensus failures
+	// this is not run in an errgroup since only returning an error will not exit immedately
+	go monitorBlockProductionExit(ctx, conf)
+
 	// wait for keygen to be completed
 	// if setup is skipped, we assume that the keygen is already completed
 	if !skipSetup {

cmd/zetae2e/local/monitor_block_production.go

@@ -0,0 +1,55 @@
+package local
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	rpchttp "github.com/cometbft/cometbft/rpc/client/http"
+	cometbft_types "github.com/cometbft/cometbft/types"
+
+	"github.com/zeta-chain/node/e2e/config"
+)
+
+// monitorBlockProductionExit calls monitorBlockProduction and exits upon any error
+func monitorBlockProductionExit(ctx context.Context, conf config.Config) {
+	err := monitorBlockProduction(ctx, conf)
+	if err != nil {
+		fmt.Printf("❌ block monitor: %v\n", err)
+		os.Exit(2)
+	}
+}
+
+// monitorBlockProduction subscribes to new block events to monitor if blocks are being produced
+// at least every four seconds
+func monitorBlockProduction(ctx context.Context, conf config.Config) error {
+	rpcClient, err := rpchttp.New(conf.RPCs.ZetaCoreRPC, "/websocket")
+	if err != nil {
+		return fmt.Errorf("new zetacore rpc: %w", err)
+	}
+
+	err = rpcClient.WSEvents.Start()
+	if err != nil {
+		return fmt.Errorf("start ws events: %w", err)
+	}
+	blockEventChan, err := rpcClient.WSEvents.Subscribe(ctx, "", "tm.event='NewBlock'")
+	if err != nil {
+		return fmt.Errorf("subscribe: %w", err)
+	}
+	latestNewBlockEvent := cometbft_types.EventDataNewBlock{}
+	for {
+		select {
+		case event := <-blockEventChan:
+			newBlockEvent, ok := event.Data.(cometbft_types.EventDataNewBlock)
+			if !ok {
+				return fmt.Errorf("expecting new block event, got %T", event.Data)
+			}
+			latestNewBlockEvent = newBlockEvent
+		case <-time.After(4 * time.Second):
+			return fmt.Errorf("timed out waiting for new block (last block %d)", latestNewBlockEvent.Block.Height)
+		case <-ctx.Done():
+			return nil
+		}
+	}
+}

contrib/localnet/docker-compose.yml

@@ -67,7 +67,8 @@ services:
       - ~/.zetacored/genesis_data:/root/genesis_data
 
   zetacore1:
-    image: zetanode:latest
+    image: ${ZETACORE1_IMAGE-zetanode:latest}
+    platform: ${ZETACORE1_PLATFORM-}
     container_name: zetacore1
     hostname: zetacore1
     networks: