Skip to content

Commit

Permalink
test: add e2e consensus test (#3126)
Browse files Browse the repository at this point in the history
* test: add e2e consensus test

* move to merge queue

* fix

* add e2e block production monitor
  • Loading branch information
gartnera authored Nov 7, 2024
1 parent 946eeda commit 0e86494
Show file tree
Hide file tree
Showing 5 changed files with 97 additions and 9 deletions.
36 changes: 29 additions & 7 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ jobs:
outputs:
DEFAULT_TESTS: ${{ steps.matrix-conditionals.outputs.DEFAULT_TESTS }}
UPGRADE_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_TESTS }}
CONSENSUS_TESTS: ${{ steps.matrix-conditionals.outputs.CONSENSUS_TESTS }}
UPGRADE_LIGHT_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_LIGHT_TESTS }}
UPGRADE_IMPORT_MAINNET_TESTS: ${{ steps.matrix-conditionals.outputs.UPGRADE_IMPORT_MAINNET_TESTS }}
ADMIN_TESTS: ${{ steps.matrix-conditionals.outputs.ADMIN_TESTS }}
Expand All @@ -128,16 +129,22 @@ jobs:
uses: actions/github-script@v7
with:
script: |
if (context.eventName === 'pull_request') {
const getPrLabels = async (pull_number) => {
const { data: pr } = await github.rest.pulls.get({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: context.payload.pull_request.number,
});
const labels = pr.labels.map(label => label.name);
console.log("labels:", labels);
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: pull_number,
});
const labels = pr.labels.map(label => label.name);
console.log(`labels for ${pull_number}:`, labels);
return labels;
}
if (context.eventName === 'pull_request') {
const labels = await getPrLabels(context.payload.pull_request.number);
core.setOutput('DEFAULT_TESTS', true);
core.setOutput('UPGRADE_TESTS', labels.includes('UPGRADE_TESTS'));
core.setOutput('CONSENSUS_TESTS', labels.includes('CONSENSUS_TESTS'));
core.setOutput('UPGRADE_LIGHT_TESTS', labels.includes('UPGRADE_LIGHT_TESTS'));
core.setOutput('UPGRADE_IMPORT_MAINNET_TESTS', labels.includes('UPGRADE_IMPORT_MAINNET_TESTS'));
core.setOutput('ADMIN_TESTS', labels.includes('ADMIN_TESTS'));
Expand All @@ -150,8 +157,20 @@ jobs:
core.setOutput('V2_MIGRATION_TESTS', labels.includes('V2_MIGRATION_TESTS')); // for v2 tests, TODO: remove this once we fully migrate to v2 (https://github.com/zeta-chain/node/issues/2627)
core.setOutput('ENABLE_MONITORING', labels.includes('ENABLE_MONITORING'));
} else if (context.eventName === 'merge_group') {
// default mergequeue tests
core.setOutput('DEFAULT_TESTS', true);
core.setOutput('UPGRADE_LIGHT_TESTS', true);
// conditional tests based on PR labels
const commit_message = context.payload.merge_group.head_commit.message;
const pr_match = commit_message.split('\n')[0].match(/\(#(\d+)\)$/);
if (!pr_match) {
console.error("unable to extract PR number from mergequeue commit message");
return;
}
const pr_number = pr_match[1];
const pr_labels = await getPrLabels(pr_number);
core.setOutput('CONSENSUS_TESTS', !pr_labels.includes('CONSENSUS_BREAKING_ACK'));
} else if (context.eventName === 'push' && context.ref === 'refs/heads/develop') {
core.setOutput('DEFAULT_TESTS', true);
} else if (context.eventName === 'push' && context.ref.startsWith('refs/heads/release/')) {
Expand Down Expand Up @@ -206,6 +225,9 @@ jobs:
- make-target: "start-e2e-test"
runs-on: ubuntu-20.04
run: ${{ needs.matrix-conditionals.outputs.DEFAULT_TESTS == 'true' }}
- make-target: "start-e2e-consensus-test"
runs-on: ubuntu-20.04
run: ${{ needs.matrix-conditionals.outputs.CONSENSUS_TESTS == 'true' }}
- make-target: "start-upgrade-test"
runs-on: ubuntu-20.04
run: ${{ needs.matrix-conditionals.outputs.UPGRADE_TESTS == 'true' }}
Expand Down
8 changes: 7 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@ solana:

start-e2e-test: e2e-images
@echo "--> Starting e2e test"
cd contrib/localnet/ && $(DOCKER_COMPOSE) up -d
cd contrib/localnet/ && $(DOCKER_COMPOSE) up -d

start-e2e-admin-test: e2e-images
@echo "--> Starting e2e admin test"
Expand All @@ -286,6 +286,12 @@ start-e2e-import-mainnet-test: e2e-images
export ZETACORED_START_PERIOD=15m && \
cd contrib/localnet/ && ./scripts/import-data.sh mainnet && $(DOCKER_COMPOSE) up -d

start-e2e-consensus-test: e2e-images
@echo "--> Starting e2e consensus test"
export ZETACORE1_IMAGE=ghcr.io/zeta-chain/zetanode:develop && \
export ZETACORE1_PLATFORM=linux/amd64 && \
cd contrib/localnet/ && $(DOCKER_COMPOSE) up -d

start-stress-test: e2e-images
@echo "--> Starting stress test"
cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile stress up -d
Expand Down
4 changes: 4 additions & 0 deletions cmd/zetae2e/local/local.go
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,10 @@ func localE2ETest(cmd *cobra.Command, _ []string) {
noError(deployerRunner.FundEmissionsPool())
}

// monitor block production to ensure we fail fast if there are consensus failures
// this is not run in an errgroup since only returning an error will not exit immedately
go monitorBlockProductionExit(ctx, conf)

// wait for keygen to be completed
// if setup is skipped, we assume that the keygen is already completed
if !skipSetup {
Expand Down
55 changes: 55 additions & 0 deletions cmd/zetae2e/local/monitor_block_production.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package local

import (
"context"
"fmt"
"os"
"time"

rpchttp "github.com/cometbft/cometbft/rpc/client/http"
cometbft_types "github.com/cometbft/cometbft/types"

"github.com/zeta-chain/node/e2e/config"
)

// monitorBlockProductionExit calls monitorBlockProduction and exits upon any error
func monitorBlockProductionExit(ctx context.Context, conf config.Config) {
err := monitorBlockProduction(ctx, conf)
if err != nil {
fmt.Printf("❌ block monitor: %v\n", err)
os.Exit(2)
}
}

// monitorBlockProduction subscribes to new block events to monitor if blocks are being produced
// at least every four seconds
func monitorBlockProduction(ctx context.Context, conf config.Config) error {
rpcClient, err := rpchttp.New(conf.RPCs.ZetaCoreRPC, "/websocket")
if err != nil {
return fmt.Errorf("new zetacore rpc: %w", err)
}

err = rpcClient.WSEvents.Start()
if err != nil {
return fmt.Errorf("start ws events: %w", err)
}
blockEventChan, err := rpcClient.WSEvents.Subscribe(ctx, "", "tm.event='NewBlock'")
if err != nil {
return fmt.Errorf("subscribe: %w", err)
}
latestNewBlockEvent := cometbft_types.EventDataNewBlock{}
for {
select {
case event := <-blockEventChan:
newBlockEvent, ok := event.Data.(cometbft_types.EventDataNewBlock)
if !ok {
return fmt.Errorf("expecting new block event, got %T", event.Data)
}
latestNewBlockEvent = newBlockEvent
case <-time.After(4 * time.Second):
return fmt.Errorf("timed out waiting for new block (last block %d)", latestNewBlockEvent.Block.Height)
case <-ctx.Done():
return nil
}
}
}
3 changes: 2 additions & 1 deletion contrib/localnet/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,8 @@ services:
- ~/.zetacored/genesis_data:/root/genesis_data

zetacore1:
image: zetanode:latest
image: ${ZETACORE1_IMAGE-zetanode:latest}
platform: ${ZETACORE1_PLATFORM-}
container_name: zetacore1
hostname: zetacore1
networks:
Expand Down

0 comments on commit 0e86494

Please sign in to comment.