From 06c8a77a33fdc51f43a08555dc77f04ec87612e6 Mon Sep 17 00:00:00 2001 From: skosito Date: Tue, 10 Dec 2024 19:56:52 +0100 Subject: [PATCH 01/22] stress tests deposit solana --- Makefile | 2 +- cmd/zetae2e/local/local.go | 15 +++-- cmd/zetae2e/local/performance.go | 49 ++++++++++++++ contrib/localnet/docker-compose.yml | 1 + e2e/e2etests/e2etests.go | 12 ++++ e2e/e2etests/test_stress_solana_deposit.go | 77 ++++++++++++++++++++++ 6 files changed, 149 insertions(+), 7 deletions(-) create mode 100644 e2e/e2etests/test_stress_solana_deposit.go diff --git a/Makefile b/Makefile index 82e9c89a0a..32895faeb4 100644 --- a/Makefile +++ b/Makefile @@ -275,7 +275,7 @@ start-e2e-admin-test: e2e-images export E2E_ARGS="--skip-regular --test-admin" && \ cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile eth2 up -d -start-e2e-performance-test: e2e-images +start-e2e-performance-test: e2e-images solana @echo "--> Starting e2e performance test" export E2E_ARGS="--test-performance" && \ cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile stress up -d diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index b8e11048cb..5f17d16106 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -64,7 +64,7 @@ func NewLocalCmd() *cobra.Command { cmd.Flags().Bool(flagContractsDeployed, false, "set to to true if running tests again with existing state") cmd.Flags().Int64(flagWaitForHeight, 0, "block height for tests to begin, ex. --wait-for 100") cmd.Flags().String(FlagConfigFile, "", "config file to use for the tests") - cmd.Flags().Bool(flagVerbose, false, "set to true to enable verbose logging") + cmd.Flags().Bool(flagVerbose, true, "set to true to enable verbose logging") cmd.Flags().Bool(flagTestAdmin, false, "set to true to run admin tests") cmd.Flags().Bool(flagTestPerformance, false, "set to true to run performance tests") cmd.Flags().Bool(flagTestSolana, false, "set to true to run solana tests") @@ -110,6 +110,7 @@ func localE2ETest(cmd *cobra.Command, _ []string) { testLegacy = must(cmd.Flags().GetBool(flagTestLegacy)) skipPrecompiles = must(cmd.Flags().GetBool(flagSkipPrecompiles)) upgradeContracts = must(cmd.Flags().GetBool(flagUpgradeContracts)) + setupSolana = testSolana || testPerformance ) logger := runner.NewLogger(verbose, color.FgWhite, "setup") @@ -123,8 +124,9 @@ func localE2ETest(cmd *cobra.Command, _ []string) { } if testPerformance { - logger.Print("⚠️ performance tests enabled, regular tests will be skipped") + logger.Print("⚠️ performance tests enabled, not related tests will be skipped") skipRegular = true + skipPrecompiles = true } // start timer @@ -226,7 +228,7 @@ func localE2ETest(cmd *cobra.Command, _ []string) { // setup protocol contracts on the connected EVM chain deployerRunner.SetupEVM() - if testSolana { + if setupSolana { deployerRunner.SetupSolana( conf.Contracts.Solana.GatewayProgramID.String(), conf.AdditionalAccounts.UserSolana.SolanaPrivateKey.String(), @@ -240,7 +242,7 @@ func localE2ETest(cmd *cobra.Command, _ []string) { ERC20Addr: deployerRunner.ERC20Addr, SPLAddr: nil, } - if testSolana { + if setupSolana { zrc20Deployment.SPLAddr = deployerRunner.SPLAddr.ToPointer() } deployerRunner.SetupZEVMZRC20s(zrc20Deployment) @@ -379,8 +381,9 @@ func localE2ETest(cmd *cobra.Command, _ []string) { } if testPerformance { - eg.Go(ethereumDepositPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherDepositName)) - eg.Go(ethereumWithdrawPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherWithdrawName)) + // eg.Go(ethereumDepositPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherDepositName)) + // eg.Go(ethereumWithdrawPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherWithdrawName)) + eg.Go(solanaDepositPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressSolanaDepositName)) } if testSolana { diff --git a/cmd/zetae2e/local/performance.go b/cmd/zetae2e/local/performance.go index 9f0398b449..a46db03cd3 100644 --- a/cmd/zetae2e/local/performance.go +++ b/cmd/zetae2e/local/performance.go @@ -106,3 +106,52 @@ func ethereumWithdrawPerformanceRoutine( return err } } + +// solanaDepositPerformanceRoutine runs performance tests for Sol deposits +func solanaDepositPerformanceRoutine( + conf config.Config, + deployerRunner *runner.E2ERunner, + verbose bool, + testNames ...string, +) func() error { + return func() (err error) { + // initialize runner for solana test + r, err := initTestRunner( + "solana", + conf, + deployerRunner, + conf.AdditionalAccounts.UserSolana, + runner.NewLogger(verbose, color.FgCyan, "solana"), + runner.WithZetaTxServer(deployerRunner.ZetaTxServer), + ) + if err != nil { + return err + } + + if r.ReceiptTimeout == 0 { + r.ReceiptTimeout = 15 * time.Minute + } + if r.CctxTimeout == 0 { + r.CctxTimeout = 15 * time.Minute + } + + r.Logger.Print("🏃 starting solana deposit performance tests") + startTime := time.Now() + + tests, err := r.GetE2ETestsToRunByName( + e2etests.AllE2ETests, + testNames..., + ) + if err != nil { + return fmt.Errorf("solana deposit performance test failed: %v", err) + } + + if err := r.RunE2ETests(tests); err != nil { + return fmt.Errorf("solana deposit performance test failed: %v", err) + } + + r.Logger.Print("🍾 solana deposit performance test completed in %s", time.Since(startTime).String()) + + return err + } +} diff --git a/contrib/localnet/docker-compose.yml b/contrib/localnet/docker-compose.yml index 2494cc0247..8ea5c85551 100644 --- a/contrib/localnet/docker-compose.yml +++ b/contrib/localnet/docker-compose.yml @@ -223,6 +223,7 @@ services: profiles: - solana - all + - stress ports: - "8899:8899" networks: diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index 946d3af407..206773ba0f 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -111,6 +111,7 @@ const ( TestStressBTCWithdrawName = "stress_btc_withdraw" TestStressEtherDepositName = "stress_eth_deposit" TestStressBTCDepositName = "stress_btc_deposit" + TestStressSolanaDepositName = "stress_solana_deposit" /* Admin tests @@ -767,6 +768,17 @@ var AllE2ETests = []runner.E2ETest{ }, TestStressBTCDeposit, ), + runner.NewE2ETest( + TestStressSolanaDepositName, + "stress test SOL/SPL deposit", + []runner.ArgDefinition{ + {Description: "amount in lamports", DefaultValue: "120000"}, + {Description: "count of SOL deposits", DefaultValue: "10"}, + {Description: "amount in SPL tokens", DefaultValue: "120000"}, + {Description: "count of SPL deposits", DefaultValue: "0"}, + }, + TestStressSolanaDeposit, + ), /* Admin tests */ diff --git a/e2e/e2etests/test_stress_solana_deposit.go b/e2e/e2etests/test_stress_solana_deposit.go new file mode 100644 index 0000000000..8bab80764f --- /dev/null +++ b/e2e/e2etests/test_stress_solana_deposit.go @@ -0,0 +1,77 @@ +package e2etests + +import ( + "fmt" + "time" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" + + "github.com/zeta-chain/node/e2e/runner" + "github.com/zeta-chain/node/e2e/utils" + crosschaintypes "github.com/zeta-chain/node/x/crosschain/types" +) + +// TestStressSolanaDeposit tests the stressing deposit of SOL/SPL +func TestStressSolanaDeposit(r *runner.E2ERunner, args []string) { + require.Len(r, args, 4) + + depositSOLAmount := utils.ParseBigInt(r, args[0]) + numDepositsSOL := utils.ParseInt(r, args[1]) + + depositSPLAmount := utils.ParseBigInt(r, args[2]) + numDepositsSPL := utils.ParseInt(r, args[3]) + + // load deployer private key + privKey := r.GetSolanaPrivKey() + + r.Logger.Print("starting stress test of %d SOL and %d SPL deposits", numDepositsSOL, numDepositsSPL) + + // create a wait group to wait for all the deposits to complete + var eg errgroup.Group + + // send the deposits SOL + for i := 0; i < numDepositsSOL; i++ { + i := i + + // execute the deposit SOL transaction + sig := r.SOLDepositAndCall(nil, r.EVMAddress(), depositSOLAmount, nil) + r.Logger.Print("index %d: starting SOL deposit, sig: %s", i, sig.String()) + + eg.Go(func() error { return monitorDeposit(r, sig, i, time.Now()) }) + } + + // send the deposits SPL + for i := 0; i < numDepositsSPL; i++ { + i := i + + // execute the deposit SPL transaction + sig := r.SPLDepositAndCall(&privKey, depositSPLAmount.Uint64(), r.SPLAddr, r.EVMAddress(), nil) + r.Logger.Print("index %d: starting SPL deposit, sig: %s", i, sig.String()) + + eg.Go(func() error { return monitorDeposit(r, sig, i, time.Now()) }) + } + + require.NoError(r, eg.Wait()) + + r.Logger.Print("all deposits completed") +} + +// monitorDeposit monitors the deposit of SOL/SPL, returns once the deposit is complete +func monitorDeposit(r *runner.E2ERunner, sig solana.Signature, index int, startTime time.Time) error { + cctx := utils.WaitCctxMinedByInboundHash(r.Ctx, sig.String(), r.CctxClient, r.Logger, r.ReceiptTimeout) + if cctx.CctxStatus.Status != crosschaintypes.CctxStatus_OutboundMined { + return fmt.Errorf( + "index %d: deposit cctx failed with status %s, message %s, cctx index %s", + index, + cctx.CctxStatus.Status, + cctx.CctxStatus.StatusMessage, + cctx.Index, + ) + } + timeToComplete := time.Since(startTime) + r.Logger.Print("index %d: deposit cctx success in %s", index, timeToComplete.String()) + + return nil +} From 3fbe758e934dec84a8b39b05396ba10b9301b74c Mon Sep 17 00:00:00 2001 From: skosito Date: Wed, 11 Dec 2024 03:43:52 +0100 Subject: [PATCH 02/22] perf tests for sol withdrawals --- cmd/zetae2e/local/local.go | 1 + cmd/zetae2e/local/performance.go | 76 +++++++++++++- contrib/localnet/solana/start-solana.sh | 4 +- e2e/e2etests/e2etests.go | 28 +++-- e2e/e2etests/test_stress_solana_withdraw.go | 111 ++++++++++++++++++++ e2e/runner/solana.go | 2 +- 6 files changed, 209 insertions(+), 13 deletions(-) create mode 100644 e2e/e2etests/test_stress_solana_withdraw.go diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index 5f17d16106..faced3eab2 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -384,6 +384,7 @@ func localE2ETest(cmd *cobra.Command, _ []string) { // eg.Go(ethereumDepositPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherDepositName)) // eg.Go(ethereumWithdrawPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherWithdrawName)) eg.Go(solanaDepositPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressSolanaDepositName)) + eg.Go(solanaWithdrawPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressSolanaWithdrawName)) } if testSolana { diff --git a/cmd/zetae2e/local/performance.go b/cmd/zetae2e/local/performance.go index a46db03cd3..35e98ab967 100644 --- a/cmd/zetae2e/local/performance.go +++ b/cmd/zetae2e/local/performance.go @@ -5,6 +5,7 @@ package local import ( "fmt" + "math/big" "time" "github.com/fatih/color" @@ -12,6 +13,8 @@ import ( "github.com/zeta-chain/node/e2e/config" "github.com/zeta-chain/node/e2e/e2etests" "github.com/zeta-chain/node/e2e/runner" + "github.com/zeta-chain/node/e2e/utils" + "github.com/zeta-chain/node/x/crosschain/types" ) // ethereumDepositPerformanceRoutine runs performance tests for Ether deposit @@ -107,7 +110,7 @@ func ethereumWithdrawPerformanceRoutine( } } -// solanaDepositPerformanceRoutine runs performance tests for Sol deposits +// solanaDepositPerformanceRoutine runs performance tests for solana deposits func solanaDepositPerformanceRoutine( conf config.Config, deployerRunner *runner.E2ERunner, @@ -121,7 +124,7 @@ func solanaDepositPerformanceRoutine( conf, deployerRunner, conf.AdditionalAccounts.UserSolana, - runner.NewLogger(verbose, color.FgCyan, "solana"), + runner.NewLogger(verbose, color.FgHiMagenta, "perf_sol_deposit"), runner.WithZetaTxServer(deployerRunner.ZetaTxServer), ) if err != nil { @@ -155,3 +158,72 @@ func solanaDepositPerformanceRoutine( return err } } + +// solanaWithdrawPerformanceRoutine runs performance tests for solana withdrawals +func solanaWithdrawPerformanceRoutine( + conf config.Config, + deployerRunner *runner.E2ERunner, + verbose bool, + testNames ...string, +) func() error { + return func() (err error) { + // initialize runner for solana test + r, err := initTestRunner( + "solana", + conf, + deployerRunner, + conf.AdditionalAccounts.UserSolana, + runner.NewLogger(verbose, color.FgHiGreen, "perf_sol_withdraw"), + runner.WithZetaTxServer(deployerRunner.ZetaTxServer), + ) + if err != nil { + return err + } + + if r.ReceiptTimeout == 0 { + r.ReceiptTimeout = 15 * time.Minute + } + if r.CctxTimeout == 0 { + r.CctxTimeout = 15 * time.Minute + } + + r.Logger.Print("🏃 starting solana withdraw performance tests") + startTime := time.Now() + + // load deployer private key + privKey := r.GetSolanaPrivKey() + + // execute the deposit sol transaction + amount := big.NewInt(0).Mul(big.NewInt(1e9), big.NewInt(100)) // 100 sol in lamports + sig := r.SOLDepositAndCall(nil, r.EVMAddress(), amount, nil) + + // wait for the cctx to be mined + cctx := utils.WaitCctxMinedByInboundHash(r.Ctx, sig.String(), r.CctxClient, r.Logger, r.CctxTimeout) + r.Logger.CCTX(*cctx, "solana_deposit") + utils.RequireCCTXStatus(r, cctx, types.CctxStatus_OutboundMined) + + // same amount for spl + sig = r.SPLDepositAndCall(&privKey, amount.Uint64(), r.SPLAddr, r.EVMAddress(), nil) + + // wait for the cctx to be mined + cctx = utils.WaitCctxMinedByInboundHash(r.Ctx, sig.String(), r.CctxClient, r.Logger, r.CctxTimeout) + r.Logger.CCTX(*cctx, "solana_deposit_spl") + utils.RequireCCTXStatus(r, cctx, types.CctxStatus_OutboundMined) + + tests, err := r.GetE2ETestsToRunByName( + e2etests.AllE2ETests, + testNames..., + ) + if err != nil { + return fmt.Errorf("solana withdraw performance test failed: %v", err) + } + + if err := r.RunE2ETests(tests); err != nil { + return fmt.Errorf("solana withdraw performance test failed: %v", err) + } + + r.Logger.Print("🍾 solana withdraw performance test completed in %s", time.Since(startTime).String()) + + return err + } +} diff --git a/contrib/localnet/solana/start-solana.sh b/contrib/localnet/solana/start-solana.sh index d87e9672ae..f435cdd472 100644 --- a/contrib/localnet/solana/start-solana.sh +++ b/contrib/localnet/solana/start-solana.sh @@ -9,8 +9,8 @@ solana-test-validator & sleep 5 # airdrop to e2e sol account -solana airdrop 100 -solana airdrop 100 37yGiHAnLvWZUNVwu9esp74YQFqxU1qHCbABkDvRddUQ +solana airdrop 1000 +solana airdrop 1000 37yGiHAnLvWZUNVwu9esp74YQFqxU1qHCbABkDvRddUQ solana program deploy gateway.so diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index 206773ba0f..0028b282fb 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -107,11 +107,12 @@ const ( Stress tests Test stressing networks with many cross-chain transactions */ - TestStressEtherWithdrawName = "stress_eth_withdraw" - TestStressBTCWithdrawName = "stress_btc_withdraw" - TestStressEtherDepositName = "stress_eth_deposit" - TestStressBTCDepositName = "stress_btc_deposit" - TestStressSolanaDepositName = "stress_solana_deposit" + TestStressEtherWithdrawName = "stress_eth_withdraw" + TestStressBTCWithdrawName = "stress_btc_withdraw" + TestStressEtherDepositName = "stress_eth_deposit" + TestStressBTCDepositName = "stress_btc_deposit" + TestStressSolanaDepositName = "stress_solana_deposit" + TestStressSolanaWithdrawName = "stress_solana_withdraw" /* Admin tests @@ -772,13 +773,24 @@ var AllE2ETests = []runner.E2ETest{ TestStressSolanaDepositName, "stress test SOL/SPL deposit", []runner.ArgDefinition{ - {Description: "amount in lamports", DefaultValue: "120000"}, + {Description: "amount in lamports", DefaultValue: "1200000"}, {Description: "count of SOL deposits", DefaultValue: "10"}, - {Description: "amount in SPL tokens", DefaultValue: "120000"}, - {Description: "count of SPL deposits", DefaultValue: "0"}, + {Description: "amount in SPL tokens", DefaultValue: "1200000"}, + {Description: "count of SPL deposits", DefaultValue: "10"}, }, TestStressSolanaDeposit, ), + runner.NewE2ETest( + TestStressSolanaWithdrawName, + "stress test SOL/SPL withdrawals", + []runner.ArgDefinition{ + {Description: "amount in lamports", DefaultValue: "1000000"}, + {Description: "count of SOL withdrawals", DefaultValue: "10"}, + {Description: "amount in SPL tokens", DefaultValue: "1000000"}, + {Description: "count of SPL withdrawals", DefaultValue: "10"}, + }, + TestStressSolanaWithdraw, + ), /* Admin tests */ diff --git a/e2e/e2etests/test_stress_solana_withdraw.go b/e2e/e2etests/test_stress_solana_withdraw.go new file mode 100644 index 0000000000..daad564d4c --- /dev/null +++ b/e2e/e2etests/test_stress_solana_withdraw.go @@ -0,0 +1,111 @@ +package e2etests + +import ( + "fmt" + "math/big" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" + + "github.com/zeta-chain/node/e2e/runner" + "github.com/zeta-chain/node/e2e/utils" + crosschaintypes "github.com/zeta-chain/node/x/crosschain/types" +) + +// TestStressSolanaWithdraw tests the stressing withdrawal of SOL/SPL +func TestStressSolanaWithdraw(r *runner.E2ERunner, args []string) { + require.Len(r, args, 4) + + withdrawSOLAmount := utils.ParseBigInt(r, args[0]) + numWithdrawalsSOL := utils.ParseInt(r, args[1]) + withdrawSPLAmount := utils.ParseBigInt(r, args[2]) + numWithdrawalsSPL := utils.ParseInt(r, args[3]) + + balanceBefore, err := r.SOLZRC20.BalanceOf(&bind.CallOpts{}, r.EVMAddress()) + require.NoError(r, err) + r.Logger.Info("runner balance of SOL before withdraw: %s", balanceBefore.String()) + + balanceBefore, err = r.SPLZRC20.BalanceOf(&bind.CallOpts{}, r.EVMAddress()) + require.NoError(r, err) + r.Logger.Info("runner balance of SPL before withdraw: %s", balanceBefore.String()) + + // load deployer private key + privKey := r.GetSolanaPrivKey() + + r.Logger.Print("starting stress test of %d SOL and %d SPL withdrawals", numWithdrawalsSOL, numWithdrawalsSPL) + + tx, err := r.SOLZRC20.Approve(r.ZEVMAuth, r.SOLZRC20Addr, big.NewInt(1e18)) + require.NoError(r, err) + receipt := utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) + utils.RequireTxSuccessful(r, receipt, "approve_sol") + + tx, err = r.SPLZRC20.Approve(r.ZEVMAuth, r.SPLZRC20Addr, big.NewInt(1e18)) + require.NoError(r, err) + receipt = utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) + utils.RequireTxSuccessful(r, receipt, "approve_spl") + + tx, err = r.SOLZRC20.Approve(r.ZEVMAuth, r.SPLZRC20Addr, big.NewInt(1e18)) + require.NoError(r, err) + receipt = utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) + utils.RequireTxSuccessful(r, receipt, "approve_spl_sol") + + // create a wait group to wait for all the withdrawals to complete + var eg errgroup.Group + + // send the withdrawals SOL + for i := 0; i < numWithdrawalsSOL; i++ { + i := i + + // execute the withdraw SOL transaction + tx, err = r.SOLZRC20.Withdraw(r.ZEVMAuth, []byte(privKey.PublicKey().String()), withdrawSOLAmount) + require.NoError(r, err) + + receipt := utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) + utils.RequireTxSuccessful(r, receipt) + + r.Logger.Print("index %d: starting SOL withdraw, tx hash: %s", i, tx.Hash().Hex()) + + eg.Go(func() error { return monitorWithdrawal(r, tx.Hash(), i, time.Now()) }) + } + + // send the withdrawals SPL + for i := 0; i < numWithdrawalsSPL; i++ { + i := i + + // execute the withdraw SPL transaction + tx, err = r.SPLZRC20.Withdraw(r.ZEVMAuth, []byte(privKey.PublicKey().String()), withdrawSPLAmount) + require.NoError(r, err) + + receipt := utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) + utils.RequireTxSuccessful(r, receipt) + + r.Logger.Print("index %d: starting SPL withdraw, tx hash: %s", i, tx.Hash().Hex()) + + eg.Go(func() error { return monitorWithdrawal(r, tx.Hash(), i, time.Now()) }) + } + + require.NoError(r, eg.Wait()) + + r.Logger.Print("all withdrawals completed") +} + +// monitorWithdrawal monitors the withdrawal of SOL/SPL, returns once the withdrawal is complete +func monitorWithdrawal(r *runner.E2ERunner, hash ethcommon.Hash, index int, startTime time.Time) error { + cctx := utils.WaitCctxMinedByInboundHash(r.Ctx, hash.String(), r.CctxClient, r.Logger, r.ReceiptTimeout) + if cctx.CctxStatus.Status != crosschaintypes.CctxStatus_OutboundMined { + return fmt.Errorf( + "index %d: withdraw cctx failed with status %s, message %s, cctx index %s", + index, + cctx.CctxStatus.Status, + cctx.CctxStatus.StatusMessage, + cctx.Index, + ) + } + timeToComplete := time.Since(startTime) + r.Logger.Print("index %d: withdraw cctx success in %s", index, timeToComplete.String()) + + return nil +} diff --git a/e2e/runner/solana.go b/e2e/runner/solana.go index cc9e6dbe2b..8ee9c7ab56 100644 --- a/e2e/runner/solana.go +++ b/e2e/runner/solana.go @@ -274,7 +274,7 @@ func (r *E2ERunner) DeploySPL(privateKey *solana.PrivateKey, whitelist bool) *so // minting some tokens to deployer for testing ata := r.ResolveSolanaATA(*privateKey, privateKey.PublicKey(), mintAccount.PublicKey()) - mintToInstruction := token.NewMintToInstruction(uint64(1_000_000_000), mintAccount.PublicKey(), ata, privateKey.PublicKey(), []solana.PublicKey{}). + mintToInstruction := token.NewMintToInstruction(uint64(100_000_000_000_000), mintAccount.PublicKey(), ata, privateKey.PublicKey(), []solana.PublicKey{}). Build() signedTx = r.CreateSignedTransaction( []solana.Instruction{mintToInstruction}, From d4cacb66c7751b67f80c92bbd8aa91cec09320e7 Mon Sep 17 00:00:00 2001 From: skosito Date: Wed, 11 Dec 2024 05:05:43 +0100 Subject: [PATCH 03/22] split withdrawal stress tests into separate tests --- cmd/zetae2e/local/local.go | 4 +- e2e/e2etests/e2etests.go | 16 ++- e2e/e2etests/test_stress_solana_withdraw.go | 90 ++++++++-------- e2e/e2etests/test_stress_spl_withdraw.go | 114 ++++++++++++++++++++ 4 files changed, 170 insertions(+), 54 deletions(-) create mode 100644 e2e/e2etests/test_stress_spl_withdraw.go diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index faced3eab2..ac47e9d184 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -383,8 +383,8 @@ func localE2ETest(cmd *cobra.Command, _ []string) { if testPerformance { // eg.Go(ethereumDepositPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherDepositName)) // eg.Go(ethereumWithdrawPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherWithdrawName)) - eg.Go(solanaDepositPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressSolanaDepositName)) - eg.Go(solanaWithdrawPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressSolanaWithdrawName)) + // eg.Go(solanaDepositPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressSolanaDepositName)) + eg.Go(solanaWithdrawPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressSolanaWithdrawName, e2etests.TestStressSPLWithdrawName)) } if testSolana { diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index 0028b282fb..8e5cd55d46 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -113,6 +113,7 @@ const ( TestStressBTCDepositName = "stress_btc_deposit" TestStressSolanaDepositName = "stress_solana_deposit" TestStressSolanaWithdrawName = "stress_solana_withdraw" + TestStressSPLWithdrawName = "stress_spl_withdraw" /* Admin tests @@ -782,15 +783,22 @@ var AllE2ETests = []runner.E2ETest{ ), runner.NewE2ETest( TestStressSolanaWithdrawName, - "stress test SOL/SPL withdrawals", + "stress test SOL withdrawals", []runner.ArgDefinition{ {Description: "amount in lamports", DefaultValue: "1000000"}, - {Description: "count of SOL withdrawals", DefaultValue: "10"}, - {Description: "amount in SPL tokens", DefaultValue: "1000000"}, - {Description: "count of SPL withdrawals", DefaultValue: "10"}, + {Description: "count of SOL withdrawals", DefaultValue: "50"}, }, TestStressSolanaWithdraw, ), + runner.NewE2ETest( + TestStressSPLWithdrawName, + "stress test SPL withdrawals", + []runner.ArgDefinition{ + {Description: "amount in SPL tokens", DefaultValue: "1000000"}, + {Description: "count of SPL withdrawals", DefaultValue: "50"}, + }, + TestStressSPLWithdraw, + ), /* Admin tests */ diff --git a/e2e/e2etests/test_stress_solana_withdraw.go b/e2e/e2etests/test_stress_solana_withdraw.go index daad564d4c..930461fda8 100644 --- a/e2e/e2etests/test_stress_solana_withdraw.go +++ b/e2e/e2etests/test_stress_solana_withdraw.go @@ -3,10 +3,11 @@ package e2etests import ( "fmt" "math/big" + "sync" "time" "github.com/ethereum/go-ethereum/accounts/abi/bind" - ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/montanaflynn/stats" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" @@ -17,12 +18,10 @@ import ( // TestStressSolanaWithdraw tests the stressing withdrawal of SOL/SPL func TestStressSolanaWithdraw(r *runner.E2ERunner, args []string) { - require.Len(r, args, 4) + require.Len(r, args, 2) withdrawSOLAmount := utils.ParseBigInt(r, args[0]) numWithdrawalsSOL := utils.ParseInt(r, args[1]) - withdrawSPLAmount := utils.ParseBigInt(r, args[2]) - numWithdrawalsSPL := utils.ParseInt(r, args[3]) balanceBefore, err := r.SOLZRC20.BalanceOf(&bind.CallOpts{}, r.EVMAddress()) require.NoError(r, err) @@ -35,26 +34,20 @@ func TestStressSolanaWithdraw(r *runner.E2ERunner, args []string) { // load deployer private key privKey := r.GetSolanaPrivKey() - r.Logger.Print("starting stress test of %d SOL and %d SPL withdrawals", numWithdrawalsSOL, numWithdrawalsSPL) + r.Logger.Print("starting stress test of %d SOL withdrawals", numWithdrawalsSOL) tx, err := r.SOLZRC20.Approve(r.ZEVMAuth, r.SOLZRC20Addr, big.NewInt(1e18)) require.NoError(r, err) receipt := utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) utils.RequireTxSuccessful(r, receipt, "approve_sol") - tx, err = r.SPLZRC20.Approve(r.ZEVMAuth, r.SPLZRC20Addr, big.NewInt(1e18)) - require.NoError(r, err) - receipt = utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) - utils.RequireTxSuccessful(r, receipt, "approve_spl") - - tx, err = r.SOLZRC20.Approve(r.ZEVMAuth, r.SPLZRC20Addr, big.NewInt(1e18)) - require.NoError(r, err) - receipt = utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) - utils.RequireTxSuccessful(r, receipt, "approve_spl_sol") - // create a wait group to wait for all the withdrawals to complete var eg errgroup.Group + // store durations as float64 seconds like prometheus + withdrawDurations := []float64{} + withdrawDurationsLock := sync.Mutex{} + // send the withdrawals SOL for i := 0; i < numWithdrawalsSOL; i++ { i := i @@ -68,44 +61,45 @@ func TestStressSolanaWithdraw(r *runner.E2ERunner, args []string) { r.Logger.Print("index %d: starting SOL withdraw, tx hash: %s", i, tx.Hash().Hex()) - eg.Go(func() error { return monitorWithdrawal(r, tx.Hash(), i, time.Now()) }) + eg.Go(func() error { + startTime := time.Now() + cctx := utils.WaitCctxMinedByInboundHash(r.Ctx, tx.Hash().Hex(), r.CctxClient, r.Logger, r.ReceiptTimeout) + if cctx.CctxStatus.Status != crosschaintypes.CctxStatus_OutboundMined { + return fmt.Errorf( + "index %d: withdraw cctx failed with status %s, message %s, cctx index %s", + i, + cctx.CctxStatus.Status, + cctx.CctxStatus.StatusMessage, + cctx.Index, + ) + } + timeToComplete := time.Since(startTime) + r.Logger.Print("index %d: withdraw cctx success in %s", i, timeToComplete.String()) + + withdrawDurationsLock.Lock() + withdrawDurations = append(withdrawDurations, timeToComplete.Seconds()) + withdrawDurationsLock.Unlock() + + return nil + }) } - // send the withdrawals SPL - for i := 0; i < numWithdrawalsSPL; i++ { - i := i - - // execute the withdraw SPL transaction - tx, err = r.SPLZRC20.Withdraw(r.ZEVMAuth, []byte(privKey.PublicKey().String()), withdrawSPLAmount) - require.NoError(r, err) - - receipt := utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) - utils.RequireTxSuccessful(r, receipt) - - r.Logger.Print("index %d: starting SPL withdraw, tx hash: %s", i, tx.Hash().Hex()) + err = eg.Wait() - eg.Go(func() error { return monitorWithdrawal(r, tx.Hash(), i, time.Now()) }) + desc, descErr := stats.Describe(withdrawDurations, false, &[]float64{50.0, 75.0, 90.0, 95.0}) + if descErr != nil { + r.Logger.Print("❌ failed to calculate latency report: %v", descErr) } - require.NoError(r, eg.Wait()) - - r.Logger.Print("all withdrawals completed") -} - -// monitorWithdrawal monitors the withdrawal of SOL/SPL, returns once the withdrawal is complete -func monitorWithdrawal(r *runner.E2ERunner, hash ethcommon.Hash, index int, startTime time.Time) error { - cctx := utils.WaitCctxMinedByInboundHash(r.Ctx, hash.String(), r.CctxClient, r.Logger, r.ReceiptTimeout) - if cctx.CctxStatus.Status != crosschaintypes.CctxStatus_OutboundMined { - return fmt.Errorf( - "index %d: withdraw cctx failed with status %s, message %s, cctx index %s", - index, - cctx.CctxStatus.Status, - cctx.CctxStatus.StatusMessage, - cctx.Index, - ) + r.Logger.Print("Latency report:") + r.Logger.Print("min: %.2f", desc.Min) + r.Logger.Print("max: %.2f", desc.Max) + r.Logger.Print("mean: %.2f", desc.Mean) + r.Logger.Print("std: %.2f", desc.Std) + for _, p := range desc.DescriptionPercentiles { + r.Logger.Print("p%.0f: %.2f", p.Percentile, p.Value) } - timeToComplete := time.Since(startTime) - r.Logger.Print("index %d: withdraw cctx success in %s", index, timeToComplete.String()) - return nil + require.NoError(r, err) + r.Logger.Print("all SOL withdrawals completed") } diff --git a/e2e/e2etests/test_stress_spl_withdraw.go b/e2e/e2etests/test_stress_spl_withdraw.go new file mode 100644 index 0000000000..3484532d23 --- /dev/null +++ b/e2e/e2etests/test_stress_spl_withdraw.go @@ -0,0 +1,114 @@ +package e2etests + +import ( + "fmt" + "math/big" + "sync" + "time" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/montanaflynn/stats" + "github.com/stretchr/testify/require" + "github.com/zeta-chain/node/e2e/runner" + "github.com/zeta-chain/node/e2e/utils" + crosschaintypes "github.com/zeta-chain/node/x/crosschain/types" + "golang.org/x/sync/errgroup" +) + +// TestStressSPLWithdraw tests the stressing withdrawal of SOL/SPL +func TestStressSPLWithdraw(r *runner.E2ERunner, args []string) { + require.Len(r, args, 2) + + withdrawSPLAmount := utils.ParseBigInt(r, args[0]) + numWithdrawalsSPL := utils.ParseInt(r, args[1]) + + balanceBefore, err := r.SOLZRC20.BalanceOf(&bind.CallOpts{}, r.EVMAddress()) + require.NoError(r, err) + r.Logger.Info("runner balance of SOL before withdraw: %s", balanceBefore.String()) + + balanceBefore, err = r.SPLZRC20.BalanceOf(&bind.CallOpts{}, r.EVMAddress()) + require.NoError(r, err) + r.Logger.Info("runner balance of SPL before withdraw: %s", balanceBefore.String()) + + // load deployer private key + privKey := r.GetSolanaPrivKey() + + r.Logger.Print("starting stress test of %d SPL withdrawals", numWithdrawalsSPL) + + tx, err := r.SOLZRC20.Approve(r.ZEVMAuth, r.SOLZRC20Addr, big.NewInt(1e18)) + require.NoError(r, err) + receipt := utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) + utils.RequireTxSuccessful(r, receipt, "approve_sol") + + tx, err = r.SPLZRC20.Approve(r.ZEVMAuth, r.SPLZRC20Addr, big.NewInt(1e18)) + require.NoError(r, err) + receipt = utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) + utils.RequireTxSuccessful(r, receipt, "approve_spl") + + tx, err = r.SOLZRC20.Approve(r.ZEVMAuth, r.SPLZRC20Addr, big.NewInt(1e18)) + require.NoError(r, err) + receipt = utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) + utils.RequireTxSuccessful(r, receipt, "approve_spl_sol") + + // create a wait group to wait for all the withdrawals to complete + var eg errgroup.Group + + // store durations as float64 seconds like prometheus + withdrawDurations := []float64{} + withdrawDurationsLock := sync.Mutex{} + + // send the withdrawals SPL + for i := 0; i < numWithdrawalsSPL; i++ { + i := i + + // execute the withdraw SPL transaction + tx, err = r.SPLZRC20.Withdraw(r.ZEVMAuth, []byte(privKey.PublicKey().String()), withdrawSPLAmount) + require.NoError(r, err) + + receipt := utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) + utils.RequireTxSuccessful(r, receipt) + + r.Logger.Print("index %d: starting SPL withdraw, tx hash: %s", i, tx.Hash().Hex()) + + eg.Go(func() error { + startTime := time.Now() + cctx := utils.WaitCctxMinedByInboundHash(r.Ctx, tx.Hash().Hex(), r.CctxClient, r.Logger, r.ReceiptTimeout) + if cctx.CctxStatus.Status != crosschaintypes.CctxStatus_OutboundMined { + return fmt.Errorf( + "index %d: withdraw cctx failed with status %s, message %s, cctx index %s", + i, + cctx.CctxStatus.Status, + cctx.CctxStatus.StatusMessage, + cctx.Index, + ) + } + timeToComplete := time.Since(startTime) + r.Logger.Print("index %d: withdraw cctx success in %s", i, timeToComplete.String()) + + withdrawDurationsLock.Lock() + withdrawDurations = append(withdrawDurations, timeToComplete.Seconds()) + withdrawDurationsLock.Unlock() + + return nil + }) + } + + err = eg.Wait() + + desc, descErr := stats.Describe(withdrawDurations, false, &[]float64{50.0, 75.0, 90.0, 95.0}) + if descErr != nil { + r.Logger.Print("❌ failed to calculate latency report: %v", descErr) + } + + r.Logger.Print("Latency report:") + r.Logger.Print("min: %.2f", desc.Min) + r.Logger.Print("max: %.2f", desc.Max) + r.Logger.Print("mean: %.2f", desc.Mean) + r.Logger.Print("std: %.2f", desc.Std) + for _, p := range desc.DescriptionPercentiles { + r.Logger.Print("p%.0f: %.2f", p.Percentile, p.Value) + } + + require.NoError(r, err) + r.Logger.Print("all SPL withdrawals completed") +} From a5bf1c144e9584692cccf58c6883351544aa75fc Mon Sep 17 00:00:00 2001 From: skosito Date: Wed, 11 Dec 2024 12:12:45 +0100 Subject: [PATCH 04/22] separate deposit stress tests --- cmd/zetae2e/local/local.go | 6 ++-- e2e/e2etests/e2etests.go | 16 ++++++--- e2e/e2etests/test_stress_solana_deposit.go | 23 ++---------- e2e/e2etests/test_stress_spl_deposit.go | 41 ++++++++++++++++++++++ 4 files changed, 59 insertions(+), 27 deletions(-) create mode 100644 e2e/e2etests/test_stress_spl_deposit.go diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index ac47e9d184..eeaff65f9e 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -381,9 +381,9 @@ func localE2ETest(cmd *cobra.Command, _ []string) { } if testPerformance { - // eg.Go(ethereumDepositPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherDepositName)) - // eg.Go(ethereumWithdrawPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherWithdrawName)) - // eg.Go(solanaDepositPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressSolanaDepositName)) + eg.Go(ethereumDepositPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherDepositName)) + eg.Go(ethereumWithdrawPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherWithdrawName)) + eg.Go(solanaDepositPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressSolanaDepositName, e2etests.TestStressSPLDepositName)) eg.Go(solanaWithdrawPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressSolanaWithdrawName, e2etests.TestStressSPLWithdrawName)) } diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index 8e5cd55d46..46149e2346 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -112,6 +112,7 @@ const ( TestStressEtherDepositName = "stress_eth_deposit" TestStressBTCDepositName = "stress_btc_deposit" TestStressSolanaDepositName = "stress_solana_deposit" + TestStressSPLDepositName = "stress_spl_deposit" TestStressSolanaWithdrawName = "stress_solana_withdraw" TestStressSPLWithdrawName = "stress_spl_withdraw" @@ -772,15 +773,22 @@ var AllE2ETests = []runner.E2ETest{ ), runner.NewE2ETest( TestStressSolanaDepositName, - "stress test SOL/SPL deposit", + "stress test SOL deposit", []runner.ArgDefinition{ {Description: "amount in lamports", DefaultValue: "1200000"}, - {Description: "count of SOL deposits", DefaultValue: "10"}, - {Description: "amount in SPL tokens", DefaultValue: "1200000"}, - {Description: "count of SPL deposits", DefaultValue: "10"}, + {Description: "count of SOL deposits", DefaultValue: "50"}, }, TestStressSolanaDeposit, ), + runner.NewE2ETest( + TestStressSPLDepositName, + "stress test SPL deposit", + []runner.ArgDefinition{ + {Description: "amount in SPL tokens", DefaultValue: "1200000"}, + {Description: "count of SPL deposits", DefaultValue: "50"}, + }, + TestStressSPLDeposit, + ), runner.NewE2ETest( TestStressSolanaWithdrawName, "stress test SOL withdrawals", diff --git a/e2e/e2etests/test_stress_solana_deposit.go b/e2e/e2etests/test_stress_solana_deposit.go index 8bab80764f..03d7907d9c 100644 --- a/e2e/e2etests/test_stress_solana_deposit.go +++ b/e2e/e2etests/test_stress_solana_deposit.go @@ -13,20 +13,14 @@ import ( crosschaintypes "github.com/zeta-chain/node/x/crosschain/types" ) -// TestStressSolanaDeposit tests the stressing deposit of SOL/SPL +// TestStressSolanaDeposit tests the stressing deposit of SOL func TestStressSolanaDeposit(r *runner.E2ERunner, args []string) { - require.Len(r, args, 4) + require.Len(r, args, 2) depositSOLAmount := utils.ParseBigInt(r, args[0]) numDepositsSOL := utils.ParseInt(r, args[1]) - depositSPLAmount := utils.ParseBigInt(r, args[2]) - numDepositsSPL := utils.ParseInt(r, args[3]) - - // load deployer private key - privKey := r.GetSolanaPrivKey() - - r.Logger.Print("starting stress test of %d SOL and %d SPL deposits", numDepositsSOL, numDepositsSPL) + r.Logger.Print("starting stress test of %d SOL deposits", numDepositsSOL) // create a wait group to wait for all the deposits to complete var eg errgroup.Group @@ -42,17 +36,6 @@ func TestStressSolanaDeposit(r *runner.E2ERunner, args []string) { eg.Go(func() error { return monitorDeposit(r, sig, i, time.Now()) }) } - // send the deposits SPL - for i := 0; i < numDepositsSPL; i++ { - i := i - - // execute the deposit SPL transaction - sig := r.SPLDepositAndCall(&privKey, depositSPLAmount.Uint64(), r.SPLAddr, r.EVMAddress(), nil) - r.Logger.Print("index %d: starting SPL deposit, sig: %s", i, sig.String()) - - eg.Go(func() error { return monitorDeposit(r, sig, i, time.Now()) }) - } - require.NoError(r, eg.Wait()) r.Logger.Print("all deposits completed") diff --git a/e2e/e2etests/test_stress_spl_deposit.go b/e2e/e2etests/test_stress_spl_deposit.go new file mode 100644 index 0000000000..a37e383254 --- /dev/null +++ b/e2e/e2etests/test_stress_spl_deposit.go @@ -0,0 +1,41 @@ +package e2etests + +import ( + "time" + + "github.com/stretchr/testify/require" + "github.com/zeta-chain/node/e2e/runner" + "github.com/zeta-chain/node/e2e/utils" + "golang.org/x/sync/errgroup" +) + +// TestStressSPLDeposit tests the stressing deposit of SPL +func TestStressSPLDeposit(r *runner.E2ERunner, args []string) { + require.Len(r, args, 2) + + depositSPLAmount := utils.ParseBigInt(r, args[0]) + numDepositsSPL := utils.ParseInt(r, args[1]) + + // load deployer private key + privKey := r.GetSolanaPrivKey() + + r.Logger.Print("starting stress test of %d SPL deposits", numDepositsSPL) + + // create a wait group to wait for all the deposits to complete + var eg errgroup.Group + + // send the deposits SPL + for i := 0; i < numDepositsSPL; i++ { + i := i + + // execute the deposit SPL transaction + sig := r.SPLDepositAndCall(&privKey, depositSPLAmount.Uint64(), r.SPLAddr, r.EVMAddress(), nil) + r.Logger.Print("index %d: starting SPL deposit, sig: %s", i, sig.String()) + + eg.Go(func() error { return monitorDeposit(r, sig, i, time.Now()) }) + } + + require.NoError(r, eg.Wait()) + + r.Logger.Print("all deposits completed") +} From aca829c89e1c5e268014baf02ef862664c5949b2 Mon Sep 17 00:00:00 2001 From: skosito Date: Wed, 11 Dec 2024 16:04:09 +0100 Subject: [PATCH 05/22] add restart always to solana container --- contrib/localnet/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/localnet/docker-compose.yml b/contrib/localnet/docker-compose.yml index 8ea5c85551..354d5e1a4b 100644 --- a/contrib/localnet/docker-compose.yml +++ b/contrib/localnet/docker-compose.yml @@ -224,6 +224,7 @@ services: - solana - all - stress + restart: always ports: - "8899:8899" networks: From eb23c01e674d15c315db4f4d2dd6a94e407111b6 Mon Sep 17 00:00:00 2001 From: skosito Date: Wed, 11 Dec 2024 16:36:53 +0100 Subject: [PATCH 06/22] tweak params --- e2e/e2etests/e2etests.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index 46149e2346..a62504a135 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -776,7 +776,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SOL deposit", []runner.ArgDefinition{ {Description: "amount in lamports", DefaultValue: "1200000"}, - {Description: "count of SOL deposits", DefaultValue: "50"}, + {Description: "count of SOL deposits", DefaultValue: "30"}, }, TestStressSolanaDeposit, ), @@ -785,7 +785,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SPL deposit", []runner.ArgDefinition{ {Description: "amount in SPL tokens", DefaultValue: "1200000"}, - {Description: "count of SPL deposits", DefaultValue: "50"}, + {Description: "count of SPL deposits", DefaultValue: "30"}, }, TestStressSPLDeposit, ), @@ -794,7 +794,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SOL withdrawals", []runner.ArgDefinition{ {Description: "amount in lamports", DefaultValue: "1000000"}, - {Description: "count of SOL withdrawals", DefaultValue: "50"}, + {Description: "count of SOL withdrawals", DefaultValue: "30"}, }, TestStressSolanaWithdraw, ), @@ -803,7 +803,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SPL withdrawals", []runner.ArgDefinition{ {Description: "amount in SPL tokens", DefaultValue: "1000000"}, - {Description: "count of SPL withdrawals", DefaultValue: "50"}, + {Description: "count of SPL withdrawals", DefaultValue: "30"}, }, TestStressSPLWithdraw, ), From c0b21d43e5a6ced8fc209fa3688d00294ce660e8 Mon Sep 17 00:00:00 2001 From: skosito Date: Wed, 11 Dec 2024 18:41:47 +0100 Subject: [PATCH 07/22] fmt and changelog --- changelog.md | 1 + cmd/zetae2e/local/local.go | 20 ++++++++++++++++++-- e2e/e2etests/test_stress_spl_deposit.go | 3 ++- e2e/e2etests/test_stress_spl_withdraw.go | 3 ++- 4 files changed, 23 insertions(+), 4 deletions(-) diff --git a/changelog.md b/changelog.md index 45eb187472..42776f8e47 100644 --- a/changelog.md +++ b/changelog.md @@ -10,6 +10,7 @@ * [3205](https://github.com/zeta-chain/node/issues/3205) - move Bitcoin revert address test to advanced group to avoid upgrade test failure * [3254](https://github.com/zeta-chain/node/pull/3254) - rename v2 E2E tests as evm tests and rename old evm tests as legacy +* [3276](https://github.com/zeta-chain/node/pull/3276) - solana performance tests ## Refactor diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index eeaff65f9e..5104752f4c 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -383,8 +383,24 @@ func localE2ETest(cmd *cobra.Command, _ []string) { if testPerformance { eg.Go(ethereumDepositPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherDepositName)) eg.Go(ethereumWithdrawPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherWithdrawName)) - eg.Go(solanaDepositPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressSolanaDepositName, e2etests.TestStressSPLDepositName)) - eg.Go(solanaWithdrawPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressSolanaWithdrawName, e2etests.TestStressSPLWithdrawName)) + eg.Go( + solanaDepositPerformanceRoutine( + conf, + deployerRunner, + verbose, + e2etests.TestStressSolanaDepositName, + e2etests.TestStressSPLDepositName, + ), + ) + eg.Go( + solanaWithdrawPerformanceRoutine( + conf, + deployerRunner, + verbose, + e2etests.TestStressSolanaWithdrawName, + e2etests.TestStressSPLWithdrawName, + ), + ) } if testSolana { diff --git a/e2e/e2etests/test_stress_spl_deposit.go b/e2e/e2etests/test_stress_spl_deposit.go index a37e383254..71181419d9 100644 --- a/e2e/e2etests/test_stress_spl_deposit.go +++ b/e2e/e2etests/test_stress_spl_deposit.go @@ -4,9 +4,10 @@ import ( "time" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" + "github.com/zeta-chain/node/e2e/runner" "github.com/zeta-chain/node/e2e/utils" - "golang.org/x/sync/errgroup" ) // TestStressSPLDeposit tests the stressing deposit of SPL diff --git a/e2e/e2etests/test_stress_spl_withdraw.go b/e2e/e2etests/test_stress_spl_withdraw.go index 3484532d23..25a45422ab 100644 --- a/e2e/e2etests/test_stress_spl_withdraw.go +++ b/e2e/e2etests/test_stress_spl_withdraw.go @@ -9,10 +9,11 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/montanaflynn/stats" "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" + "github.com/zeta-chain/node/e2e/runner" "github.com/zeta-chain/node/e2e/utils" crosschaintypes "github.com/zeta-chain/node/x/crosschain/types" - "golang.org/x/sync/errgroup" ) // TestStressSPLWithdraw tests the stressing withdrawal of SOL/SPL From 646a7d24c1f69ad9cf64bce880d905e6c983c20f Mon Sep 17 00:00:00 2001 From: skosito Date: Wed, 11 Dec 2024 18:55:36 +0100 Subject: [PATCH 08/22] cleanup --- cmd/zetae2e/local/local.go | 4 ++-- e2e/e2etests/test_stress_solana_deposit.go | 2 +- e2e/e2etests/test_stress_solana_withdraw.go | 11 +---------- e2e/e2etests/test_stress_spl_deposit.go | 2 +- e2e/e2etests/test_stress_spl_withdraw.go | 11 +---------- 5 files changed, 6 insertions(+), 24 deletions(-) diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index 5104752f4c..09b790d5ac 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -64,7 +64,7 @@ func NewLocalCmd() *cobra.Command { cmd.Flags().Bool(flagContractsDeployed, false, "set to to true if running tests again with existing state") cmd.Flags().Int64(flagWaitForHeight, 0, "block height for tests to begin, ex. --wait-for 100") cmd.Flags().String(FlagConfigFile, "", "config file to use for the tests") - cmd.Flags().Bool(flagVerbose, true, "set to true to enable verbose logging") + cmd.Flags().Bool(flagVerbose, false, "set to true to enable verbose logging") cmd.Flags().Bool(flagTestAdmin, false, "set to true to run admin tests") cmd.Flags().Bool(flagTestPerformance, false, "set to true to run performance tests") cmd.Flags().Bool(flagTestSolana, false, "set to true to run solana tests") @@ -124,7 +124,7 @@ func localE2ETest(cmd *cobra.Command, _ []string) { } if testPerformance { - logger.Print("⚠️ performance tests enabled, not related tests will be skipped") + logger.Print("⚠️ performance tests enabled, regular tests will be skipped") skipRegular = true skipPrecompiles = true } diff --git a/e2e/e2etests/test_stress_solana_deposit.go b/e2e/e2etests/test_stress_solana_deposit.go index 03d7907d9c..f56e366d9a 100644 --- a/e2e/e2etests/test_stress_solana_deposit.go +++ b/e2e/e2etests/test_stress_solana_deposit.go @@ -38,7 +38,7 @@ func TestStressSolanaDeposit(r *runner.E2ERunner, args []string) { require.NoError(r, eg.Wait()) - r.Logger.Print("all deposits completed") + r.Logger.Print("all SOL deposits completed") } // monitorDeposit monitors the deposit of SOL/SPL, returns once the deposit is complete diff --git a/e2e/e2etests/test_stress_solana_withdraw.go b/e2e/e2etests/test_stress_solana_withdraw.go index 930461fda8..4d162f4bd6 100644 --- a/e2e/e2etests/test_stress_solana_withdraw.go +++ b/e2e/e2etests/test_stress_solana_withdraw.go @@ -6,7 +6,6 @@ import ( "sync" "time" - "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/montanaflynn/stats" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" @@ -16,21 +15,13 @@ import ( crosschaintypes "github.com/zeta-chain/node/x/crosschain/types" ) -// TestStressSolanaWithdraw tests the stressing withdrawal of SOL/SPL +// TestStressSolanaWithdraw tests the stressing withdrawal of SOL func TestStressSolanaWithdraw(r *runner.E2ERunner, args []string) { require.Len(r, args, 2) withdrawSOLAmount := utils.ParseBigInt(r, args[0]) numWithdrawalsSOL := utils.ParseInt(r, args[1]) - balanceBefore, err := r.SOLZRC20.BalanceOf(&bind.CallOpts{}, r.EVMAddress()) - require.NoError(r, err) - r.Logger.Info("runner balance of SOL before withdraw: %s", balanceBefore.String()) - - balanceBefore, err = r.SPLZRC20.BalanceOf(&bind.CallOpts{}, r.EVMAddress()) - require.NoError(r, err) - r.Logger.Info("runner balance of SPL before withdraw: %s", balanceBefore.String()) - // load deployer private key privKey := r.GetSolanaPrivKey() diff --git a/e2e/e2etests/test_stress_spl_deposit.go b/e2e/e2etests/test_stress_spl_deposit.go index 71181419d9..6a46241a87 100644 --- a/e2e/e2etests/test_stress_spl_deposit.go +++ b/e2e/e2etests/test_stress_spl_deposit.go @@ -38,5 +38,5 @@ func TestStressSPLDeposit(r *runner.E2ERunner, args []string) { require.NoError(r, eg.Wait()) - r.Logger.Print("all deposits completed") + r.Logger.Print("all SPL deposits completed") } diff --git a/e2e/e2etests/test_stress_spl_withdraw.go b/e2e/e2etests/test_stress_spl_withdraw.go index 25a45422ab..7dacb06058 100644 --- a/e2e/e2etests/test_stress_spl_withdraw.go +++ b/e2e/e2etests/test_stress_spl_withdraw.go @@ -6,7 +6,6 @@ import ( "sync" "time" - "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/montanaflynn/stats" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" @@ -16,21 +15,13 @@ import ( crosschaintypes "github.com/zeta-chain/node/x/crosschain/types" ) -// TestStressSPLWithdraw tests the stressing withdrawal of SOL/SPL +// TestStressSPLWithdraw tests the stressing withdrawal of SPL func TestStressSPLWithdraw(r *runner.E2ERunner, args []string) { require.Len(r, args, 2) withdrawSPLAmount := utils.ParseBigInt(r, args[0]) numWithdrawalsSPL := utils.ParseInt(r, args[1]) - balanceBefore, err := r.SOLZRC20.BalanceOf(&bind.CallOpts{}, r.EVMAddress()) - require.NoError(r, err) - r.Logger.Info("runner balance of SOL before withdraw: %s", balanceBefore.String()) - - balanceBefore, err = r.SPLZRC20.BalanceOf(&bind.CallOpts{}, r.EVMAddress()) - require.NoError(r, err) - r.Logger.Info("runner balance of SPL before withdraw: %s", balanceBefore.String()) - // load deployer private key privKey := r.GetSolanaPrivKey() From 56c94b3e943e0e8d50551a30d6a3bce497a47974 Mon Sep 17 00:00:00 2001 From: skosito Date: Wed, 11 Dec 2024 18:58:50 +0100 Subject: [PATCH 09/22] making tests execution more granular --- cmd/zetae2e/local/local.go | 14 ++++++++++++++ e2e/e2etests/e2etests.go | 8 ++++---- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index 09b790d5ac..0cd6b3d5e3 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -389,6 +389,13 @@ func localE2ETest(cmd *cobra.Command, _ []string) { deployerRunner, verbose, e2etests.TestStressSolanaDepositName, + ), + ) + eg.Go( + solanaDepositPerformanceRoutine( + conf, + deployerRunner, + verbose, e2etests.TestStressSPLDepositName, ), ) @@ -398,6 +405,13 @@ func localE2ETest(cmd *cobra.Command, _ []string) { deployerRunner, verbose, e2etests.TestStressSolanaWithdrawName, + ), + ) + eg.Go( + solanaWithdrawPerformanceRoutine( + conf, + deployerRunner, + verbose, e2etests.TestStressSPLWithdrawName, ), ) diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index a62504a135..46149e2346 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -776,7 +776,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SOL deposit", []runner.ArgDefinition{ {Description: "amount in lamports", DefaultValue: "1200000"}, - {Description: "count of SOL deposits", DefaultValue: "30"}, + {Description: "count of SOL deposits", DefaultValue: "50"}, }, TestStressSolanaDeposit, ), @@ -785,7 +785,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SPL deposit", []runner.ArgDefinition{ {Description: "amount in SPL tokens", DefaultValue: "1200000"}, - {Description: "count of SPL deposits", DefaultValue: "30"}, + {Description: "count of SPL deposits", DefaultValue: "50"}, }, TestStressSPLDeposit, ), @@ -794,7 +794,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SOL withdrawals", []runner.ArgDefinition{ {Description: "amount in lamports", DefaultValue: "1000000"}, - {Description: "count of SOL withdrawals", DefaultValue: "30"}, + {Description: "count of SOL withdrawals", DefaultValue: "50"}, }, TestStressSolanaWithdraw, ), @@ -803,7 +803,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SPL withdrawals", []runner.ArgDefinition{ {Description: "amount in SPL tokens", DefaultValue: "1000000"}, - {Description: "count of SPL withdrawals", DefaultValue: "30"}, + {Description: "count of SPL withdrawals", DefaultValue: "50"}, }, TestStressSPLWithdraw, ), From d1bf9c55299ed2014efae935a6f716ebf99fc671 Mon Sep 17 00:00:00 2001 From: skosito Date: Wed, 11 Dec 2024 19:41:14 +0100 Subject: [PATCH 10/22] revert last change --- cmd/zetae2e/local/local.go | 14 -------------- e2e/e2etests/e2etests.go | 8 ++++---- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index 0cd6b3d5e3..09b790d5ac 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -389,13 +389,6 @@ func localE2ETest(cmd *cobra.Command, _ []string) { deployerRunner, verbose, e2etests.TestStressSolanaDepositName, - ), - ) - eg.Go( - solanaDepositPerformanceRoutine( - conf, - deployerRunner, - verbose, e2etests.TestStressSPLDepositName, ), ) @@ -405,13 +398,6 @@ func localE2ETest(cmd *cobra.Command, _ []string) { deployerRunner, verbose, e2etests.TestStressSolanaWithdrawName, - ), - ) - eg.Go( - solanaWithdrawPerformanceRoutine( - conf, - deployerRunner, - verbose, e2etests.TestStressSPLWithdrawName, ), ) diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index 46149e2346..a62504a135 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -776,7 +776,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SOL deposit", []runner.ArgDefinition{ {Description: "amount in lamports", DefaultValue: "1200000"}, - {Description: "count of SOL deposits", DefaultValue: "50"}, + {Description: "count of SOL deposits", DefaultValue: "30"}, }, TestStressSolanaDeposit, ), @@ -785,7 +785,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SPL deposit", []runner.ArgDefinition{ {Description: "amount in SPL tokens", DefaultValue: "1200000"}, - {Description: "count of SPL deposits", DefaultValue: "50"}, + {Description: "count of SPL deposits", DefaultValue: "30"}, }, TestStressSPLDeposit, ), @@ -794,7 +794,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SOL withdrawals", []runner.ArgDefinition{ {Description: "amount in lamports", DefaultValue: "1000000"}, - {Description: "count of SOL withdrawals", DefaultValue: "50"}, + {Description: "count of SOL withdrawals", DefaultValue: "30"}, }, TestStressSolanaWithdraw, ), @@ -803,7 +803,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SPL withdrawals", []runner.ArgDefinition{ {Description: "amount in SPL tokens", DefaultValue: "1000000"}, - {Description: "count of SPL withdrawals", DefaultValue: "50"}, + {Description: "count of SPL withdrawals", DefaultValue: "30"}, }, TestStressSPLWithdraw, ), From 563124805d0158ecd67591bbc75ece6cfe3e81a8 Mon Sep 17 00:00:00 2001 From: skosito Date: Thu, 12 Dec 2024 02:05:50 +0100 Subject: [PATCH 11/22] split perf tests --- cmd/zetae2e/config/local.yml | 5 ++++ cmd/zetae2e/config/localnet.yml | 5 ++++ cmd/zetae2e/local/local.go | 23 +++++++++++++++++++ cmd/zetae2e/local/monitor_block_production.go | 2 +- cmd/zetae2e/local/performance.go | 16 ++++++++----- e2e/config/config.go | 6 +++++ e2e/e2etests/e2etests.go | 8 +++---- e2e/e2etests/test_stress_solana_withdraw.go | 2 +- e2e/e2etests/test_stress_spl_withdraw.go | 2 +- 9 files changed, 56 insertions(+), 13 deletions(-) diff --git a/cmd/zetae2e/config/local.yml b/cmd/zetae2e/config/local.yml index e2cc047757..b86b98adbc 100644 --- a/cmd/zetae2e/config/local.yml +++ b/cmd/zetae2e/config/local.yml @@ -31,6 +31,11 @@ additional_accounts: evm_address: "0x103FD9224F00ce3013e95629e52DFc31D805D68d" private_key: "dd53f191113d18e57bd4a5494a64a020ba7919c815d0a6d34a42ebb2839e9d95" solana_private_key: "4yqSQxDeTBvn86BuxcN5jmZb2gaobFXrBqu8kiE9rZxNkVMe3LfXmFigRsU4sRp7vk4vVP1ZCFiejDKiXBNWvs2C" + user_spl: + bech32_address: "zeta17e77anpmzhuuam67hg6x3mtqrulqh80z9chv70" + evm_address: "0xf67deecc3B15F9CEeF5eba3468ed601f3e0B9de2" + private_key: "2b3306a8ac43dbf0e350b87876c131e7e12bd49563a16de9ce8aeb664b94d559" + solana_private_key: "4yqSQxDeTBvn86BuxcN5jmZb2gaobFXrBqu8kiE9rZxNkVMe3LfXmFigRsU4sRp7vk4vVP1ZCFiejDKiXBNWvs2C" user_legacy_ether: bech32_address: "zeta134rakuus43xn63yucgxhn88ywj8ewcv6ezn2ga" evm_address: "0x8D47Db7390AC4D3D449Cc20D799ce4748F97619A" diff --git a/cmd/zetae2e/config/localnet.yml b/cmd/zetae2e/config/localnet.yml index 111f1ad7df..cb1a10dfcd 100644 --- a/cmd/zetae2e/config/localnet.yml +++ b/cmd/zetae2e/config/localnet.yml @@ -29,6 +29,11 @@ additional_accounts: evm_address: "0x103FD9224F00ce3013e95629e52DFc31D805D68d" private_key: "dd53f191113d18e57bd4a5494a64a020ba7919c815d0a6d34a42ebb2839e9d95" solana_private_key: "4yqSQxDeTBvn86BuxcN5jmZb2gaobFXrBqu8kiE9rZxNkVMe3LfXmFigRsU4sRp7vk4vVP1ZCFiejDKiXBNWvs2C" + user_spl: + bech32_address: "zeta17e77anpmzhuuam67hg6x3mtqrulqh80z9chv70" + evm_address: "0xf67deecc3B15F9CEeF5eba3468ed601f3e0B9de2" + private_key: "2b3306a8ac43dbf0e350b87876c131e7e12bd49563a16de9ce8aeb664b94d559" + solana_private_key: "4yqSQxDeTBvn86BuxcN5jmZb2gaobFXrBqu8kiE9rZxNkVMe3LfXmFigRsU4sRp7vk4vVP1ZCFiejDKiXBNWvs2C" user_legacy_ether: bech32_address: "zeta134rakuus43xn63yucgxhn88ywj8ewcv6ezn2ga" evm_address: "0x8D47Db7390AC4D3D449Cc20D799ce4748F97619A" diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index 09b790d5ac..8e6eb773dd 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -386,18 +386,41 @@ func localE2ETest(cmd *cobra.Command, _ []string) { eg.Go( solanaDepositPerformanceRoutine( conf, + "perf_sol_deposit", deployerRunner, verbose, + conf.AdditionalAccounts.UserSolana, e2etests.TestStressSolanaDepositName, + ), + ) + eg.Go( + solanaDepositPerformanceRoutine( + conf, + "perf_spl_deposit", + deployerRunner, + verbose, + conf.AdditionalAccounts.UserSPL, e2etests.TestStressSPLDepositName, ), ) eg.Go( solanaWithdrawPerformanceRoutine( conf, + "perf_sol_withdraw", deployerRunner, verbose, + conf.AdditionalAccounts.UserSolana, e2etests.TestStressSolanaWithdrawName, + ), + ) + + eg.Go( + solanaWithdrawPerformanceRoutine( + conf, + "perf_spl_withdraw", + deployerRunner, + verbose, + conf.AdditionalAccounts.UserSPL, e2etests.TestStressSPLWithdrawName, ), ) diff --git a/cmd/zetae2e/local/monitor_block_production.go b/cmd/zetae2e/local/monitor_block_production.go index 801dfc921b..4b2ddbe25d 100644 --- a/cmd/zetae2e/local/monitor_block_production.go +++ b/cmd/zetae2e/local/monitor_block_production.go @@ -46,7 +46,7 @@ func monitorBlockProduction(ctx context.Context, conf config.Config) error { return fmt.Errorf("expecting new block event, got %T", event.Data) } latestNewBlockEvent = newBlockEvent - case <-time.After(4 * time.Second): + case <-time.After(5 * time.Second): return fmt.Errorf("timed out waiting for new block (last block %d)", latestNewBlockEvent.Block.Height) case <-ctx.Done(): return nil diff --git a/cmd/zetae2e/local/performance.go b/cmd/zetae2e/local/performance.go index 35e98ab967..55a1cdf44a 100644 --- a/cmd/zetae2e/local/performance.go +++ b/cmd/zetae2e/local/performance.go @@ -113,8 +113,10 @@ func ethereumWithdrawPerformanceRoutine( // solanaDepositPerformanceRoutine runs performance tests for solana deposits func solanaDepositPerformanceRoutine( conf config.Config, + name string, deployerRunner *runner.E2ERunner, verbose bool, + account config.Account, testNames ...string, ) func() error { return func() (err error) { @@ -123,8 +125,8 @@ func solanaDepositPerformanceRoutine( "solana", conf, deployerRunner, - conf.AdditionalAccounts.UserSolana, - runner.NewLogger(verbose, color.FgHiMagenta, "perf_sol_deposit"), + account, + runner.NewLogger(verbose, color.FgHiMagenta, name), runner.WithZetaTxServer(deployerRunner.ZetaTxServer), ) if err != nil { @@ -135,7 +137,7 @@ func solanaDepositPerformanceRoutine( r.ReceiptTimeout = 15 * time.Minute } if r.CctxTimeout == 0 { - r.CctxTimeout = 15 * time.Minute + r.CctxTimeout = 30 * time.Minute } r.Logger.Print("🏃 starting solana deposit performance tests") @@ -162,8 +164,10 @@ func solanaDepositPerformanceRoutine( // solanaWithdrawPerformanceRoutine runs performance tests for solana withdrawals func solanaWithdrawPerformanceRoutine( conf config.Config, + name string, deployerRunner *runner.E2ERunner, verbose bool, + account config.Account, testNames ...string, ) func() error { return func() (err error) { @@ -172,8 +176,8 @@ func solanaWithdrawPerformanceRoutine( "solana", conf, deployerRunner, - conf.AdditionalAccounts.UserSolana, - runner.NewLogger(verbose, color.FgHiGreen, "perf_sol_withdraw"), + account, + runner.NewLogger(verbose, color.FgHiGreen, name), runner.WithZetaTxServer(deployerRunner.ZetaTxServer), ) if err != nil { @@ -184,7 +188,7 @@ func solanaWithdrawPerformanceRoutine( r.ReceiptTimeout = 15 * time.Minute } if r.CctxTimeout == 0 { - r.CctxTimeout = 15 * time.Minute + r.CctxTimeout = 30 * time.Minute } r.Logger.Print("🏃 starting solana withdraw performance tests") diff --git a/e2e/config/config.go b/e2e/config/config.go index 06377abe37..5f0b67446a 100644 --- a/e2e/config/config.go +++ b/e2e/config/config.go @@ -68,6 +68,7 @@ type AdditionalAccounts struct { UserBitcoinDeposit Account `yaml:"user_bitcoin_deposit"` UserBitcoinWithdraw Account `yaml:"user_bitcoin_withdraw"` UserSolana Account `yaml:"user_solana"` + UserSPL Account `yaml:"user_spl"` UserMisc Account `yaml:"user_misc"` UserAdmin Account `yaml:"user_admin"` UserMigration Account `yaml:"user_migration"` // used for TSS migration, TODO: rename (https://github.com/zeta-chain/node/issues/2780) @@ -240,6 +241,7 @@ func (a AdditionalAccounts) AsSlice() []Account { a.UserBitcoinDeposit, a.UserBitcoinWithdraw, a.UserSolana, + a.UserSPL, a.UserLegacyEther, a.UserMisc, a.UserAdmin, @@ -330,6 +332,10 @@ func (c *Config) GenerateKeys() error { if err != nil { return err } + c.AdditionalAccounts.UserSPL, err = generateAccount() + if err != nil { + return err + } c.AdditionalAccounts.UserLegacyEther, err = generateAccount() if err != nil { return err diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index a62504a135..46149e2346 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -776,7 +776,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SOL deposit", []runner.ArgDefinition{ {Description: "amount in lamports", DefaultValue: "1200000"}, - {Description: "count of SOL deposits", DefaultValue: "30"}, + {Description: "count of SOL deposits", DefaultValue: "50"}, }, TestStressSolanaDeposit, ), @@ -785,7 +785,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SPL deposit", []runner.ArgDefinition{ {Description: "amount in SPL tokens", DefaultValue: "1200000"}, - {Description: "count of SPL deposits", DefaultValue: "30"}, + {Description: "count of SPL deposits", DefaultValue: "50"}, }, TestStressSPLDeposit, ), @@ -794,7 +794,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SOL withdrawals", []runner.ArgDefinition{ {Description: "amount in lamports", DefaultValue: "1000000"}, - {Description: "count of SOL withdrawals", DefaultValue: "30"}, + {Description: "count of SOL withdrawals", DefaultValue: "50"}, }, TestStressSolanaWithdraw, ), @@ -803,7 +803,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SPL withdrawals", []runner.ArgDefinition{ {Description: "amount in SPL tokens", DefaultValue: "1000000"}, - {Description: "count of SPL withdrawals", DefaultValue: "30"}, + {Description: "count of SPL withdrawals", DefaultValue: "50"}, }, TestStressSPLWithdraw, ), diff --git a/e2e/e2etests/test_stress_solana_withdraw.go b/e2e/e2etests/test_stress_solana_withdraw.go index 4d162f4bd6..75c23ee524 100644 --- a/e2e/e2etests/test_stress_solana_withdraw.go +++ b/e2e/e2etests/test_stress_solana_withdraw.go @@ -65,7 +65,7 @@ func TestStressSolanaWithdraw(r *runner.E2ERunner, args []string) { ) } timeToComplete := time.Since(startTime) - r.Logger.Print("index %d: withdraw cctx success in %s", i, timeToComplete.String()) + r.Logger.Print("index %d: withdraw SOL cctx success in %s", i, timeToComplete.String()) withdrawDurationsLock.Lock() withdrawDurations = append(withdrawDurations, timeToComplete.Seconds()) diff --git a/e2e/e2etests/test_stress_spl_withdraw.go b/e2e/e2etests/test_stress_spl_withdraw.go index 7dacb06058..296def012b 100644 --- a/e2e/e2etests/test_stress_spl_withdraw.go +++ b/e2e/e2etests/test_stress_spl_withdraw.go @@ -75,7 +75,7 @@ func TestStressSPLWithdraw(r *runner.E2ERunner, args []string) { ) } timeToComplete := time.Since(startTime) - r.Logger.Print("index %d: withdraw cctx success in %s", i, timeToComplete.String()) + r.Logger.Print("index %d: withdraw SPL cctx success in %s", i, timeToComplete.String()) withdrawDurationsLock.Lock() withdrawDurations = append(withdrawDurations, timeToComplete.Seconds()) From 170b261a62de846c027bad0265061ea805a9013c Mon Sep 17 00:00:00 2001 From: skosito Date: Thu, 12 Dec 2024 03:42:02 +0100 Subject: [PATCH 12/22] tweak params --- cmd/zetae2e/local/performance.go | 6 +++--- e2e/e2etests/e2etests.go | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/zetae2e/local/performance.go b/cmd/zetae2e/local/performance.go index 55a1cdf44a..6a0b32ea58 100644 --- a/cmd/zetae2e/local/performance.go +++ b/cmd/zetae2e/local/performance.go @@ -79,7 +79,7 @@ func ethereumWithdrawPerformanceRoutine( } if r.ReceiptTimeout == 0 { - r.ReceiptTimeout = 15 * time.Minute + r.ReceiptTimeout = 30 * time.Minute } if r.CctxTimeout == 0 { r.CctxTimeout = 15 * time.Minute @@ -134,7 +134,7 @@ func solanaDepositPerformanceRoutine( } if r.ReceiptTimeout == 0 { - r.ReceiptTimeout = 15 * time.Minute + r.ReceiptTimeout = 30 * time.Minute } if r.CctxTimeout == 0 { r.CctxTimeout = 30 * time.Minute @@ -185,7 +185,7 @@ func solanaWithdrawPerformanceRoutine( } if r.ReceiptTimeout == 0 { - r.ReceiptTimeout = 15 * time.Minute + r.ReceiptTimeout = 30 * time.Minute } if r.CctxTimeout == 0 { r.CctxTimeout = 30 * time.Minute diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index 46149e2346..1dae6f58fb 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -776,7 +776,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SOL deposit", []runner.ArgDefinition{ {Description: "amount in lamports", DefaultValue: "1200000"}, - {Description: "count of SOL deposits", DefaultValue: "50"}, + {Description: "count of SOL deposits", DefaultValue: "40"}, }, TestStressSolanaDeposit, ), @@ -785,7 +785,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SPL deposit", []runner.ArgDefinition{ {Description: "amount in SPL tokens", DefaultValue: "1200000"}, - {Description: "count of SPL deposits", DefaultValue: "50"}, + {Description: "count of SPL deposits", DefaultValue: "40"}, }, TestStressSPLDeposit, ), @@ -794,7 +794,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SOL withdrawals", []runner.ArgDefinition{ {Description: "amount in lamports", DefaultValue: "1000000"}, - {Description: "count of SOL withdrawals", DefaultValue: "50"}, + {Description: "count of SOL withdrawals", DefaultValue: "40"}, }, TestStressSolanaWithdraw, ), @@ -803,7 +803,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SPL withdrawals", []runner.ArgDefinition{ {Description: "amount in SPL tokens", DefaultValue: "1000000"}, - {Description: "count of SPL withdrawals", DefaultValue: "50"}, + {Description: "count of SPL withdrawals", DefaultValue: "40"}, }, TestStressSPLWithdraw, ), From 925d5cac461152c8935a015345b2ad623bc6ba05 Mon Sep 17 00:00:00 2001 From: skosito Date: Thu, 12 Dec 2024 05:39:53 +0100 Subject: [PATCH 13/22] decrease num of withdrawals back to 30 --- e2e/e2etests/e2etests.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index 1dae6f58fb..a62504a135 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -776,7 +776,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SOL deposit", []runner.ArgDefinition{ {Description: "amount in lamports", DefaultValue: "1200000"}, - {Description: "count of SOL deposits", DefaultValue: "40"}, + {Description: "count of SOL deposits", DefaultValue: "30"}, }, TestStressSolanaDeposit, ), @@ -785,7 +785,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SPL deposit", []runner.ArgDefinition{ {Description: "amount in SPL tokens", DefaultValue: "1200000"}, - {Description: "count of SPL deposits", DefaultValue: "40"}, + {Description: "count of SPL deposits", DefaultValue: "30"}, }, TestStressSPLDeposit, ), @@ -794,7 +794,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SOL withdrawals", []runner.ArgDefinition{ {Description: "amount in lamports", DefaultValue: "1000000"}, - {Description: "count of SOL withdrawals", DefaultValue: "40"}, + {Description: "count of SOL withdrawals", DefaultValue: "30"}, }, TestStressSolanaWithdraw, ), @@ -803,7 +803,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SPL withdrawals", []runner.ArgDefinition{ {Description: "amount in SPL tokens", DefaultValue: "1000000"}, - {Description: "count of SPL withdrawals", DefaultValue: "40"}, + {Description: "count of SPL withdrawals", DefaultValue: "30"}, }, TestStressSPLWithdraw, ), From d31a7328f1af840736146021ccd9b1a938266106 Mon Sep 17 00:00:00 2001 From: skosito Date: Thu, 12 Dec 2024 06:55:18 +0100 Subject: [PATCH 14/22] run only withdrawals --- cmd/zetae2e/local/local.go | 44 +++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index 8e6eb773dd..3759521936 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -381,28 +381,28 @@ func localE2ETest(cmd *cobra.Command, _ []string) { } if testPerformance { - eg.Go(ethereumDepositPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherDepositName)) - eg.Go(ethereumWithdrawPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherWithdrawName)) - eg.Go( - solanaDepositPerformanceRoutine( - conf, - "perf_sol_deposit", - deployerRunner, - verbose, - conf.AdditionalAccounts.UserSolana, - e2etests.TestStressSolanaDepositName, - ), - ) - eg.Go( - solanaDepositPerformanceRoutine( - conf, - "perf_spl_deposit", - deployerRunner, - verbose, - conf.AdditionalAccounts.UserSPL, - e2etests.TestStressSPLDepositName, - ), - ) + // eg.Go(ethereumDepositPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherDepositName)) + // eg.Go(ethereumWithdrawPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherWithdrawName)) + // eg.Go( + // solanaDepositPerformanceRoutine( + // conf, + // "perf_sol_deposit", + // deployerRunner, + // verbose, + // conf.AdditionalAccounts.UserSolana, + // e2etests.TestStressSolanaDepositName, + // ), + // ) + // eg.Go( + // solanaDepositPerformanceRoutine( + // conf, + // "perf_spl_deposit", + // deployerRunner, + // verbose, + // conf.AdditionalAccounts.UserSPL, + // e2etests.TestStressSPLDepositName, + // ), + // ) eg.Go( solanaWithdrawPerformanceRoutine( conf, From cd01cb18f3af2ac7690831ffe293ee582d8453f0 Mon Sep 17 00:00:00 2001 From: skosito Date: Thu, 12 Dec 2024 07:11:50 +0100 Subject: [PATCH 15/22] run sequentiallyx --- cmd/zetae2e/local/local.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index 3759521936..e861b7da66 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -411,19 +411,20 @@ func localE2ETest(cmd *cobra.Command, _ []string) { verbose, conf.AdditionalAccounts.UserSolana, e2etests.TestStressSolanaWithdrawName, - ), - ) - - eg.Go( - solanaWithdrawPerformanceRoutine( - conf, - "perf_spl_withdraw", - deployerRunner, - verbose, - conf.AdditionalAccounts.UserSPL, e2etests.TestStressSPLWithdrawName, ), ) + + // eg.Go( + // solanaWithdrawPerformanceRoutine( + // conf, + // "perf_spl_withdraw", + // deployerRunner, + // verbose, + // conf.AdditionalAccounts.UserSPL, + // e2etests.TestStressSPLWithdrawName, + // ), + // ) } if testSolana { From 4daa5c39e63485d7f04e60d15e56d02447f1504d Mon Sep 17 00:00:00 2001 From: skosito Date: Thu, 12 Dec 2024 07:41:40 +0100 Subject: [PATCH 16/22] uncomment tests --- cmd/zetae2e/local/local.go | 55 +++++++++++++++----------------------- 1 file changed, 22 insertions(+), 33 deletions(-) diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index e861b7da66..696e518a74 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -381,28 +381,28 @@ func localE2ETest(cmd *cobra.Command, _ []string) { } if testPerformance { - // eg.Go(ethereumDepositPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherDepositName)) - // eg.Go(ethereumWithdrawPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherWithdrawName)) - // eg.Go( - // solanaDepositPerformanceRoutine( - // conf, - // "perf_sol_deposit", - // deployerRunner, - // verbose, - // conf.AdditionalAccounts.UserSolana, - // e2etests.TestStressSolanaDepositName, - // ), - // ) - // eg.Go( - // solanaDepositPerformanceRoutine( - // conf, - // "perf_spl_deposit", - // deployerRunner, - // verbose, - // conf.AdditionalAccounts.UserSPL, - // e2etests.TestStressSPLDepositName, - // ), - // ) + eg.Go(ethereumDepositPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherDepositName)) + eg.Go(ethereumWithdrawPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherWithdrawName)) + eg.Go( + solanaDepositPerformanceRoutine( + conf, + "perf_sol_deposit", + deployerRunner, + verbose, + conf.AdditionalAccounts.UserSolana, + e2etests.TestStressSolanaDepositName, + ), + ) + eg.Go( + solanaDepositPerformanceRoutine( + conf, + "perf_spl_deposit", + deployerRunner, + verbose, + conf.AdditionalAccounts.UserSPL, + e2etests.TestStressSPLDepositName, + ), + ) eg.Go( solanaWithdrawPerformanceRoutine( conf, @@ -414,17 +414,6 @@ func localE2ETest(cmd *cobra.Command, _ []string) { e2etests.TestStressSPLWithdrawName, ), ) - - // eg.Go( - // solanaWithdrawPerformanceRoutine( - // conf, - // "perf_spl_withdraw", - // deployerRunner, - // verbose, - // conf.AdditionalAccounts.UserSPL, - // e2etests.TestStressSPLWithdrawName, - // ), - // ) } if testSolana { From f4349a3d7520ebf1e59bd9c3dbd056950f85fba6 Mon Sep 17 00:00:00 2001 From: skosito Date: Thu, 12 Dec 2024 07:58:55 +0100 Subject: [PATCH 17/22] params --- e2e/e2etests/e2etests.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index a62504a135..46149e2346 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -776,7 +776,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SOL deposit", []runner.ArgDefinition{ {Description: "amount in lamports", DefaultValue: "1200000"}, - {Description: "count of SOL deposits", DefaultValue: "30"}, + {Description: "count of SOL deposits", DefaultValue: "50"}, }, TestStressSolanaDeposit, ), @@ -785,7 +785,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SPL deposit", []runner.ArgDefinition{ {Description: "amount in SPL tokens", DefaultValue: "1200000"}, - {Description: "count of SPL deposits", DefaultValue: "30"}, + {Description: "count of SPL deposits", DefaultValue: "50"}, }, TestStressSPLDeposit, ), @@ -794,7 +794,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SOL withdrawals", []runner.ArgDefinition{ {Description: "amount in lamports", DefaultValue: "1000000"}, - {Description: "count of SOL withdrawals", DefaultValue: "30"}, + {Description: "count of SOL withdrawals", DefaultValue: "50"}, }, TestStressSolanaWithdraw, ), @@ -803,7 +803,7 @@ var AllE2ETests = []runner.E2ETest{ "stress test SPL withdrawals", []runner.ArgDefinition{ {Description: "amount in SPL tokens", DefaultValue: "1000000"}, - {Description: "count of SPL withdrawals", DefaultValue: "30"}, + {Description: "count of SPL withdrawals", DefaultValue: "50"}, }, TestStressSPLWithdraw, ), From e4955b738cd8c38a1d3c46f5649a27472a775480 Mon Sep 17 00:00:00 2001 From: Charlie Chen <34498985+ws4charlie@users.noreply.github.com> Date: Fri, 13 Dec 2024 16:36:18 -0600 Subject: [PATCH 18/22] fix: speed up Solana withdraw stress test in two go routines (#3295) --- cmd/zetae2e/local/local.go | 9 ++ .../signer/outbound_tracker_reporter.go | 8 +- zetaclient/chains/solana/signer/signer.go | 95 +++++++++++++++---- zetaclient/orchestrator/orchestrator.go | 16 +++- 4 files changed, 99 insertions(+), 29 deletions(-) diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index 696e518a74..d2980e7f9b 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -411,6 +411,15 @@ func localE2ETest(cmd *cobra.Command, _ []string) { verbose, conf.AdditionalAccounts.UserSolana, e2etests.TestStressSolanaWithdrawName, + ), + ) + eg.Go( + solanaWithdrawPerformanceRoutine( + conf, + "perf_spl_withdraw", + deployerRunner, + verbose, + conf.AdditionalAccounts.UserSPL, e2etests.TestStressSPLWithdrawName, ), ) diff --git a/zetaclient/chains/solana/signer/outbound_tracker_reporter.go b/zetaclient/chains/solana/signer/outbound_tracker_reporter.go index 3fc8acf181..56ce65dc32 100644 --- a/zetaclient/chains/solana/signer/outbound_tracker_reporter.go +++ b/zetaclient/chains/solana/signer/outbound_tracker_reporter.go @@ -13,12 +13,6 @@ import ( "github.com/zeta-chain/node/zetaclient/logs" ) -const ( - // SolanaTransactionTimeout is the timeout for waiting for an outbound to be confirmed - // Transaction referencing a blockhash older than 150 blocks will expire and be rejected by Solana. - SolanaTransactionTimeout = 2 * time.Minute -) - // reportToOutboundTracker launch a go routine with timeout to check for tx confirmation; // it reports tx to outbound tracker only if it's confirmed by the Solana network. func (signer *Signer) reportToOutboundTracker( @@ -56,7 +50,7 @@ func (signer *Signer) reportToOutboundTracker( time.Sleep(5 * time.Second) // give up if we know the tx is too old and already expired - if time.Since(start) > SolanaTransactionTimeout { + if time.Since(start) > solanaTransactionTimeout { logger.Info().Msg("outbound is expired") return nil } diff --git a/zetaclient/chains/solana/signer/signer.go b/zetaclient/chains/solana/signer/signer.go index 72174ab5e9..6b559d4be8 100644 --- a/zetaclient/chains/solana/signer/signer.go +++ b/zetaclient/chains/solana/signer/signer.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "time" "cosmossdk.io/errors" ethcommon "github.com/ethereum/go-ethereum/common" @@ -11,6 +12,7 @@ import ( "github.com/gagliardetto/solana-go" "github.com/gagliardetto/solana-go/programs/token" "github.com/gagliardetto/solana-go/rpc" + "github.com/near/borsh-go" "github.com/rs/zerolog" "github.com/zeta-chain/node/pkg/chains" @@ -22,10 +24,24 @@ import ( "github.com/zeta-chain/node/zetaclient/chains/interfaces" "github.com/zeta-chain/node/zetaclient/compliance" "github.com/zeta-chain/node/zetaclient/keys" + "github.com/zeta-chain/node/zetaclient/logs" "github.com/zeta-chain/node/zetaclient/metrics" "github.com/zeta-chain/node/zetaclient/outboundprocessor" ) +const ( + // solanaTransactionTimeout is the timeout for waiting for an outbound to be confirmed. + // Transaction referencing a blockhash older than 150 blocks (60 ~90 secs) will expire and be rejected by Solana. + solanaTransactionTimeout = 2 * time.Minute + + // broadcastBackoff is the initial backoff duration for retrying broadcast + broadcastBackoff = 1 * time.Second + + // broadcastRetries is the maximum number of retries for broadcasting a transaction + // 6 retries will span over 1 + 2 + 4 + 8 + 16 + 32 + 64 = 127 seconds, good enough for the 2 minute timeout + broadcastRetries = 7 +) + var _ interfaces.ChainSigner = (*Signer)(nil) // Signer deals with signing Solana transactions and implements the ChainSigner interface @@ -170,26 +186,69 @@ func (signer *Signer) TryProcessOutbound( // set relayer balance metrics signer.SetRelayerBalanceMetrics(ctx) - // broadcast the signed tx to the Solana network with preflight check - txSig, err := signer.client.SendTransactionWithOpts( - ctx, - tx, - // Commitment "finalized" is too conservative for preflight check and - // it results in repeated broadcast attempts that only 1 will succeed. - // Commitment "processed" will simulate tx against more recent state - // thus fails faster once a tx is already broadcasted and processed by the cluster. - // This reduces the number of "failed" txs due to repeated broadcast attempts. - rpc.TransactionOpts{PreflightCommitment: rpc.CommitmentProcessed}, - ) - if err != nil { - logger.Error(). - Err(err). - Msgf("TryProcessOutbound: broadcast error") - return + // broadcast the signed tx to the Solana network + signer.broadcastOutbound(ctx, tx, chainID, nonce, logger, zetacoreClient) +} + +// broadcastOutbound sends the signed transaction to the Solana network +func (signer *Signer) broadcastOutbound( + ctx context.Context, + tx *solana.Transaction, + chainID int64, + nonce uint64, + logger zerolog.Logger, + zetacoreClient interfaces.ZetacoreClient, +) { + // prepare logger fields + lf := map[string]any{ + logs.FieldMethod: "broadcastOutbound", + logs.FieldNonce: nonce, + logs.FieldTx: tx.Signatures[0].String(), } - // report the outbound to the outbound tracker - signer.reportToOutboundTracker(ctx, zetacoreClient, chainID, nonce, txSig, logger) + // try broacasting tx with increasing backoff (1s, 2s, 4s, 8s, 16s, 32s, 64s) + // to tolerate tx nonce mismatch with PDA nonce or unknown RPC error + backOff := broadcastBackoff + for i := 0; i < broadcastRetries; i++ { + time.Sleep(backOff) + + // PDA nonce may already be increased by other relayer, no need to retry + pdaInfo, err := signer.client.GetAccountInfo(ctx, signer.pda) + if err != nil { + logger.Error().Err(err).Fields(lf).Msgf("unable to get PDA account info") + } else { + pda := contracts.PdaInfo{} + err = borsh.Deserialize(&pda, pdaInfo.Bytes()) + if err != nil { + logger.Error().Err(err).Fields(lf).Msgf("unable to deserialize PDA info") + } else if pda.Nonce > nonce { + logger.Info().Err(err).Fields(lf).Msgf("PDA nonce %d is greater than outbound nonce, stop retrying", pda.Nonce) + break + } + } + + // broadcast the signed tx to the Solana network with preflight check + txSig, err := signer.client.SendTransactionWithOpts( + ctx, + tx, + // Commitment "finalized" is too conservative for preflight check and + // it results in repeated broadcast attempts that only 1 will succeed. + // Commitment "processed" will simulate tx against more recent state + // thus fails faster once a tx is already broadcasted and processed by the cluster. + // This reduces the number of "failed" txs due to repeated broadcast attempts. + rpc.TransactionOpts{PreflightCommitment: rpc.CommitmentProcessed}, + ) + if err != nil { + logger.Warn().Err(err).Fields(lf).Msgf("SendTransactionWithOpts failed") + backOff *= 2 + continue + } + logger.Info().Fields(lf).Msgf("broadcasted Solana outbound successfully") + + // successful broadcast; report to the outbound tracker + signer.reportToOutboundTracker(ctx, zetacoreClient, chainID, nonce, txSig, logger) + break + } } func (signer *Signer) prepareWithdrawTx( diff --git a/zetaclient/orchestrator/orchestrator.go b/zetaclient/orchestrator/orchestrator.go index 0e50777f10..36a2588ba4 100644 --- a/zetaclient/orchestrator/orchestrator.go +++ b/zetaclient/orchestrator/orchestrator.go @@ -34,11 +34,11 @@ import ( ) const ( - // evmOutboundLookbackFactor is the factor to determine how many nonces to look back for pending cctxs + // outboundLookbackFactor is the factor to determine how many nonces to look back for pending cctxs // For example, give OutboundScheduleLookahead of 120, pending NonceLow of 1000 and factor of 1.0, // the scheduler need to be able to pick up and schedule any pending cctx with nonce < 880 (1000 - 120 * 1.0) // NOTE: 1.0 means look back the same number of cctxs as we look ahead - evmOutboundLookbackFactor = 1.0 + outboundLookbackFactor = 1.0 // sampling rate for sampled orchestrator logger loggerSamplingRate = 10 @@ -428,7 +428,7 @@ func (oc *Orchestrator) ScheduleCctxEVM( } outboundScheduleLookahead := observer.ChainParams().OutboundScheduleLookahead // #nosec G115 always in range - outboundScheduleLookback := uint64(float64(outboundScheduleLookahead) * evmOutboundLookbackFactor) + outboundScheduleLookback := uint64(float64(outboundScheduleLookahead) * outboundLookbackFactor) // #nosec G115 positive outboundScheduleInterval := uint64(observer.ChainParams().OutboundScheduleInterval) criticalInterval := uint64(10) // for critical pending outbound we reduce re-try interval @@ -596,8 +596,11 @@ func (oc *Orchestrator) ScheduleCctxSolana( oc.logger.Error().Msgf("ScheduleCctxSolana: chain observer is not a solana observer") return } - // #nosec G115 positive + + // outbound keysign scheduler parameters interval := uint64(observer.ChainParams().OutboundScheduleInterval) + outboundScheduleLookahead := observer.ChainParams().OutboundScheduleLookahead + outboundScheduleLookback := uint64(float64(outboundScheduleLookahead) * outboundLookbackFactor) // schedule keysign for each pending cctx for _, cctx := range cctxList { @@ -610,6 +613,11 @@ func (oc *Orchestrator) ScheduleCctxSolana( Msgf("ScheduleCctxSolana: outbound %s chainid mismatch: want %d, got %d", outboundID, chainID, params.ReceiverChainId) continue } + if params.TssNonce > cctxList[0].GetCurrentOutboundParam().TssNonce+outboundScheduleLookback { + oc.logger.Warn().Msgf("ScheduleCctxSolana: nonce too high: signing %d, earliest pending %d", + params.TssNonce, cctxList[0].GetCurrentOutboundParam().TssNonce) + break + } // vote outbound if it's already confirmed continueKeysign, err := solObserver.VoteOutboundIfConfirmed(ctx, cctx) From c5c85d3e32eedd3ff87c954e1fc1206a79981ccb Mon Sep 17 00:00:00 2001 From: skosito Date: Sun, 15 Dec 2024 06:08:11 +0100 Subject: [PATCH 19/22] PR comment and lint --- changelog.md | 2 +- zetaclient/orchestrator/orchestrator.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 081c1e1bd6..3e8842a4ab 100644 --- a/changelog.md +++ b/changelog.md @@ -11,7 +11,7 @@ * [3205](https://github.com/zeta-chain/node/issues/3205) - move Bitcoin revert address test to advanced group to avoid upgrade test failure * [3254](https://github.com/zeta-chain/node/pull/3254) - rename v2 E2E tests as evm tests and rename old evm tests as legacy * [3095](https://github.com/zeta-chain/node/pull/3095) - initialize simulation tests for custom zetachain modules -* [3276](https://github.com/zeta-chain/node/pull/3276) - solana performance tests +* [3276](https://github.com/zeta-chain/node/pull/3276) - add Solana E2E performance tests ## Refactor diff --git a/zetaclient/orchestrator/orchestrator.go b/zetaclient/orchestrator/orchestrator.go index 36a2588ba4..6e818329b1 100644 --- a/zetaclient/orchestrator/orchestrator.go +++ b/zetaclient/orchestrator/orchestrator.go @@ -598,6 +598,7 @@ func (oc *Orchestrator) ScheduleCctxSolana( } // outbound keysign scheduler parameters + // #nosec G115 positive interval := uint64(observer.ChainParams().OutboundScheduleInterval) outboundScheduleLookahead := observer.ChainParams().OutboundScheduleLookahead outboundScheduleLookback := uint64(float64(outboundScheduleLookahead) * outboundLookbackFactor) From 602fa5f7e9276e1a3da0c0b2d5381a8e2573e1e2 Mon Sep 17 00:00:00 2001 From: skosito Date: Sun, 15 Dec 2024 06:27:51 +0100 Subject: [PATCH 20/22] changelog --- changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 3e8842a4ab..f04eeaa30a 100644 --- a/changelog.md +++ b/changelog.md @@ -11,7 +11,7 @@ * [3205](https://github.com/zeta-chain/node/issues/3205) - move Bitcoin revert address test to advanced group to avoid upgrade test failure * [3254](https://github.com/zeta-chain/node/pull/3254) - rename v2 E2E tests as evm tests and rename old evm tests as legacy * [3095](https://github.com/zeta-chain/node/pull/3095) - initialize simulation tests for custom zetachain modules -* [3276](https://github.com/zeta-chain/node/pull/3276) - add Solana E2E performance tests +* [3276](https://github.com/zeta-chain/node/pull/3276) - add Solana E2E performance tests and improve Solana outbounds performance ## Refactor From 61f8aace8c5c8b3254db50cbef3fc19d07c08a37 Mon Sep 17 00:00:00 2001 From: skosito Date: Mon, 16 Dec 2024 15:09:00 +0100 Subject: [PATCH 21/22] move account info decoding to solana package --- pkg/contracts/solana/account_info.go | 30 +++++++++++++++++++++++ zetaclient/chains/solana/signer/signer.go | 15 ++---------- 2 files changed, 32 insertions(+), 13 deletions(-) create mode 100644 pkg/contracts/solana/account_info.go diff --git a/pkg/contracts/solana/account_info.go b/pkg/contracts/solana/account_info.go new file mode 100644 index 0000000000..31bec2356e --- /dev/null +++ b/pkg/contracts/solana/account_info.go @@ -0,0 +1,30 @@ +package solana + +import ( + bin "github.com/gagliardetto/binary" + "github.com/gagliardetto/solana-go/programs/token" + solrpc "github.com/gagliardetto/solana-go/rpc" + "github.com/near/borsh-go" +) + +func DeserializePdaInfo(pdaInfo *solrpc.GetAccountInfoResult) (PdaInfo, error) { + pda := PdaInfo{} + err := borsh.Deserialize(&pda, pdaInfo.Bytes()) + if err != nil { + return PdaInfo{}, err + } + + return pda, nil +} + +func DeserializeMintAccountInfo(mintInfo *solrpc.GetAccountInfoResult) (token.Mint, error) { + var mint token.Mint + // Account{}.Data.GetBinary() returns the *decoded* binary data + // regardless the original encoding (it can handle them all). + err := bin.NewBinDecoder(mintInfo.Value.Data.GetBinary()).Decode(&mint) + if err != nil { + return token.Mint{}, err + } + + return mint, err +} diff --git a/zetaclient/chains/solana/signer/signer.go b/zetaclient/chains/solana/signer/signer.go index 6b559d4be8..b52d0419b2 100644 --- a/zetaclient/chains/solana/signer/signer.go +++ b/zetaclient/chains/solana/signer/signer.go @@ -8,11 +8,9 @@ import ( "cosmossdk.io/errors" ethcommon "github.com/ethereum/go-ethereum/common" - bin "github.com/gagliardetto/binary" "github.com/gagliardetto/solana-go" "github.com/gagliardetto/solana-go/programs/token" "github.com/gagliardetto/solana-go/rpc" - "github.com/near/borsh-go" "github.com/rs/zerolog" "github.com/zeta-chain/node/pkg/chains" @@ -217,8 +215,7 @@ func (signer *Signer) broadcastOutbound( if err != nil { logger.Error().Err(err).Fields(lf).Msgf("unable to get PDA account info") } else { - pda := contracts.PdaInfo{} - err = borsh.Deserialize(&pda, pdaInfo.Bytes()) + pda, err := contracts.DeserializePdaInfo(pdaInfo) if err != nil { logger.Error().Err(err).Fields(lf).Msgf("unable to deserialize PDA info") } else if pda.Nonce > nonce { @@ -381,15 +378,7 @@ func (signer *Signer) decodeMintAccountDetails(ctx context.Context, asset string return token.Mint{}, err } - var mint token.Mint - // Account{}.Data.GetBinary() returns the *decoded* binary data - // regardless the original encoding (it can handle them all). - err = bin.NewBinDecoder(info.Value.Data.GetBinary()).Decode(&mint) - if err != nil { - return token.Mint{}, err - } - - return mint, nil + return contracts.DeserializeMintAccountInfo(info) } // SetGatewayAddress sets the gateway address From 4218f407529207cca57235f18fc27e8efe1562f9 Mon Sep 17 00:00:00 2001 From: skosito Date: Mon, 16 Dec 2024 15:19:30 +0100 Subject: [PATCH 22/22] revert timeouts to prev values --- cmd/zetae2e/local/performance.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/zetae2e/local/performance.go b/cmd/zetae2e/local/performance.go index 6a0b32ea58..312dc91665 100644 --- a/cmd/zetae2e/local/performance.go +++ b/cmd/zetae2e/local/performance.go @@ -79,7 +79,7 @@ func ethereumWithdrawPerformanceRoutine( } if r.ReceiptTimeout == 0 { - r.ReceiptTimeout = 30 * time.Minute + r.ReceiptTimeout = 15 * time.Minute } if r.CctxTimeout == 0 { r.CctxTimeout = 15 * time.Minute @@ -134,10 +134,10 @@ func solanaDepositPerformanceRoutine( } if r.ReceiptTimeout == 0 { - r.ReceiptTimeout = 30 * time.Minute + r.ReceiptTimeout = 15 * time.Minute } if r.CctxTimeout == 0 { - r.CctxTimeout = 30 * time.Minute + r.CctxTimeout = 15 * time.Minute } r.Logger.Print("🏃 starting solana deposit performance tests") @@ -185,10 +185,10 @@ func solanaWithdrawPerformanceRoutine( } if r.ReceiptTimeout == 0 { - r.ReceiptTimeout = 30 * time.Minute + r.ReceiptTimeout = 15 * time.Minute } if r.CctxTimeout == 0 { - r.CctxTimeout = 30 * time.Minute + r.CctxTimeout = 15 * time.Minute } r.Logger.Print("🏃 starting solana withdraw performance tests")