diff --git a/Makefile b/Makefile index 7efc19b988..b348184f54 100644 --- a/Makefile +++ b/Makefile @@ -275,7 +275,7 @@ start-e2e-admin-test: e2e-images export E2E_ARGS="--skip-regular --test-admin" && \ cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile eth2 up -d -start-e2e-performance-test: e2e-images +start-e2e-performance-test: e2e-images solana @echo "--> Starting e2e performance test" export E2E_ARGS="--test-performance" && \ cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile stress up -d diff --git a/cmd/zetae2e/config/local.yml b/cmd/zetae2e/config/local.yml index 15de10ceda..3b48fde053 100644 --- a/cmd/zetae2e/config/local.yml +++ b/cmd/zetae2e/config/local.yml @@ -29,6 +29,11 @@ additional_accounts: evm_address: "0x103FD9224F00ce3013e95629e52DFc31D805D68d" private_key: "dd53f191113d18e57bd4a5494a64a020ba7919c815d0a6d34a42ebb2839e9d95" solana_private_key: "4yqSQxDeTBvn86BuxcN5jmZb2gaobFXrBqu8kiE9rZxNkVMe3LfXmFigRsU4sRp7vk4vVP1ZCFiejDKiXBNWvs2C" + user_spl: + bech32_address: "zeta17e77anpmzhuuam67hg6x3mtqrulqh80z9chv70" + evm_address: "0xf67deecc3B15F9CEeF5eba3468ed601f3e0B9de2" + private_key: "2b3306a8ac43dbf0e350b87876c131e7e12bd49563a16de9ce8aeb664b94d559" + solana_private_key: "4yqSQxDeTBvn86BuxcN5jmZb2gaobFXrBqu8kiE9rZxNkVMe3LfXmFigRsU4sRp7vk4vVP1ZCFiejDKiXBNWvs2C" user_ether: bech32_address: "zeta134rakuus43xn63yucgxhn88ywj8ewcv6ezn2ga" evm_address: "0x8D47Db7390AC4D3D449Cc20D799ce4748F97619A" diff --git a/cmd/zetae2e/config/localnet.yml b/cmd/zetae2e/config/localnet.yml index 6e1c2b392c..2b64781876 100644 --- a/cmd/zetae2e/config/localnet.yml +++ b/cmd/zetae2e/config/localnet.yml @@ -29,6 +29,11 @@ additional_accounts: evm_address: "0x103FD9224F00ce3013e95629e52DFc31D805D68d" private_key: "dd53f191113d18e57bd4a5494a64a020ba7919c815d0a6d34a42ebb2839e9d95" solana_private_key: "4yqSQxDeTBvn86BuxcN5jmZb2gaobFXrBqu8kiE9rZxNkVMe3LfXmFigRsU4sRp7vk4vVP1ZCFiejDKiXBNWvs2C" + user_spl: + bech32_address: "zeta17e77anpmzhuuam67hg6x3mtqrulqh80z9chv70" + evm_address: "0xf67deecc3B15F9CEeF5eba3468ed601f3e0B9de2" + private_key: "2b3306a8ac43dbf0e350b87876c131e7e12bd49563a16de9ce8aeb664b94d559" + solana_private_key: "4yqSQxDeTBvn86BuxcN5jmZb2gaobFXrBqu8kiE9rZxNkVMe3LfXmFigRsU4sRp7vk4vVP1ZCFiejDKiXBNWvs2C" user_ether: bech32_address: "zeta134rakuus43xn63yucgxhn88ywj8ewcv6ezn2ga" evm_address: "0x8D47Db7390AC4D3D449Cc20D799ce4748F97619A" diff --git a/cmd/zetae2e/local/local.go b/cmd/zetae2e/local/local.go index acc69f3be8..6f939a3757 100644 --- a/cmd/zetae2e/local/local.go +++ b/cmd/zetae2e/local/local.go @@ -116,6 +116,7 @@ func localE2ETest(cmd *cobra.Command, _ []string) { testV2Migration = must(cmd.Flags().GetBool(flagTestV2Migration)) skipPrecompiles = must(cmd.Flags().GetBool(flagSkipPrecompiles)) upgradeContracts = must(cmd.Flags().GetBool(flagUpgradeContracts)) + setupSolana = testSolana || testPerformance ) logger := runner.NewLogger(verbose, color.FgWhite, "setup") @@ -230,7 +231,7 @@ func localE2ETest(cmd *cobra.Command, _ []string) { deployerRunner.SetupEVMV2() - if testSolana { + if setupSolana { deployerRunner.SetupSolana( conf.Contracts.Solana.GatewayProgramID.String(), conf.AdditionalAccounts.UserSolana.SolanaPrivateKey.String(), @@ -246,7 +247,7 @@ func localE2ETest(cmd *cobra.Command, _ []string) { ERC20Addr: deployerRunner.ERC20Addr, SPLAddr: nil, } - if testSolana { + if setupSolana { zrc20Deployment.SPLAddr = deployerRunner.SPLAddr.ToPointer() } @@ -433,6 +434,46 @@ func localE2ETest(cmd *cobra.Command, _ []string) { if testPerformance { eg.Go(ethereumDepositPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherDepositName)) eg.Go(ethereumWithdrawPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherWithdrawName)) + eg.Go( + solanaDepositPerformanceRoutine( + conf, + "perf_sol_deposit", + deployerRunner, + verbose, + conf.AdditionalAccounts.UserSolana, + e2etests.TestStressSolanaDepositName, + ), + ) + eg.Go( + solanaDepositPerformanceRoutine( + conf, + "perf_spl_deposit", + deployerRunner, + verbose, + conf.AdditionalAccounts.UserSPL, + e2etests.TestStressSPLDepositName, + ), + ) + eg.Go( + solanaWithdrawPerformanceRoutine( + conf, + "perf_sol_withdraw", + deployerRunner, + verbose, + conf.AdditionalAccounts.UserSolana, + e2etests.TestStressSolanaWithdrawName, + ), + ) + eg.Go( + solanaWithdrawPerformanceRoutine( + conf, + "perf_spl_withdraw", + deployerRunner, + verbose, + conf.AdditionalAccounts.UserSPL, + e2etests.TestStressSPLWithdrawName, + ), + ) } if testCustom { eg.Go(miscTestRoutine(conf, deployerRunner, verbose, e2etests.TestMyTestName)) diff --git a/cmd/zetae2e/local/monitor_block_production.go b/cmd/zetae2e/local/monitor_block_production.go index 89de03012e..40bfe5e48f 100644 --- a/cmd/zetae2e/local/monitor_block_production.go +++ b/cmd/zetae2e/local/monitor_block_production.go @@ -46,7 +46,7 @@ func monitorBlockProduction(ctx context.Context, conf config.Config) error { return fmt.Errorf("expecting new block event, got %T", event.Data) } latestNewBlockEvent = newBlockEvent - case <-time.After(4 * time.Second): + case <-time.After(5 * time.Second): return fmt.Errorf("timed out waiting for new block (last block %d)", latestNewBlockEvent.Block.Height) case <-ctx.Done(): return nil diff --git a/cmd/zetae2e/local/performance.go b/cmd/zetae2e/local/performance.go index 0d3fea6e5b..94f45b5da3 100644 --- a/cmd/zetae2e/local/performance.go +++ b/cmd/zetae2e/local/performance.go @@ -5,6 +5,7 @@ package local import ( "fmt" + "math/big" "time" "github.com/fatih/color" @@ -12,6 +13,8 @@ import ( "github.com/zeta-chain/node/e2e/config" "github.com/zeta-chain/node/e2e/e2etests" "github.com/zeta-chain/node/e2e/runner" + "github.com/zeta-chain/node/e2e/utils" + "github.com/zeta-chain/node/x/crosschain/types" ) // ethereumDepositPerformanceRoutine runs performance tests for Ether deposit @@ -106,3 +109,125 @@ func ethereumWithdrawPerformanceRoutine( return err } } + +// solanaDepositPerformanceRoutine runs performance tests for solana deposits +func solanaDepositPerformanceRoutine( + conf config.Config, + name string, + deployerRunner *runner.E2ERunner, + verbose bool, + account config.Account, + testNames ...string, +) func() error { + return func() (err error) { + // initialize runner for solana test + r, err := initTestRunner( + "solana", + conf, + deployerRunner, + account, + runner.NewLogger(verbose, color.FgHiMagenta, name), + runner.WithZetaTxServer(deployerRunner.ZetaTxServer), + ) + if err != nil { + return err + } + + if r.ReceiptTimeout == 0 { + r.ReceiptTimeout = 15 * time.Minute + } + if r.CctxTimeout == 0 { + r.CctxTimeout = 15 * time.Minute + } + + r.Logger.Print("🏃 starting solana deposit performance tests") + startTime := time.Now() + + tests, err := r.GetE2ETestsToRunByName( + e2etests.AllE2ETests, + testNames..., + ) + if err != nil { + return fmt.Errorf("solana deposit performance test failed: %v", err) + } + + if err := r.RunE2ETests(tests); err != nil { + return fmt.Errorf("solana deposit performance test failed: %v", err) + } + + r.Logger.Print("🍾 solana deposit performance test completed in %s", time.Since(startTime).String()) + + return err + } +} + +// solanaWithdrawPerformanceRoutine runs performance tests for solana withdrawals +func solanaWithdrawPerformanceRoutine( + conf config.Config, + name string, + deployerRunner *runner.E2ERunner, + verbose bool, + account config.Account, + testNames ...string, +) func() error { + return func() (err error) { + // initialize runner for solana test + r, err := initTestRunner( + "solana", + conf, + deployerRunner, + account, + runner.NewLogger(verbose, color.FgHiGreen, name), + runner.WithZetaTxServer(deployerRunner.ZetaTxServer), + ) + if err != nil { + return err + } + + if r.ReceiptTimeout == 0 { + r.ReceiptTimeout = 15 * time.Minute + } + if r.CctxTimeout == 0 { + r.CctxTimeout = 15 * time.Minute + } + + r.Logger.Print("🏃 starting solana withdraw performance tests") + startTime := time.Now() + + // load deployer private key + privKey := r.GetSolanaPrivKey() + + // execute the deposit sol transaction + amount := big.NewInt(0).Mul(big.NewInt(1e9), big.NewInt(100)) // 100 sol in lamports + sig := r.SOLDepositAndCall(nil, r.EVMAddress(), amount, nil) + + // wait for the cctx to be mined + cctx := utils.WaitCctxMinedByInboundHash(r.Ctx, sig.String(), r.CctxClient, r.Logger, r.CctxTimeout) + r.Logger.CCTX(*cctx, "solana_deposit") + utils.RequireCCTXStatus(r, cctx, types.CctxStatus_OutboundMined) + + // same amount for spl + sig = r.SPLDepositAndCall(&privKey, amount.Uint64(), r.SPLAddr, r.EVMAddress(), nil) + + // wait for the cctx to be mined + cctx = utils.WaitCctxMinedByInboundHash(r.Ctx, sig.String(), r.CctxClient, r.Logger, r.CctxTimeout) + r.Logger.CCTX(*cctx, "solana_deposit_spl") + utils.RequireCCTXStatus(r, cctx, types.CctxStatus_OutboundMined) + + tests, err := r.GetE2ETestsToRunByName( + e2etests.AllE2ETests, + testNames..., + ) + if err != nil { + return fmt.Errorf("solana withdraw performance test failed: %v", err) + } + + if err := r.RunE2ETests(tests); err != nil { + return fmt.Errorf("solana withdraw performance test failed: %v", err) + } + + r.Logger.Print("🍾 solana withdraw performance test completed in %s", time.Since(startTime).String()) + + return err + } +} diff --git a/contrib/localnet/docker-compose.yml b/contrib/localnet/docker-compose.yml index e0078230e7..2cc24fb770 100644 --- a/contrib/localnet/docker-compose.yml +++ b/contrib/localnet/docker-compose.yml @@ -218,6 +218,8 @@ services: profiles: - solana - all + - stress + restart: always ports: - "8899:8899" networks: diff --git a/contrib/localnet/solana/start-solana.sh b/contrib/localnet/solana/start-solana.sh index d87e9672ae..f435cdd472 100644 --- a/contrib/localnet/solana/start-solana.sh +++ b/contrib/localnet/solana/start-solana.sh @@ -9,8 +9,8 @@ solana-test-validator & sleep 5 # airdrop to e2e sol account -solana airdrop 100 -solana airdrop 100 37yGiHAnLvWZUNVwu9esp74YQFqxU1qHCbABkDvRddUQ +solana airdrop 1000 +solana airdrop 1000 37yGiHAnLvWZUNVwu9esp74YQFqxU1qHCbABkDvRddUQ solana program deploy gateway.so diff --git a/e2e/config/config.go b/e2e/config/config.go index 1ae7d4109a..d3eb6d6c05 100644 --- a/e2e/config/config.go +++ b/e2e/config/config.go @@ -68,6 +68,7 @@ type AdditionalAccounts struct { UserBitcoinWithdraw Account `yaml:"user_bitcoin_withdraw"` UserSolana Account `yaml:"user_solana"` UserEther Account `yaml:"user_ether"` + UserSPL Account `yaml:"user_spl"` UserMisc Account `yaml:"user_misc"` UserAdmin Account `yaml:"user_admin"` UserMigration Account `yaml:"user_migration"` // used for TSS migration, TODO: rename (https://github.com/zeta-chain/node/issues/2780) @@ -240,6 +241,7 @@ func (a AdditionalAccounts) AsSlice() []Account { a.UserBitcoinDeposit, a.UserBitcoinWithdraw, a.UserSolana, + a.UserSPL, a.UserEther, a.UserMisc, a.UserAdmin, @@ -330,6 +332,10 @@ func (c *Config) GenerateKeys() error { if err != nil { return err } + c.AdditionalAccounts.UserSPL, err = generateAccount() + if err != nil { + return err + } c.AdditionalAccounts.UserEther, err = generateAccount() if err != nil { return err diff --git a/e2e/e2etests/e2etests.go b/e2e/e2etests/e2etests.go index db99a5ce0d..74b9709bcc 100644 --- a/e2e/e2etests/e2etests.go +++ b/e2e/e2etests/e2etests.go @@ -119,10 +119,14 @@ const ( Stress tests Test stressing networks with many cross-chain transactions */ - TestStressEtherWithdrawName = "stress_eth_withdraw" - TestStressBTCWithdrawName = "stress_btc_withdraw" - TestStressEtherDepositName = "stress_eth_deposit" - TestStressBTCDepositName = "stress_btc_deposit" + TestStressEtherWithdrawName = "stress_eth_withdraw" + TestStressBTCWithdrawName = "stress_btc_withdraw" + TestStressEtherDepositName = "stress_eth_deposit" + TestStressBTCDepositName = "stress_btc_deposit" + TestStressSolanaDepositName = "stress_solana_deposit" + TestStressSPLDepositName = "stress_spl_deposit" + TestStressSolanaWithdrawName = "stress_solana_withdraw" + TestStressSPLWithdrawName = "stress_spl_withdraw" /* Admin tests @@ -784,6 +788,42 @@ var AllE2ETests = []runner.E2ETest{ }, TestStressBTCDeposit, ), + runner.NewE2ETest( + TestStressSolanaDepositName, + "stress test SOL deposit", + []runner.ArgDefinition{ + {Description: "amount in lamports", DefaultValue: "1200000"}, + {Description: "count of SOL deposits", DefaultValue: "50"}, + }, + TestStressSolanaDeposit, + ), + runner.NewE2ETest( + TestStressSPLDepositName, + "stress test SPL deposit", + []runner.ArgDefinition{ + {Description: "amount in SPL tokens", DefaultValue: "1200000"}, + {Description: "count of SPL deposits", DefaultValue: "50"}, + }, + TestStressSPLDeposit, + ), + runner.NewE2ETest( + TestStressSolanaWithdrawName, + "stress test SOL withdrawals", + []runner.ArgDefinition{ + {Description: "amount in lamports", DefaultValue: "1000000"}, + {Description: "count of SOL withdrawals", DefaultValue: "50"}, + }, + TestStressSolanaWithdraw, + ), + runner.NewE2ETest( + TestStressSPLWithdrawName, + "stress test SPL withdrawals", + []runner.ArgDefinition{ + {Description: "amount in SPL tokens", DefaultValue: "1000000"}, + {Description: "count of SPL withdrawals", DefaultValue: "50"}, + }, + TestStressSPLWithdraw, + ), /* Admin tests */ diff --git a/e2e/e2etests/test_stress_solana_deposit.go b/e2e/e2etests/test_stress_solana_deposit.go new file mode 100644 index 0000000000..f56e366d9a --- /dev/null +++ b/e2e/e2etests/test_stress_solana_deposit.go @@ -0,0 +1,60 @@ +package e2etests + +import ( + "fmt" + "time" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" + + "github.com/zeta-chain/node/e2e/runner" + "github.com/zeta-chain/node/e2e/utils" + crosschaintypes "github.com/zeta-chain/node/x/crosschain/types" +) + +// TestStressSolanaDeposit tests the stressing deposit of SOL +func TestStressSolanaDeposit(r *runner.E2ERunner, args []string) { + require.Len(r, args, 2) + + depositSOLAmount := utils.ParseBigInt(r, args[0]) + numDepositsSOL := utils.ParseInt(r, args[1]) + + r.Logger.Print("starting stress test of %d SOL deposits", numDepositsSOL) + + // create a wait group to wait for all the deposits to complete + var eg errgroup.Group + + // send the deposits SOL + for i := 0; i < numDepositsSOL; i++ { + i := i + + // execute the deposit SOL transaction + sig := r.SOLDepositAndCall(nil, r.EVMAddress(), depositSOLAmount, nil) + r.Logger.Print("index %d: starting SOL deposit, sig: %s", i, sig.String()) + + eg.Go(func() error { return monitorDeposit(r, sig, i, time.Now()) }) + } + + require.NoError(r, eg.Wait()) + + r.Logger.Print("all SOL deposits completed") +} + +// monitorDeposit monitors the deposit of SOL/SPL, returns once the deposit is complete +func monitorDeposit(r *runner.E2ERunner, sig solana.Signature, index int, startTime time.Time) error { + cctx := utils.WaitCctxMinedByInboundHash(r.Ctx, sig.String(), r.CctxClient, r.Logger, r.ReceiptTimeout) + if cctx.CctxStatus.Status != crosschaintypes.CctxStatus_OutboundMined { + return fmt.Errorf( + "index %d: deposit cctx failed with status %s, message %s, cctx index %s", + index, + cctx.CctxStatus.Status, + cctx.CctxStatus.StatusMessage, + cctx.Index, + ) + } + timeToComplete := time.Since(startTime) + r.Logger.Print("index %d: deposit cctx success in %s", index, timeToComplete.String()) + + return nil +} diff --git a/e2e/e2etests/test_stress_solana_withdraw.go b/e2e/e2etests/test_stress_solana_withdraw.go new file mode 100644 index 0000000000..75c23ee524 --- /dev/null +++ b/e2e/e2etests/test_stress_solana_withdraw.go @@ -0,0 +1,96 @@ +package e2etests + +import ( + "fmt" + "math/big" + "sync" + "time" + + "github.com/montanaflynn/stats" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" + + "github.com/zeta-chain/node/e2e/runner" + "github.com/zeta-chain/node/e2e/utils" + crosschaintypes "github.com/zeta-chain/node/x/crosschain/types" +) + +// TestStressSolanaWithdraw tests the stressing withdrawal of SOL +func TestStressSolanaWithdraw(r *runner.E2ERunner, args []string) { + require.Len(r, args, 2) + + withdrawSOLAmount := utils.ParseBigInt(r, args[0]) + numWithdrawalsSOL := utils.ParseInt(r, args[1]) + + // load deployer private key + privKey := r.GetSolanaPrivKey() + + r.Logger.Print("starting stress test of %d SOL withdrawals", numWithdrawalsSOL) + + tx, err := r.SOLZRC20.Approve(r.ZEVMAuth, r.SOLZRC20Addr, big.NewInt(1e18)) + require.NoError(r, err) + receipt := utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) + utils.RequireTxSuccessful(r, receipt, "approve_sol") + + // create a wait group to wait for all the withdrawals to complete + var eg errgroup.Group + + // store durations as float64 seconds like prometheus + withdrawDurations := []float64{} + withdrawDurationsLock := sync.Mutex{} + + // send the withdrawals SOL + for i := 0; i < numWithdrawalsSOL; i++ { + i := i + + // execute the withdraw SOL transaction + tx, err = r.SOLZRC20.Withdraw(r.ZEVMAuth, []byte(privKey.PublicKey().String()), withdrawSOLAmount) + require.NoError(r, err) + + receipt := utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) + utils.RequireTxSuccessful(r, receipt) + + r.Logger.Print("index %d: starting SOL withdraw, tx hash: %s", i, tx.Hash().Hex()) + + eg.Go(func() error { + startTime := time.Now() + cctx := utils.WaitCctxMinedByInboundHash(r.Ctx, tx.Hash().Hex(), r.CctxClient, r.Logger, r.ReceiptTimeout) + if cctx.CctxStatus.Status != crosschaintypes.CctxStatus_OutboundMined { + return fmt.Errorf( + "index %d: withdraw cctx failed with status %s, message %s, cctx index %s", + i, + cctx.CctxStatus.Status, + cctx.CctxStatus.StatusMessage, + cctx.Index, + ) + } + timeToComplete := time.Since(startTime) + r.Logger.Print("index %d: withdraw SOL cctx success in %s", i, timeToComplete.String()) + + withdrawDurationsLock.Lock() + withdrawDurations = append(withdrawDurations, timeToComplete.Seconds()) + withdrawDurationsLock.Unlock() + + return nil + }) + } + + err = eg.Wait() + + desc, descErr := stats.Describe(withdrawDurations, false, &[]float64{50.0, 75.0, 90.0, 95.0}) + if descErr != nil { + r.Logger.Print("❌ failed to calculate latency report: %v", descErr) + } + + r.Logger.Print("Latency report:") + r.Logger.Print("min: %.2f", desc.Min) + r.Logger.Print("max: %.2f", desc.Max) + r.Logger.Print("mean: %.2f", desc.Mean) + r.Logger.Print("std: %.2f", desc.Std) + for _, p := range desc.DescriptionPercentiles { + r.Logger.Print("p%.0f: %.2f", p.Percentile, p.Value) + } + + require.NoError(r, err) + r.Logger.Print("all SOL withdrawals completed") +} diff --git a/e2e/e2etests/test_stress_spl_deposit.go b/e2e/e2etests/test_stress_spl_deposit.go new file mode 100644 index 0000000000..6a46241a87 --- /dev/null +++ b/e2e/e2etests/test_stress_spl_deposit.go @@ -0,0 +1,42 @@ +package e2etests + +import ( + "time" + + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" + + "github.com/zeta-chain/node/e2e/runner" + "github.com/zeta-chain/node/e2e/utils" +) + +// TestStressSPLDeposit tests the stressing deposit of SPL +func TestStressSPLDeposit(r *runner.E2ERunner, args []string) { + require.Len(r, args, 2) + + depositSPLAmount := utils.ParseBigInt(r, args[0]) + numDepositsSPL := utils.ParseInt(r, args[1]) + + // load deployer private key + privKey := r.GetSolanaPrivKey() + + r.Logger.Print("starting stress test of %d SPL deposits", numDepositsSPL) + + // create a wait group to wait for all the deposits to complete + var eg errgroup.Group + + // send the deposits SPL + for i := 0; i < numDepositsSPL; i++ { + i := i + + // execute the deposit SPL transaction + sig := r.SPLDepositAndCall(&privKey, depositSPLAmount.Uint64(), r.SPLAddr, r.EVMAddress(), nil) + r.Logger.Print("index %d: starting SPL deposit, sig: %s", i, sig.String()) + + eg.Go(func() error { return monitorDeposit(r, sig, i, time.Now()) }) + } + + require.NoError(r, eg.Wait()) + + r.Logger.Print("all SPL deposits completed") +} diff --git a/e2e/e2etests/test_stress_spl_withdraw.go b/e2e/e2etests/test_stress_spl_withdraw.go new file mode 100644 index 0000000000..296def012b --- /dev/null +++ b/e2e/e2etests/test_stress_spl_withdraw.go @@ -0,0 +1,106 @@ +package e2etests + +import ( + "fmt" + "math/big" + "sync" + "time" + + "github.com/montanaflynn/stats" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" + + "github.com/zeta-chain/node/e2e/runner" + "github.com/zeta-chain/node/e2e/utils" + crosschaintypes "github.com/zeta-chain/node/x/crosschain/types" +) + +// TestStressSPLWithdraw tests the stressing withdrawal of SPL +func TestStressSPLWithdraw(r *runner.E2ERunner, args []string) { + require.Len(r, args, 2) + + withdrawSPLAmount := utils.ParseBigInt(r, args[0]) + numWithdrawalsSPL := utils.ParseInt(r, args[1]) + + // load deployer private key + privKey := r.GetSolanaPrivKey() + + r.Logger.Print("starting stress test of %d SPL withdrawals", numWithdrawalsSPL) + + tx, err := r.SOLZRC20.Approve(r.ZEVMAuth, r.SOLZRC20Addr, big.NewInt(1e18)) + require.NoError(r, err) + receipt := utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) + utils.RequireTxSuccessful(r, receipt, "approve_sol") + + tx, err = r.SPLZRC20.Approve(r.ZEVMAuth, r.SPLZRC20Addr, big.NewInt(1e18)) + require.NoError(r, err) + receipt = utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) + utils.RequireTxSuccessful(r, receipt, "approve_spl") + + tx, err = r.SOLZRC20.Approve(r.ZEVMAuth, r.SPLZRC20Addr, big.NewInt(1e18)) + require.NoError(r, err) + receipt = utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) + utils.RequireTxSuccessful(r, receipt, "approve_spl_sol") + + // create a wait group to wait for all the withdrawals to complete + var eg errgroup.Group + + // store durations as float64 seconds like prometheus + withdrawDurations := []float64{} + withdrawDurationsLock := sync.Mutex{} + + // send the withdrawals SPL + for i := 0; i < numWithdrawalsSPL; i++ { + i := i + + // execute the withdraw SPL transaction + tx, err = r.SPLZRC20.Withdraw(r.ZEVMAuth, []byte(privKey.PublicKey().String()), withdrawSPLAmount) + require.NoError(r, err) + + receipt := utils.MustWaitForTxReceipt(r.Ctx, r.ZEVMClient, tx, r.Logger, r.ReceiptTimeout) + utils.RequireTxSuccessful(r, receipt) + + r.Logger.Print("index %d: starting SPL withdraw, tx hash: %s", i, tx.Hash().Hex()) + + eg.Go(func() error { + startTime := time.Now() + cctx := utils.WaitCctxMinedByInboundHash(r.Ctx, tx.Hash().Hex(), r.CctxClient, r.Logger, r.ReceiptTimeout) + if cctx.CctxStatus.Status != crosschaintypes.CctxStatus_OutboundMined { + return fmt.Errorf( + "index %d: withdraw cctx failed with status %s, message %s, cctx index %s", + i, + cctx.CctxStatus.Status, + cctx.CctxStatus.StatusMessage, + cctx.Index, + ) + } + timeToComplete := time.Since(startTime) + r.Logger.Print("index %d: withdraw SPL cctx success in %s", i, timeToComplete.String()) + + withdrawDurationsLock.Lock() + withdrawDurations = append(withdrawDurations, timeToComplete.Seconds()) + withdrawDurationsLock.Unlock() + + return nil + }) + } + + err = eg.Wait() + + desc, descErr := stats.Describe(withdrawDurations, false, &[]float64{50.0, 75.0, 90.0, 95.0}) + if descErr != nil { + r.Logger.Print("❌ failed to calculate latency report: %v", descErr) + } + + r.Logger.Print("Latency report:") + r.Logger.Print("min: %.2f", desc.Min) + r.Logger.Print("max: %.2f", desc.Max) + r.Logger.Print("mean: %.2f", desc.Mean) + r.Logger.Print("std: %.2f", desc.Std) + for _, p := range desc.DescriptionPercentiles { + r.Logger.Print("p%.0f: %.2f", p.Percentile, p.Value) + } + + require.NoError(r, err) + r.Logger.Print("all SPL withdrawals completed") +} diff --git a/e2e/runner/solana.go b/e2e/runner/solana.go index cc9e6dbe2b..8ee9c7ab56 100644 --- a/e2e/runner/solana.go +++ b/e2e/runner/solana.go @@ -274,7 +274,7 @@ func (r *E2ERunner) DeploySPL(privateKey *solana.PrivateKey, whitelist bool) *so // minting some tokens to deployer for testing ata := r.ResolveSolanaATA(*privateKey, privateKey.PublicKey(), mintAccount.PublicKey()) - mintToInstruction := token.NewMintToInstruction(uint64(1_000_000_000), mintAccount.PublicKey(), ata, privateKey.PublicKey(), []solana.PublicKey{}). + mintToInstruction := token.NewMintToInstruction(uint64(100_000_000_000_000), mintAccount.PublicKey(), ata, privateKey.PublicKey(), []solana.PublicKey{}). Build() signedTx = r.CreateSignedTransaction( []solana.Instruction{mintToInstruction}, diff --git a/pkg/contracts/solana/account_info.go b/pkg/contracts/solana/account_info.go new file mode 100644 index 0000000000..31bec2356e --- /dev/null +++ b/pkg/contracts/solana/account_info.go @@ -0,0 +1,30 @@ +package solana + +import ( + bin "github.com/gagliardetto/binary" + "github.com/gagliardetto/solana-go/programs/token" + solrpc "github.com/gagliardetto/solana-go/rpc" + "github.com/near/borsh-go" +) + +func DeserializePdaInfo(pdaInfo *solrpc.GetAccountInfoResult) (PdaInfo, error) { + pda := PdaInfo{} + err := borsh.Deserialize(&pda, pdaInfo.Bytes()) + if err != nil { + return PdaInfo{}, err + } + + return pda, nil +} + +func DeserializeMintAccountInfo(mintInfo *solrpc.GetAccountInfoResult) (token.Mint, error) { + var mint token.Mint + // Account{}.Data.GetBinary() returns the *decoded* binary data + // regardless the original encoding (it can handle them all). + err := bin.NewBinDecoder(mintInfo.Value.Data.GetBinary()).Decode(&mint) + if err != nil { + return token.Mint{}, err + } + + return mint, err +} diff --git a/zetaclient/chains/solana/signer/outbound_tracker_reporter.go b/zetaclient/chains/solana/signer/outbound_tracker_reporter.go index 3fc8acf181..56ce65dc32 100644 --- a/zetaclient/chains/solana/signer/outbound_tracker_reporter.go +++ b/zetaclient/chains/solana/signer/outbound_tracker_reporter.go @@ -13,12 +13,6 @@ import ( "github.com/zeta-chain/node/zetaclient/logs" ) -const ( - // SolanaTransactionTimeout is the timeout for waiting for an outbound to be confirmed - // Transaction referencing a blockhash older than 150 blocks will expire and be rejected by Solana. - SolanaTransactionTimeout = 2 * time.Minute -) - // reportToOutboundTracker launch a go routine with timeout to check for tx confirmation; // it reports tx to outbound tracker only if it's confirmed by the Solana network. func (signer *Signer) reportToOutboundTracker( @@ -56,7 +50,7 @@ func (signer *Signer) reportToOutboundTracker( time.Sleep(5 * time.Second) // give up if we know the tx is too old and already expired - if time.Since(start) > SolanaTransactionTimeout { + if time.Since(start) > solanaTransactionTimeout { logger.Info().Msg("outbound is expired") return nil } diff --git a/zetaclient/chains/solana/signer/signer.go b/zetaclient/chains/solana/signer/signer.go index 1e3d6914f3..84bf1845d6 100644 --- a/zetaclient/chains/solana/signer/signer.go +++ b/zetaclient/chains/solana/signer/signer.go @@ -4,10 +4,10 @@ import ( "context" "fmt" "strings" + "time" "cosmossdk.io/errors" ethcommon "github.com/ethereum/go-ethereum/common" - bin "github.com/gagliardetto/binary" "github.com/gagliardetto/solana-go" "github.com/gagliardetto/solana-go/programs/token" "github.com/gagliardetto/solana-go/rpc" @@ -22,10 +22,24 @@ import ( "github.com/zeta-chain/node/zetaclient/chains/interfaces" "github.com/zeta-chain/node/zetaclient/compliance" "github.com/zeta-chain/node/zetaclient/keys" + "github.com/zeta-chain/node/zetaclient/logs" "github.com/zeta-chain/node/zetaclient/metrics" "github.com/zeta-chain/node/zetaclient/outboundprocessor" ) +const ( + // solanaTransactionTimeout is the timeout for waiting for an outbound to be confirmed. + // Transaction referencing a blockhash older than 150 blocks (60 ~90 secs) will expire and be rejected by Solana. + solanaTransactionTimeout = 2 * time.Minute + + // broadcastBackoff is the initial backoff duration for retrying broadcast + broadcastBackoff = 1 * time.Second + + // broadcastRetries is the maximum number of retries for broadcasting a transaction + // 6 retries will span over 1 + 2 + 4 + 8 + 16 + 32 + 64 = 127 seconds, good enough for the 2 minute timeout + broadcastRetries = 7 +) + var _ interfaces.ChainSigner = (*Signer)(nil) // Signer deals with signing Solana transactions and implements the ChainSigner interface @@ -171,26 +185,68 @@ func (signer *Signer) TryProcessOutbound( // set relayer balance metrics signer.SetRelayerBalanceMetrics(ctx) - // broadcast the signed tx to the Solana network with preflight check - txSig, err := signer.client.SendTransactionWithOpts( - ctx, - tx, - // Commitment "finalized" is too conservative for preflight check and - // it results in repeated broadcast attempts that only 1 will succeed. - // Commitment "processed" will simulate tx against more recent state - // thus fails faster once a tx is already broadcasted and processed by the cluster. - // This reduces the number of "failed" txs due to repeated broadcast attempts. - rpc.TransactionOpts{PreflightCommitment: rpc.CommitmentProcessed}, - ) - if err != nil { - logger.Error(). - Err(err). - Msgf("TryProcessOutbound: broadcast error") - return + // broadcast the signed tx to the Solana network + signer.broadcastOutbound(ctx, tx, chainID, nonce, logger, zetacoreClient) +} + +// broadcastOutbound sends the signed transaction to the Solana network +func (signer *Signer) broadcastOutbound( + ctx context.Context, + tx *solana.Transaction, + chainID int64, + nonce uint64, + logger zerolog.Logger, + zetacoreClient interfaces.ZetacoreClient, +) { + // prepare logger fields + lf := map[string]any{ + logs.FieldMethod: "broadcastOutbound", + logs.FieldNonce: nonce, + logs.FieldTx: tx.Signatures[0].String(), } - // report the outbound to the outbound tracker - signer.reportToOutboundTracker(ctx, zetacoreClient, chainID, nonce, txSig, logger) + // try broacasting tx with increasing backoff (1s, 2s, 4s, 8s, 16s, 32s, 64s) + // to tolerate tx nonce mismatch with PDA nonce or unknown RPC error + backOff := broadcastBackoff + for i := 0; i < broadcastRetries; i++ { + time.Sleep(backOff) + + // PDA nonce may already be increased by other relayer, no need to retry + pdaInfo, err := signer.client.GetAccountInfo(ctx, signer.pda) + if err != nil { + logger.Error().Err(err).Fields(lf).Msgf("unable to get PDA account info") + } else { + pda, err := contracts.DeserializePdaInfo(pdaInfo) + if err != nil { + logger.Error().Err(err).Fields(lf).Msgf("unable to deserialize PDA info") + } else if pda.Nonce > nonce { + logger.Info().Err(err).Fields(lf).Msgf("PDA nonce %d is greater than outbound nonce, stop retrying", pda.Nonce) + break + } + } + + // broadcast the signed tx to the Solana network with preflight check + txSig, err := signer.client.SendTransactionWithOpts( + ctx, + tx, + // Commitment "finalized" is too conservative for preflight check and + // it results in repeated broadcast attempts that only 1 will succeed. + // Commitment "processed" will simulate tx against more recent state + // thus fails faster once a tx is already broadcasted and processed by the cluster. + // This reduces the number of "failed" txs due to repeated broadcast attempts. + rpc.TransactionOpts{PreflightCommitment: rpc.CommitmentProcessed}, + ) + if err != nil { + logger.Warn().Err(err).Fields(lf).Msgf("SendTransactionWithOpts failed") + backOff *= 2 + continue + } + logger.Info().Fields(lf).Msgf("broadcasted Solana outbound successfully") + + // successful broadcast; report to the outbound tracker + signer.reportToOutboundTracker(ctx, zetacoreClient, chainID, nonce, txSig, logger) + break + } } func (signer *Signer) prepareWithdrawTx( @@ -323,15 +379,7 @@ func (signer *Signer) decodeMintAccountDetails(ctx context.Context, asset string return token.Mint{}, err } - var mint token.Mint - // Account{}.Data.GetBinary() returns the *decoded* binary data - // regardless the original encoding (it can handle them all). - err = bin.NewBinDecoder(info.Value.Data.GetBinary()).Decode(&mint) - if err != nil { - return token.Mint{}, err - } - - return mint, nil + return contracts.DeserializeMintAccountInfo(info) } // SetGatewayAddress sets the gateway address diff --git a/zetaclient/orchestrator/orchestrator.go b/zetaclient/orchestrator/orchestrator.go index c29b41466a..7471198cd2 100644 --- a/zetaclient/orchestrator/orchestrator.go +++ b/zetaclient/orchestrator/orchestrator.go @@ -34,11 +34,11 @@ import ( ) const ( - // evmOutboundLookbackFactor is the factor to determine how many nonces to look back for pending cctxs + // outboundLookbackFactor is the factor to determine how many nonces to look back for pending cctxs // For example, give OutboundScheduleLookahead of 120, pending NonceLow of 1000 and factor of 1.0, // the scheduler need to be able to pick up and schedule any pending cctx with nonce < 880 (1000 - 120 * 1.0) // NOTE: 1.0 means look back the same number of cctxs as we look ahead - evmOutboundLookbackFactor = 1.0 + outboundLookbackFactor = 1.0 // sampling rate for sampled orchestrator logger loggerSamplingRate = 10 @@ -429,7 +429,7 @@ func (oc *Orchestrator) ScheduleCctxEVM( } outboundScheduleLookahead := observer.ChainParams().OutboundScheduleLookahead // #nosec G115 always in range - outboundScheduleLookback := uint64(float64(outboundScheduleLookahead) * evmOutboundLookbackFactor) + outboundScheduleLookback := uint64(float64(outboundScheduleLookahead) * outboundLookbackFactor) // #nosec G115 positive outboundScheduleInterval := uint64(observer.ChainParams().OutboundScheduleInterval) criticalInterval := uint64(10) // for critical pending outbound we reduce re-try interval @@ -597,8 +597,12 @@ func (oc *Orchestrator) ScheduleCctxSolana( oc.logger.Error().Msgf("ScheduleCctxSolana: chain observer is not a solana observer") return } + + // outbound keysign scheduler parameters // #nosec G115 positive interval := uint64(observer.ChainParams().OutboundScheduleInterval) + outboundScheduleLookahead := observer.ChainParams().OutboundScheduleLookahead + outboundScheduleLookback := uint64(float64(outboundScheduleLookahead) * outboundLookbackFactor) // schedule keysign for each pending cctx for _, cctx := range cctxList { @@ -611,6 +615,11 @@ func (oc *Orchestrator) ScheduleCctxSolana( Msgf("ScheduleCctxSolana: outbound %s chainid mismatch: want %d, got %d", outboundID, chainID, params.ReceiverChainId) continue } + if params.TssNonce > cctxList[0].GetCurrentOutboundParam().TssNonce+outboundScheduleLookback { + oc.logger.Warn().Msgf("ScheduleCctxSolana: nonce too high: signing %d, earliest pending %d", + params.TssNonce, cctxList[0].GetCurrentOutboundParam().TssNonce) + break + } // vote outbound if it's already confirmed continueKeysign, err := solObserver.VoteOutboundIfConfirmed(ctx, cctx)