test: add Solana E2E performance tests and improve Solana outbounds p…

…erformance (#3276)

* stress tests deposit solana

* perf tests for sol withdrawals

* split withdrawal stress tests into separate tests

* separate deposit stress tests

* add restart always to solana container

* tweak params

* fmt and changelog

* cleanup

* making tests execution more granular

* revert last change

* split perf tests

* tweak params

* decrease num of withdrawals back to 30

* run only withdrawals

* run sequentiallyx

* uncomment tests

* params

* fix: speed up Solana withdraw stress test in two go routines (#3295)

* PR comment and lint

* changelog

* move account info decoding to solana package

* revert timeouts to prev values

---------

Co-authored-by: Charlie Chen <[email protected]>

Makefile

@@ -275,7 +275,7 @@ start-e2e-admin-test: e2e-images
 	export E2E_ARGS="--skip-regular --test-admin" && \
 	cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile eth2 up -d
 
-start-e2e-performance-test: e2e-images
+start-e2e-performance-test: e2e-images solana
 	@echo "--> Starting e2e performance test"
 	export E2E_ARGS="--test-performance" && \
 	cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile stress up -d

cmd/zetae2e/config/local.yml

@@ -29,6 +29,11 @@ additional_accounts:
     evm_address: "0x103FD9224F00ce3013e95629e52DFc31D805D68d"
     private_key: "dd53f191113d18e57bd4a5494a64a020ba7919c815d0a6d34a42ebb2839e9d95"
     solana_private_key: "4yqSQxDeTBvn86BuxcN5jmZb2gaobFXrBqu8kiE9rZxNkVMe3LfXmFigRsU4sRp7vk4vVP1ZCFiejDKiXBNWvs2C"
+  user_spl:
+    bech32_address: "zeta17e77anpmzhuuam67hg6x3mtqrulqh80z9chv70"
+    evm_address: "0xf67deecc3B15F9CEeF5eba3468ed601f3e0B9de2"
+    private_key: "2b3306a8ac43dbf0e350b87876c131e7e12bd49563a16de9ce8aeb664b94d559"
+    solana_private_key: "4yqSQxDeTBvn86BuxcN5jmZb2gaobFXrBqu8kiE9rZxNkVMe3LfXmFigRsU4sRp7vk4vVP1ZCFiejDKiXBNWvs2C"
   user_ether:
     bech32_address: "zeta134rakuus43xn63yucgxhn88ywj8ewcv6ezn2ga"
     evm_address: "0x8D47Db7390AC4D3D449Cc20D799ce4748F97619A"

cmd/zetae2e/config/localnet.yml

@@ -29,6 +29,11 @@ additional_accounts:
     evm_address: "0x103FD9224F00ce3013e95629e52DFc31D805D68d"
     private_key: "dd53f191113d18e57bd4a5494a64a020ba7919c815d0a6d34a42ebb2839e9d95"
     solana_private_key: "4yqSQxDeTBvn86BuxcN5jmZb2gaobFXrBqu8kiE9rZxNkVMe3LfXmFigRsU4sRp7vk4vVP1ZCFiejDKiXBNWvs2C"
+  user_spl:
+    bech32_address: "zeta17e77anpmzhuuam67hg6x3mtqrulqh80z9chv70"
+    evm_address: "0xf67deecc3B15F9CEeF5eba3468ed601f3e0B9de2"
+    private_key: "2b3306a8ac43dbf0e350b87876c131e7e12bd49563a16de9ce8aeb664b94d559"
+    solana_private_key: "4yqSQxDeTBvn86BuxcN5jmZb2gaobFXrBqu8kiE9rZxNkVMe3LfXmFigRsU4sRp7vk4vVP1ZCFiejDKiXBNWvs2C"
   user_ether:
     bech32_address: "zeta134rakuus43xn63yucgxhn88ywj8ewcv6ezn2ga"
     evm_address: "0x8D47Db7390AC4D3D449Cc20D799ce4748F97619A"

cmd/zetae2e/local/local.go

@@ -116,6 +116,7 @@ func localE2ETest(cmd *cobra.Command, _ []string) {
 		testV2Migration   = must(cmd.Flags().GetBool(flagTestV2Migration))
 		skipPrecompiles   = must(cmd.Flags().GetBool(flagSkipPrecompiles))
 		upgradeContracts  = must(cmd.Flags().GetBool(flagUpgradeContracts))
+		setupSolana       = testSolana || testPerformance
 	)
 
 	logger := runner.NewLogger(verbose, color.FgWhite, "setup")
@@ -230,7 +231,7 @@ func localE2ETest(cmd *cobra.Command, _ []string) {
 
 		deployerRunner.SetupEVMV2()
 
-		if testSolana {
+		if setupSolana {
 			deployerRunner.SetupSolana(
 				conf.Contracts.Solana.GatewayProgramID.String(),
 				conf.AdditionalAccounts.UserSolana.SolanaPrivateKey.String(),
@@ -246,7 +247,7 @@ func localE2ETest(cmd *cobra.Command, _ []string) {
 			ERC20Addr: deployerRunner.ERC20Addr,
 			SPLAddr:   nil,
 		}
-		if testSolana {
+		if setupSolana {
 			zrc20Deployment.SPLAddr = deployerRunner.SPLAddr.ToPointer()
 		}
 
@@ -433,6 +434,46 @@ func localE2ETest(cmd *cobra.Command, _ []string) {
 	if testPerformance {
 		eg.Go(ethereumDepositPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherDepositName))
 		eg.Go(ethereumWithdrawPerformanceRoutine(conf, deployerRunner, verbose, e2etests.TestStressEtherWithdrawName))
+		eg.Go(
+			solanaDepositPerformanceRoutine(
+				conf,
+				"perf_sol_deposit",
+				deployerRunner,
+				verbose,
+				conf.AdditionalAccounts.UserSolana,
+				e2etests.TestStressSolanaDepositName,
+			),
+		)
+		eg.Go(
+			solanaDepositPerformanceRoutine(
+				conf,
+				"perf_spl_deposit",
+				deployerRunner,
+				verbose,
+				conf.AdditionalAccounts.UserSPL,
+				e2etests.TestStressSPLDepositName,
+			),
+		)
+		eg.Go(
+			solanaWithdrawPerformanceRoutine(
+				conf,
+				"perf_sol_withdraw",
+				deployerRunner,
+				verbose,
+				conf.AdditionalAccounts.UserSolana,
+				e2etests.TestStressSolanaWithdrawName,
+			),
+		)
+		eg.Go(
+			solanaWithdrawPerformanceRoutine(
+				conf,
+				"perf_spl_withdraw",
+				deployerRunner,
+				verbose,
+				conf.AdditionalAccounts.UserSPL,
+				e2etests.TestStressSPLWithdrawName,
+			),
+		)
 	}
 	if testCustom {
 		eg.Go(miscTestRoutine(conf, deployerRunner, verbose, e2etests.TestMyTestName))

cmd/zetae2e/local/monitor_block_production.go

@@ -46,7 +46,7 @@ func monitorBlockProduction(ctx context.Context, conf config.Config) error {
 				return fmt.Errorf("expecting new block event, got %T", event.Data)
 			}
 			latestNewBlockEvent = newBlockEvent
-		case <-time.After(4 * time.Second):
+		case <-time.After(5 * time.Second):
 			return fmt.Errorf("timed out waiting for new block (last block %d)", latestNewBlockEvent.Block.Height)
 		case <-ctx.Done():
 			return nil

cmd/zetae2e/local/performance.go

@@ -5,13 +5,16 @@ package local
 
 import (
 	"fmt"
+	"math/big"
 	"time"
 
 	"github.com/fatih/color"
 
 	"github.com/zeta-chain/node/e2e/config"
 	"github.com/zeta-chain/node/e2e/e2etests"
 	"github.com/zeta-chain/node/e2e/runner"
+	"github.com/zeta-chain/node/e2e/utils"
+	"github.com/zeta-chain/node/x/crosschain/types"
 )
 
 // ethereumDepositPerformanceRoutine runs performance tests for Ether deposit
@@ -106,3 +109,125 @@ func ethereumWithdrawPerformanceRoutine(
 		return err
 	}
 }
+
+// solanaDepositPerformanceRoutine runs performance tests for solana deposits
+func solanaDepositPerformanceRoutine(
+	conf config.Config,
+	name string,
+	deployerRunner *runner.E2ERunner,
+	verbose bool,
+	account config.Account,
+	testNames ...string,
+) func() error {
+	return func() (err error) {
+		// initialize runner for solana test
+		r, err := initTestRunner(
+			"solana",
+			conf,
+			deployerRunner,
+			account,
+			runner.NewLogger(verbose, color.FgHiMagenta, name),
+			runner.WithZetaTxServer(deployerRunner.ZetaTxServer),
+		)
+		if err != nil {
+			return err
+		}
+
+		if r.ReceiptTimeout == 0 {
+			r.ReceiptTimeout = 15 * time.Minute
+		}
+		if r.CctxTimeout == 0 {
+			r.CctxTimeout = 15 * time.Minute
+		}
+
+		r.Logger.Print("🏃 starting solana deposit performance tests")
+		startTime := time.Now()
+
+		tests, err := r.GetE2ETestsToRunByName(
+			e2etests.AllE2ETests,
+			testNames...,
+		)
+		if err != nil {
+			return fmt.Errorf("solana deposit performance test failed: %v", err)
+		}
+
+		if err := r.RunE2ETests(tests); err != nil {
+			return fmt.Errorf("solana deposit performance test failed: %v", err)
+		}
+
+		r.Logger.Print("🍾 solana deposit performance test completed in %s", time.Since(startTime).String())
+
+		return err
+	}
+}
+
+// solanaWithdrawPerformanceRoutine runs performance tests for solana withdrawals
+func solanaWithdrawPerformanceRoutine(
+	conf config.Config,
+	name string,
+	deployerRunner *runner.E2ERunner,
+	verbose bool,
+	account config.Account,
+	testNames ...string,
+) func() error {
+	return func() (err error) {
+		// initialize runner for solana test
+		r, err := initTestRunner(
+			"solana",
+			conf,
+			deployerRunner,
+			account,
+			runner.NewLogger(verbose, color.FgHiGreen, name),
+			runner.WithZetaTxServer(deployerRunner.ZetaTxServer),
+		)
+		if err != nil {
+			return err
+		}
+
+		if r.ReceiptTimeout == 0 {
+			r.ReceiptTimeout = 15 * time.Minute
+		}
+		if r.CctxTimeout == 0 {
+			r.CctxTimeout = 15 * time.Minute
+		}
+
+		r.Logger.Print("🏃 starting solana withdraw performance tests")
+		startTime := time.Now()
+
+		// load deployer private key
+		privKey := r.GetSolanaPrivKey()
+
+		// execute the deposit sol transaction
+		amount := big.NewInt(0).Mul(big.NewInt(1e9), big.NewInt(100)) // 100 sol in lamports
+		sig := r.SOLDepositAndCall(nil, r.EVMAddress(), amount, nil)
+
+		// wait for the cctx to be mined
+		cctx := utils.WaitCctxMinedByInboundHash(r.Ctx, sig.String(), r.CctxClient, r.Logger, r.CctxTimeout)
+		r.Logger.CCTX(*cctx, "solana_deposit")
+		utils.RequireCCTXStatus(r, cctx, types.CctxStatus_OutboundMined)
+
+		// same amount for spl
+		sig = r.SPLDepositAndCall(&privKey, amount.Uint64(), r.SPLAddr, r.EVMAddress(), nil)
+
+		// wait for the cctx to be mined
+		cctx = utils.WaitCctxMinedByInboundHash(r.Ctx, sig.String(), r.CctxClient, r.Logger, r.CctxTimeout)
+		r.Logger.CCTX(*cctx, "solana_deposit_spl")
+		utils.RequireCCTXStatus(r, cctx, types.CctxStatus_OutboundMined)
+
+		tests, err := r.GetE2ETestsToRunByName(
+			e2etests.AllE2ETests,
+			testNames...,
+		)
+		if err != nil {
+			return fmt.Errorf("solana withdraw performance test failed: %v", err)
+		}
+
+		if err := r.RunE2ETests(tests); err != nil {
+			return fmt.Errorf("solana withdraw performance test failed: %v", err)
+		}
+
+		r.Logger.Print("🍾 solana withdraw performance test completed in %s", time.Since(startTime).String())
+
+		return err
+	}
+}

contrib/localnet/docker-compose.yml

@@ -218,6 +218,8 @@ services:
     profiles:
       - solana
       - all
+      - stress
+    restart: always
     ports:
       - "8899:8899"
     networks:

contrib/localnet/solana/start-solana.sh

@@ -9,8 +9,8 @@ solana-test-validator &
 
 sleep 5
 # airdrop to e2e sol account
-solana airdrop 100
-solana airdrop 100 37yGiHAnLvWZUNVwu9esp74YQFqxU1qHCbABkDvRddUQ
+solana airdrop 1000
+solana airdrop 1000 37yGiHAnLvWZUNVwu9esp74YQFqxU1qHCbABkDvRddUQ
 solana program deploy gateway.so
 
 

e2e/config/config.go

@@ -68,6 +68,7 @@ type AdditionalAccounts struct {
 	UserBitcoinWithdraw   Account `yaml:"user_bitcoin_withdraw"`
 	UserSolana            Account `yaml:"user_solana"`
 	UserEther             Account `yaml:"user_ether"`
+	UserSPL               Account `yaml:"user_spl"`
 	UserMisc              Account `yaml:"user_misc"`
 	UserAdmin             Account `yaml:"user_admin"`
 	UserMigration         Account `yaml:"user_migration"` // used for TSS migration, TODO: rename (https://github.com/zeta-chain/node/issues/2780)
@@ -240,6 +241,7 @@ func (a AdditionalAccounts) AsSlice() []Account {
 		a.UserBitcoinDeposit,
 		a.UserBitcoinWithdraw,
 		a.UserSolana,
+		a.UserSPL,
 		a.UserEther,
 		a.UserMisc,
 		a.UserAdmin,
@@ -330,6 +332,10 @@ func (c *Config) GenerateKeys() error {
 	if err != nil {
 		return err
 	}
+	c.AdditionalAccounts.UserSPL, err = generateAccount()
+	if err != nil {
+		return err
+	}
 	c.AdditionalAccounts.UserEther, err = generateAccount()
 	if err != nil {
 		return err

e2e/e2etests/e2etests.go

@@ -119,10 +119,14 @@ const (
 	 Stress tests
 	 Test stressing networks with many cross-chain transactions
 	*/
-	TestStressEtherWithdrawName = "stress_eth_withdraw"
-	TestStressBTCWithdrawName   = "stress_btc_withdraw"
-	TestStressEtherDepositName  = "stress_eth_deposit"
-	TestStressBTCDepositName    = "stress_btc_deposit"
+	TestStressEtherWithdrawName  = "stress_eth_withdraw"
+	TestStressBTCWithdrawName    = "stress_btc_withdraw"
+	TestStressEtherDepositName   = "stress_eth_deposit"
+	TestStressBTCDepositName     = "stress_btc_deposit"
+	TestStressSolanaDepositName  = "stress_solana_deposit"
+	TestStressSPLDepositName     = "stress_spl_deposit"
+	TestStressSolanaWithdrawName = "stress_solana_withdraw"
+	TestStressSPLWithdrawName    = "stress_spl_withdraw"
 
 	/*
 	 Admin tests
@@ -784,6 +788,42 @@ var AllE2ETests = []runner.E2ETest{
 		},
 		TestStressBTCDeposit,
 	),
+	runner.NewE2ETest(
+		TestStressSolanaDepositName,
+		"stress test SOL deposit",
+		[]runner.ArgDefinition{
+			{Description: "amount in lamports", DefaultValue: "1200000"},
+			{Description: "count of SOL deposits", DefaultValue: "50"},
+		},
+		TestStressSolanaDeposit,
+	),
+	runner.NewE2ETest(
+		TestStressSPLDepositName,
+		"stress test SPL deposit",
+		[]runner.ArgDefinition{
+			{Description: "amount in SPL tokens", DefaultValue: "1200000"},
+			{Description: "count of SPL deposits", DefaultValue: "50"},
+		},
+		TestStressSPLDeposit,
+	),
+	runner.NewE2ETest(
+		TestStressSolanaWithdrawName,
+		"stress test SOL withdrawals",
+		[]runner.ArgDefinition{
+			{Description: "amount in lamports", DefaultValue: "1000000"},
+			{Description: "count of SOL withdrawals", DefaultValue: "50"},
+		},
+		TestStressSolanaWithdraw,
+	),
+	runner.NewE2ETest(
+		TestStressSPLWithdrawName,
+		"stress test SPL withdrawals",
+		[]runner.ArgDefinition{
+			{Description: "amount in SPL tokens", DefaultValue: "1000000"},
+			{Description: "count of SPL withdrawals", DefaultValue: "50"},
+		},
+		TestStressSPLWithdraw,
+	),
 	/*
 	 Admin tests
 	*/