From 45c18d0cdb4dbe1529bc5fa285fce12825b3a728 Mon Sep 17 00:00:00 2001 From: Joel Smith Date: Mon, 3 Jun 2024 15:49:57 -0500 Subject: [PATCH 01/12] feat: cosmos redundant rpcs --- cmd/appstate.go | 12 +++ cmd/chains.go | 35 +++++++- relayer/chain.go | 11 +-- relayer/chains/cosmos/provider.go | 121 ++++++++++++++++++++++++++-- relayer/chains/penumbra/provider.go | 7 ++ relayer/provider/provider.go | 1 + 6 files changed, 170 insertions(+), 17 deletions(-) diff --git a/cmd/appstate.go b/cmd/appstate.go index 44aff4736..9bb65fd05 100644 --- a/cmd/appstate.go +++ b/cmd/appstate.go @@ -319,3 +319,15 @@ func (a *appState) useRpcAddr(chainName string, rpcAddr string) error { return nil }) } + +func (a *appState) useBackupRpcAddrs(chainName string, rpcAddrs []string) error { + _, exists := a.config.Chains[chainName] + if !exists { + return fmt.Errorf("chain %s not found in config", chainName) + } + + return a.performConfigLockingOperation(context.Background(), func() error { + a.config.Chains[chainName].ChainProvider.SetBackupRpcAddrs(rpcAddrs) + return nil + }) +} diff --git a/cmd/chains.go b/cmd/chains.go index d39396412..634297f36 100644 --- a/cmd/chains.go +++ b/cmd/chains.go @@ -41,6 +41,7 @@ func chainsCmd(a *appState) *cobra.Command { chainsAddDirCmd(a), cmdChainsConfigure(a), cmdChainsUseRpcAddr(a), + cmdChainsUseBackupRpcAddr(a), ) return cmd @@ -48,9 +49,9 @@ func chainsCmd(a *appState) *cobra.Command { func cmdChainsUseRpcAddr(a *appState) *cobra.Command { cmd := &cobra.Command{ - Use: "set-rpc-addr chain_name valid_rpc_url", + Use: "set-rpc-addr chain_name valid_rpc_url", Aliases: []string{"rpc"}, - Short: "Sets chain's rpc address", + Short: "Sets chain's rpc address", Args: withUsage(cobra.ExactArgs(2)), Example: strings.TrimSpace(fmt.Sprintf(` $ %s chains set-rpc-addr ibc-0 https://abc.xyz.com:443 @@ -69,6 +70,36 @@ $ %s ch set-rpc-addr ibc-0 https://abc.xyz.com:443`, appName, appName)), return cmd } +func cmdChainsUseBackupRpcAddr(a *appState) *cobra.Command { + cmd := &cobra.Command{ + Use: "set-backup-rpc-addr chain_name comma_separated_valid_rpc_urls", + Aliases: []string{"rpc"}, + Short: "Sets chain's backup rpc addresses", + Args: withUsage(cobra.ExactArgs(2)), + Example: strings.TrimSpace(fmt.Sprintf(` +$ %s chains set-backup-rpc-addr ibc-0 https://abc.xyz.com:443,https://123.456.com:443 +$ %s ch set-backup-rpc-addr ibc-0 https://abc.xyz.com:443,https://123.456.com:443`, appName, appName)), + RunE: func(cmd *cobra.Command, args []string) error { + chainName := args[0] + rpc_addresses := args[1] + + // Split rpc_addresses by ',' + rpc_addresses_list := strings.Split(rpc_addresses, ",") + + // Loop through and ensure valid + for _, rpc_address := range rpc_addresses_list { + if !isValidURL(rpc_address) { + return invalidRpcAddr(rpc_address) + } + } + + return a.useBackupRpcAddrs(chainName, rpc_addresses_list) + }, + } + + return cmd +} + func chainsAddrCmd(a *appState) *cobra.Command { cmd := &cobra.Command{ Use: "address chain_name", diff --git a/relayer/chain.go b/relayer/chain.go index dfcf728cb..90d0f1bce 100644 --- a/relayer/chain.go +++ b/relayer/chain.go @@ -4,10 +4,10 @@ import ( "context" "encoding/json" "fmt" - "github.com/avast/retry-go/v4" - "net/url" "time" + "github.com/avast/retry-go/v4" + "github.com/cosmos/cosmos-sdk/crypto/hd" clienttypes "github.com/cosmos/ibc-go/v8/modules/core/02-client/types" "github.com/cosmos/relayer/v2/relayer/provider" @@ -31,7 +31,6 @@ type Chain struct { ChainProvider provider.ChainProvider Chainid string `yaml:"chain-id" json:"chain-id"` - RPCAddr string `yaml:"rpc-addr" json:"rpc-addr"` PathEnd *PathEnd `yaml:"-" json:"-"` @@ -137,12 +136,6 @@ func (c Chains) Gets(chainIDs ...string) (map[string]*Chain, error) { return out, nil } -// GetRPCPort returns the port configured for the chain -func (c *Chain) GetRPCPort() string { - u, _ := url.Parse(c.RPCAddr) - return u.Port() -} - // CreateTestKey creates a key for test chain func (c *Chain) CreateTestKey() error { if c.ChainProvider.KeyExists(c.ChainProvider.Key()) { diff --git a/relayer/chains/cosmos/provider.go b/relayer/chains/cosmos/provider.go index f34b1125d..c212696f7 100644 --- a/relayer/chains/cosmos/provider.go +++ b/relayer/chains/cosmos/provider.go @@ -38,6 +38,7 @@ type CosmosProviderConfig struct { ChainName string `json:"-" yaml:"-"` ChainID string `json:"chain-id" yaml:"chain-id"` RPCAddr string `json:"rpc-addr" yaml:"rpc-addr"` + BackupRPCAddrs []string `json:"backup-rpc-addrs" yaml:"backup-rpc-addrs"` AccountPrefix string `json:"account-prefix" yaml:"account-prefix"` KeyringBackend string `json:"keyring-backend" yaml:"keyring-backend"` DynamicGasPrice bool `json:"dynamic-gas-price" yaml:"dynamic-gas-price"` @@ -273,6 +274,13 @@ func (cc *CosmosProvider) SetRpcAddr(rpcAddr string) error { return nil } +// SetBackupRpcAddrs sets the backup rpc-addr for the chain. +// These addrs are used in the event that the primary rpc-addr is down. +func (cc *CosmosProvider) SetBackupRpcAddrs(rpcAddrs []string) error { + cc.PCfg.BackupRPCAddrs = rpcAddrs + return nil +} + // Init initializes the keystore, RPC client, amd light client provider. // Once initialization is complete an attempt to query the underlying node's tendermint version is performed. // NOTE: Init must be called after creating a new instance of CosmosProvider. @@ -295,22 +303,123 @@ func (cc *CosmosProvider) Init(ctx context.Context) error { return err } - c, err := client.NewClient(cc.PCfg.RPCAddr, timeout) + // set the RPC client + err = cc.setRpcClient(true, cc.PCfg.RPCAddr, timeout) if err != nil { return err } - lightprovider, err := prov.New(cc.PCfg.ChainID, cc.PCfg.RPCAddr) + // set the light client provider + err = cc.setLightProvider(cc.PCfg.RPCAddr) if err != nil { return err } - rpcClient := cwrapper.NewRPCClient(c) - - cc.RPCClient = rpcClient - cc.LightProvider = lightprovider + // set keybase cc.Keybase = keybase + // go routine to monitor RPC liveliness + go cc.startLivelinessChecks(ctx, timeout) + + return nil +} + +// startLivelinessChecks frequently checks the liveliness of an RPC client and resorts to backup rpcs if the active rpc is down. +// This is a blocking function; call this within a go routine. +func (cc *CosmosProvider) startLivelinessChecks(ctx context.Context, timeout time.Duration) { + // list of rpcs & index to keep track of active rpc + rpcs := append([]string{cc.PCfg.RPCAddr}, cc.PCfg.BackupRPCAddrs...) + index := 0 + + // exit routine if there is only one rpc client + if len(rpcs) <= 1 { + cc.log.Debug("No backup RPCs defined", zap.String("chain", cc.ChainName())) + return + } + + // log the number of available rpcs + cc.log.Debug("Available RPC clients", zap.String("chain", cc.ChainName()), zap.Int("count", len(rpcs))) + + // tick every 10s to ensure rpc liveliness + ticker := time.NewTicker(10 * time.Second) + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + _, err := cc.RPCClient.Status(ctx) + if err != nil { + cc.log.Error("RPC client disconnected", zap.String("chain", cc.ChainName()), zap.Error(err)) + + attempts := 0 + + // attempt to connect to the backup RPC client + for { + + attempts++ + if attempts > len(rpcs) { + cc.log.Error("All configured RPC endpoints return non-200 response", zap.String("chain", cc.ChainName()), zap.Error(err)) + break + } + + // get next rpc + index = (index + 1) % len(rpcs) + rpcAddr := rpcs[index] + + cc.log.Info("Attempting to connect to new RPC", zap.String("chain", cc.ChainName()), zap.String("rpc", rpcAddr)) + + // attempt to setup rpc client + if err = cc.setRpcClient(false, rpcAddr, timeout); err != nil { + cc.log.Debug("Failed to connect to RPC client", zap.String("chain", cc.ChainName()), zap.String("rpc", rpcAddr), zap.Error(err)) + continue + } + + // attempt to setup light client + if err = cc.setLightProvider(rpcAddr); err != nil { + cc.log.Debug("Failed to connect to light client provider", zap.String("chain", cc.ChainName()), zap.String("rpc", rpcAddr), zap.Error(err)) + continue + } + + cc.log.Info("Successfully connected to new RPC", zap.String("chain", cc.ChainName()), zap.String("rpc", rpcAddr)) + + // rpc found, escape + break + } + } + } + } +} + +// setRpcClient sets the RPC client for the chain. +func (cc *CosmosProvider) setRpcClient(onStartup bool, rpcAddr string, timeout time.Duration) error { + c, err := client.NewClient(rpcAddr, timeout) + if err != nil { + return err + } + + cc.RPCClient = cwrapper.NewRPCClient(c) + + // Only check status if not on startup, to ensure the relayer will not block on startup. + // All subsequent calls will perform the status check to ensure RPC endpoints are rotated + // as necessary. + if !onStartup { + if _, err = cc.RPCClient.Status(context.Background()); err != nil { + return err + } + } + + return nil +} + +// setLightProvider sets the light client provider for the chain. +func (cc *CosmosProvider) setLightProvider(rpcAddr string) error { + lightprovider, err := prov.New(cc.PCfg.ChainID, rpcAddr) + if err != nil { + return err + } + + cc.LightProvider = lightprovider return nil } diff --git a/relayer/chains/penumbra/provider.go b/relayer/chains/penumbra/provider.go index 345205045..bb45ecb69 100644 --- a/relayer/chains/penumbra/provider.go +++ b/relayer/chains/penumbra/provider.go @@ -45,6 +45,7 @@ type PenumbraProviderConfig struct { ChainName string `json:"-" yaml:"-"` ChainID string `json:"chain-id" yaml:"chain-id"` RPCAddr string `json:"rpc-addr" yaml:"rpc-addr"` + BackupRPCAddrs []string `json:"backup-rpc-addrs" yaml:"backup-rpc-addrs"` AccountPrefix string `json:"account-prefix" yaml:"account-prefix"` KeyringBackend string `json:"keyring-backend" yaml:"keyring-backend"` GasAdjustment float64 `json:"gas-adjustment" yaml:"gas-adjustment"` @@ -234,6 +235,12 @@ func (cc *PenumbraProvider) SetRpcAddr(rpcAddr string) error { return nil } +// SetBackupRpcAddrs implements provider.ChainProvider. +func (cc *PenumbraProvider) SetBackupRpcAddrs(rpcAddrs []string) error { + cc.PCfg.BackupRPCAddrs = rpcAddrs + return nil +} + // Init initializes the keystore, RPC client, amd light client provider. // Once initialization is complete an attempt to query the underlying node's tendermint version is performed. // NOTE: Init must be called after creating a new instance of CosmosProvider. diff --git a/relayer/provider/provider.go b/relayer/provider/provider.go index 01be3b9cf..1cb995b21 100644 --- a/relayer/provider/provider.go +++ b/relayer/provider/provider.go @@ -409,6 +409,7 @@ type ChainProvider interface { Sprint(toPrint proto.Message) (string, error) SetRpcAddr(rpcAddr string) error + SetBackupRpcAddrs(rpcAddrs []string) error } // Do we need intermediate types? i.e. can we use the SDK types for both substrate and cosmos? From f5e8a54083f98534953b650893ff2531835c145f Mon Sep 17 00:00:00 2001 From: Joel Smith Date: Mon, 3 Jun 2024 16:10:17 -0500 Subject: [PATCH 02/12] fix command alias --- cmd/chains.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/chains.go b/cmd/chains.go index 634297f36..c38422e97 100644 --- a/cmd/chains.go +++ b/cmd/chains.go @@ -72,8 +72,8 @@ $ %s ch set-rpc-addr ibc-0 https://abc.xyz.com:443`, appName, appName)), func cmdChainsUseBackupRpcAddr(a *appState) *cobra.Command { cmd := &cobra.Command{ - Use: "set-backup-rpc-addr chain_name comma_separated_valid_rpc_urls", - Aliases: []string{"rpc"}, + Use: "set-backup-rpc-addrs chain_name comma_separated_valid_rpc_urls", + Aliases: []string{"set-backup-rpcs"}, Short: "Sets chain's backup rpc addresses", Args: withUsage(cobra.ExactArgs(2)), Example: strings.TrimSpace(fmt.Sprintf(` From 551a80146a855da8f82e65e659659bfec3ec2e90 Mon Sep 17 00:00:00 2001 From: Joel Smith Date: Mon, 3 Jun 2024 17:07:40 -0500 Subject: [PATCH 03/12] add backup rpcs from registry --- cregistry/chain_info.go | 49 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/cregistry/chain_info.go b/cregistry/chain_info.go index 0bb66e3d3..3ab8bb210 100644 --- a/cregistry/chain_info.go +++ b/cregistry/chain_info.go @@ -209,6 +209,48 @@ func (c ChainInfo) GetRandomRPCEndpoint(ctx context.Context, forceAdd bool) (str return endpoint, nil } +// GetBackupRPCEndpoints returns a slice of strings to be used as fallback, backup RPC endpoints. forceAdd will +// force the use of all available RPC endpoints, regardless of health. +func (c ChainInfo) GetBackupRPCEndpoints(ctx context.Context, forceAdd bool, primaryRPC string, count uint64) ([]string, error) { + // if force add, get all rpcs, otherwise get only healthy ones + var rpcs []string + var err error + if forceAdd { + rpcs, err = c.GetAllRPCEndpoints() + } else { + rpcs, err = c.GetRPCEndpoints(ctx) + } + if err != nil { + return nil, err + } + + // if no rpcs, return error + if len(rpcs) == 0 { + if !forceAdd { + return nil, fmt.Errorf("no working RPCs found, consider using --force-add") + } else { + return nil, nil + } + } + + // Select first two endpoints + backupRpcs := []string{} + for _, endpoint := range rpcs { + if len(backupRpcs) < 2 && primaryRPC != endpoint { + backupRpcs = append(backupRpcs, endpoint) + } else { + break + } + } + + // Log endpoints + c.log.Info("Backup Endpoints selected", + zap.String("chain_name", c.ChainName), + zap.Strings("endpoints", backupRpcs), + ) + return backupRpcs, nil +} + // GetAssetList returns the asset metadata from the cosmos chain registry for this particular chain. func (c ChainInfo) GetAssetList(ctx context.Context, testnet bool, name string) (AssetList, error) { var chainRegURL string @@ -265,10 +307,17 @@ func (c ChainInfo) GetChainConfig(ctx context.Context, forceAdd, testnet bool, n return nil, err } + // select 2 healthy endpoints as backup + backupRpcs, err := c.GetBackupRPCEndpoints(ctx, forceAdd, rpc, 2) + if err != nil { + return nil, err + } + return &cosmos.CosmosProviderConfig{ Key: "default", ChainID: c.ChainID, RPCAddr: rpc, + BackupRPCAddrs: backupRpcs, AccountPrefix: c.Bech32Prefix, KeyringBackend: "test", GasAdjustment: 1.2, From 9b1286b0662a2ff842ae1e32c9b021defd7731b1 Mon Sep 17 00:00:00 2001 From: Joel Smith Date: Tue, 4 Jun 2024 12:26:03 -0500 Subject: [PATCH 04/12] feat: liveliness check for penumbra --- relayer/chains/penumbra/provider.go | 111 +++++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 4 deletions(-) diff --git a/relayer/chains/penumbra/provider.go b/relayer/chains/penumbra/provider.go index bb45ecb69..e534d1e81 100644 --- a/relayer/chains/penumbra/provider.go +++ b/relayer/chains/penumbra/provider.go @@ -256,20 +256,123 @@ func (cc *PenumbraProvider) Init(ctx context.Context) error { return err } - lightprovider, err := prov.New(cc.PCfg.ChainID, cc.PCfg.RPCAddr) + // set the RPC client + err = cc.setRpcClient(true, cc.PCfg.RPCAddr, timeout) if err != nil { return err } - c, err := client.NewClient(cc.PCfg.RPCAddr, timeout) + // set the light client provider + err = cc.setLightProvider(cc.PCfg.RPCAddr) if err != nil { return err } - cc.RPCClient = cwrapper.NewRPCClient(c) - cc.LightProvider = lightprovider + // set keybase cc.Keybase = keybase + // go routine to monitor RPC liveliness + go cc.startLivelinessChecks(ctx, timeout) + + return nil +} + +// startLivelinessChecks frequently checks the liveliness of an RPC client and resorts to backup rpcs if the active rpc is down. +// This is a blocking function; call this within a go routine. +func (cc *PenumbraProvider) startLivelinessChecks(ctx context.Context, timeout time.Duration) { + // list of rpcs & index to keep track of active rpc + rpcs := append([]string{cc.PCfg.RPCAddr}, cc.PCfg.BackupRPCAddrs...) + index := 0 + + // exit routine if there is only one rpc client + if len(rpcs) <= 1 { + cc.log.Debug("No backup RPCs defined", zap.String("chain", cc.ChainName())) + return + } + + // log the number of available rpcs + cc.log.Debug("Available RPC clients", zap.String("chain", cc.ChainName()), zap.Int("count", len(rpcs))) + + // tick every 10s to ensure rpc liveliness + ticker := time.NewTicker(10 * time.Second) + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + _, err := cc.RPCClient.Status(ctx) + if err != nil { + cc.log.Error("RPC client disconnected", zap.String("chain", cc.ChainName()), zap.Error(err)) + + attempts := 0 + + // attempt to connect to the backup RPC client + for { + + attempts++ + if attempts > len(rpcs) { + cc.log.Error("All configured RPC endpoints return non-200 response", zap.String("chain", cc.ChainName()), zap.Error(err)) + break + } + + // get next rpc + index = (index + 1) % len(rpcs) + rpcAddr := rpcs[index] + + cc.log.Info("Attempting to connect to new RPC", zap.String("chain", cc.ChainName()), zap.String("rpc", rpcAddr)) + + // attempt to setup rpc client + if err = cc.setRpcClient(false, rpcAddr, timeout); err != nil { + cc.log.Debug("Failed to connect to RPC client", zap.String("chain", cc.ChainName()), zap.String("rpc", rpcAddr), zap.Error(err)) + continue + } + + // attempt to setup light client + if err = cc.setLightProvider(rpcAddr); err != nil { + cc.log.Debug("Failed to connect to light client provider", zap.String("chain", cc.ChainName()), zap.String("rpc", rpcAddr), zap.Error(err)) + continue + } + + cc.log.Info("Successfully connected to new RPC", zap.String("chain", cc.ChainName()), zap.String("rpc", rpcAddr)) + + // rpc found, escape + break + } + } + } + } +} + +// setRpcClient sets the RPC client for the chain. +func (cc *PenumbraProvider) setRpcClient(onStartup bool, rpcAddr string, timeout time.Duration) error { + c, err := client.NewClient(rpcAddr, timeout) + if err != nil { + return err + } + + cc.RPCClient = cwrapper.NewRPCClient(c) + + // Only check status if not on startup, to ensure the relayer will not block on startup. + // All subsequent calls will perform the status check to ensure RPC endpoints are rotated + // as necessary. + if !onStartup { + if _, err = cc.RPCClient.Status(context.Background()); err != nil { + return err + } + } + + return nil +} + +// setLightProvider sets the light client provider for the chain. +func (cc *PenumbraProvider) setLightProvider(rpcAddr string) error { + lightprovider, err := prov.New(cc.PCfg.ChainID, rpcAddr) + if err != nil { + return err + } + + cc.LightProvider = lightprovider return nil } From e290aa590a7dbb09addbcf5bf3b78733d95b2228 Mon Sep 17 00:00:00 2001 From: Joel Smith Date: Thu, 6 Jun 2024 16:37:37 -0500 Subject: [PATCH 05/12] Start work on interchaintests --- interchaintest/backup_rpc_test.go | 217 ++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 interchaintest/backup_rpc_test.go diff --git a/interchaintest/backup_rpc_test.go b/interchaintest/backup_rpc_test.go new file mode 100644 index 000000000..e2560c2f0 --- /dev/null +++ b/interchaintest/backup_rpc_test.go @@ -0,0 +1,217 @@ +package interchaintest_test + +import ( + "context" + "fmt" + "testing" + + sdkmath "cosmossdk.io/math" + transfertypes "github.com/cosmos/ibc-go/v8/modules/apps/transfer/types" + "github.com/cosmos/relayer/v2/cmd" + relayertest "github.com/cosmos/relayer/v2/interchaintest" + "github.com/cosmos/relayer/v2/relayer/chains/cosmos" + "github.com/strangelove-ventures/interchaintest/v8" + iccosmos "github.com/strangelove-ventures/interchaintest/v8/chain/cosmos" + "github.com/strangelove-ventures/interchaintest/v8/ibc" + "github.com/strangelove-ventures/interchaintest/v8/testreporter" + "github.com/strangelove-ventures/interchaintest/v8/testutil" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" + "gopkg.in/yaml.v3" +) + +// TestBackupRpcs tests the functionality of falling back to secondary RPCs when the primary node goes offline or becomes unresponsive. +func TestBackupRpcs(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + + t.Parallel() + + numVals := 1 + numFullNodes := 0 + + image := ibc.DockerImage{ + Repository: "ghcr.io/cosmos/ibc-go-simd", + Version: "v8.0.0", + UidGid: "100:1000", + } + + cf := interchaintest.NewBuiltinChainFactory(zaptest.NewLogger(t), []*interchaintest.ChainSpec{ + { + Name: "ibc-go-simd", + Version: "main", + NumValidators: &numVals, + NumFullNodes: &numFullNodes, + ChainConfig: ibc.ChainConfig{ + Type: "cosmos", + Name: "simd", + ChainID: "chain-a", + Images: []ibc.DockerImage{image}, + Bin: "simd", + Bech32Prefix: "cosmos", + Denom: "stake", + CoinType: "118", + GasPrices: "0.0stake", + GasAdjustment: 1.1, + }, + }, + { + Name: "ibc-go-simd", + Version: "main", + NumValidators: &numVals, + NumFullNodes: &numFullNodes, + ChainConfig: ibc.ChainConfig{ + Type: "cosmos", + Name: "simd", + ChainID: "chain-b", + Images: []ibc.DockerImage{image}, + Bin: "simd", + Bech32Prefix: "cosmos", + Denom: "stake", + CoinType: "118", + GasPrices: "0.0stake", + GasAdjustment: 1.1, + }, + }, + }) + + chains, err := cf.Chains(t.Name()) + require.NoError(t, err) + chainA, chainB := chains[0], chains[1] + + ctx := context.Background() + client, network := interchaintest.DockerSetup(t) + + rf := relayertest.NewRelayerFactory(relayertest.RelayerConfig{InitialBlockHistory: 50}) + r := rf.Build(t, client, network) + + const pathName = "chainA-chainB" + + ic := interchaintest.NewInterchain(). + AddChain(chainA). + AddChain(chainB). + AddRelayer(r, "relayer"). + AddLink(interchaintest.InterchainLink{ + Chain1: chainA, + Chain2: chainB, + Relayer: r, + Path: pathName, + }) + + rep := testreporter.NewNopReporter() + eRep := rep.RelayerExecReporter(t) + + require.NoError(t, ic.Build(ctx, eRep, interchaintest.InterchainBuildOptions{ + TestName: t.Name(), + Client: client, + NetworkID: network, + SkipPathCreation: false, + })) + + t.Cleanup(func() { + _ = ic.Close() + }) + + // Create and fund user accs & assert initial balances. + initBal := sdkmath.NewInt(1_000_000_000_000) + users := interchaintest.GetAndFundTestUsers(t, ctx, t.Name(), initBal, chainA, chainB) + require.NoError(t, testutil.WaitForBlocks(ctx, 2, chainA, chainB)) + + userA := users[0] + userB := users[1] + + userABal, err := chainA.GetBalance(ctx, userA.FormattedAddress(), chainA.Config().Denom) + require.NoError(t, err) + require.True(t, initBal.Equal(userABal)) + + userBBal, err := chainB.GetBalance(ctx, userB.FormattedAddress(), chainB.Config().Denom) + require.NoError(t, err) + require.True(t, initBal.Equal(userBBal)) + + // Read relayer config from disk, configure memo limit, & write config back to disk. + relayer := r.(*relayertest.Relayer) + + cfg := relayer.Sys().MustGetConfig(t) + cfgOutput := new(cmd.ConfigOutputWrapper) + cfgOutput.ProviderConfigs = cmd.ProviderConfigs{} + cfgOutput.Global = cfg.Global + cfgOutput.Paths = cfg.Paths + + // Get nodes associated with chainA + for _, chain := range [](*iccosmos.CosmosChain){chainA.(*iccosmos.CosmosChain), chainB.(*iccosmos.CosmosChain)} { + + addrs := []string{} + + for _, node := range chain.FullNodes { + + rpc := fmt.Sprintf("http://%s", node.HostRPCPort) + addrs = append(addrs, rpc) + } + + t.Log(addrs) + + value := cfg.ProviderConfigs[chain.Config().ChainID].Value.(*cosmos.CosmosProviderConfig) + value.RPCAddr = addrs[0] + value.BackupRPCAddrs = addrs[1:] + + cfgOutput.ProviderConfigs[chain.Config().ChainID] = &cmd.ProviderConfigWrapper{ + Type: "cosmos", + Value: value, + } + } + + cfgBz, err := yaml.Marshal(cfgOutput) + require.NoError(t, err) + + err = relayer.Sys().WriteConfig(t, cfgBz) + require.NoError(t, err) + + err = r.StartRelayer(ctx, eRep, pathName) + require.NoError(t, err) + + t.Cleanup( + func() { + err := r.StopRelayer(ctx, eRep) + if err != nil { + t.Logf("an error occurred while stopping the relayer: %s", err) + } + }, + ) + + // Send transfers that should succeed & assert balances. + channels, err := r.GetChannels(ctx, eRep, chainA.Config().ChainID) + require.NoError(t, err) + require.Equal(t, 1, len(channels)) + + channel := channels[0] + + transferAmount := sdkmath.NewInt(1_000) + + transferAB := ibc.WalletAmount{ + Address: userB.FormattedAddress(), + Denom: chainA.Config().Denom, + Amount: transferAmount, + } + + _, err = chainA.SendIBCTransfer(ctx, channel.ChannelID, userA.KeyName(), transferAB, ibc.TransferOptions{}) + require.NoError(t, err) + + require.NoError(t, testutil.WaitForBlocks(ctx, 5, chainA, chainB)) + + // Compose the ibc denom for balance assertions on the counterparty and assert balances. + denom := transfertypes.GetPrefixedDenom( + channel.Counterparty.PortID, + channel.Counterparty.ChannelID, + chainA.Config().Denom, + ) + trace := transfertypes.ParseDenomTrace(denom) + + userABal, err = chainA.GetBalance(ctx, userA.FormattedAddress(), chainA.Config().Denom) + require.NoError(t, err) + require.True(t, userABal.Equal(initBal.Sub(transferAmount))) + + userBBal, err = chainB.GetBalance(ctx, userB.FormattedAddress(), trace.IBCDenom()) + require.NoError(t, err) + require.True(t, userBBal.Equal(transferAmount)) +} From 9628ff01c68ff25ba784f7f9cd8973d65402622c Mon Sep 17 00:00:00 2001 From: Joel Smith Date: Fri, 7 Jun 2024 17:13:28 -0500 Subject: [PATCH 06/12] update log levels --- relayer/chains/cosmos/provider.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/relayer/chains/cosmos/provider.go b/relayer/chains/cosmos/provider.go index c212696f7..c4e483f30 100644 --- a/relayer/chains/cosmos/provider.go +++ b/relayer/chains/cosmos/provider.go @@ -371,13 +371,13 @@ func (cc *CosmosProvider) startLivelinessChecks(ctx context.Context, timeout tim // attempt to setup rpc client if err = cc.setRpcClient(false, rpcAddr, timeout); err != nil { - cc.log.Debug("Failed to connect to RPC client", zap.String("chain", cc.ChainName()), zap.String("rpc", rpcAddr), zap.Error(err)) + cc.log.Error("Failed to connect to RPC client", zap.String("chain", cc.ChainName()), zap.String("rpc", rpcAddr), zap.Error(err)) continue } // attempt to setup light client if err = cc.setLightProvider(rpcAddr); err != nil { - cc.log.Debug("Failed to connect to light client provider", zap.String("chain", cc.ChainName()), zap.String("rpc", rpcAddr), zap.Error(err)) + cc.log.Error("Failed to connect to light client provider", zap.String("chain", cc.ChainName()), zap.String("rpc", rpcAddr), zap.Error(err)) continue } From 6bc0ba7bcdaa0d6ce7d63f304c2876e576704032 Mon Sep 17 00:00:00 2001 From: Joel Smith Date: Fri, 7 Jun 2024 17:14:26 -0500 Subject: [PATCH 07/12] Use relayer image & get host port via bindings --- interchaintest/backup_rpc_test.go | 103 +++++++++++++++++++----------- 1 file changed, 64 insertions(+), 39 deletions(-) diff --git a/interchaintest/backup_rpc_test.go b/interchaintest/backup_rpc_test.go index e2560c2f0..e2c787ebe 100644 --- a/interchaintest/backup_rpc_test.go +++ b/interchaintest/backup_rpc_test.go @@ -11,8 +11,9 @@ import ( relayertest "github.com/cosmos/relayer/v2/interchaintest" "github.com/cosmos/relayer/v2/relayer/chains/cosmos" "github.com/strangelove-ventures/interchaintest/v8" - iccosmos "github.com/strangelove-ventures/interchaintest/v8/chain/cosmos" + interchaintestcosmos "github.com/strangelove-ventures/interchaintest/v8/chain/cosmos" "github.com/strangelove-ventures/interchaintest/v8/ibc" + interchaintestrelayer "github.com/strangelove-ventures/interchaintest/v8/relayer" "github.com/strangelove-ventures/interchaintest/v8/testreporter" "github.com/strangelove-ventures/interchaintest/v8/testutil" "github.com/stretchr/testify/require" @@ -28,50 +29,27 @@ func TestBackupRpcs(t *testing.T) { t.Parallel() - numVals := 1 + numVals := 5 numFullNodes := 0 - image := ibc.DockerImage{ - Repository: "ghcr.io/cosmos/ibc-go-simd", - Version: "v8.0.0", - UidGid: "100:1000", - } - cf := interchaintest.NewBuiltinChainFactory(zaptest.NewLogger(t), []*interchaintest.ChainSpec{ { - Name: "ibc-go-simd", - Version: "main", + Name: "gaia", + Version: "v7.0.0", NumValidators: &numVals, NumFullNodes: &numFullNodes, + ChainConfig: ibc.ChainConfig{ - Type: "cosmos", - Name: "simd", - ChainID: "chain-a", - Images: []ibc.DockerImage{image}, - Bin: "simd", - Bech32Prefix: "cosmos", - Denom: "stake", - CoinType: "118", - GasPrices: "0.0stake", - GasAdjustment: 1.1, + GasPrices: "0.0uatom", }, }, { - Name: "ibc-go-simd", - Version: "main", + Name: "osmosis", + Version: "v11.0.0", NumValidators: &numVals, NumFullNodes: &numFullNodes, ChainConfig: ibc.ChainConfig{ - Type: "cosmos", - Name: "simd", - ChainID: "chain-b", - Images: []ibc.DockerImage{image}, - Bin: "simd", - Bech32Prefix: "cosmos", - Denom: "stake", - CoinType: "118", - GasPrices: "0.0stake", - GasAdjustment: 1.1, + GasPrices: "0.0uosmo", }, }, }) @@ -83,7 +61,15 @@ func TestBackupRpcs(t *testing.T) { ctx := context.Background() client, network := interchaintest.DockerSetup(t) - rf := relayertest.NewRelayerFactory(relayertest.RelayerConfig{InitialBlockHistory: 50}) + image := relayertest.BuildRelayerImage(t) + rf := interchaintest.NewBuiltinRelayerFactory( + ibc.CosmosRly, + zaptest.NewLogger(t), + interchaintestrelayer.CustomDockerImage(image, "latest", "100:1000"), + interchaintestrelayer.ImagePull(false), + interchaintestrelayer.StartupFlags("--processor", "events", "--block-history", "100"), + ) + r := rf.Build(t, client, network) const pathName = "chainA-chainB" @@ -138,22 +124,42 @@ func TestBackupRpcs(t *testing.T) { cfgOutput.Global = cfg.Global cfgOutput.Paths = cfg.Paths - // Get nodes associated with chainA - for _, chain := range [](*iccosmos.CosmosChain){chainA.(*iccosmos.CosmosChain), chainB.(*iccosmos.CosmosChain)} { + // Get all chains + for _, chain := range [](*interchaintestcosmos.CosmosChain){chainA.(*interchaintestcosmos.CosmosChain), chainB.(*interchaintestcosmos.CosmosChain)} { addrs := []string{} - for _, node := range chain.FullNodes { + // loop through nodes to collect rpc addrs + for _, node := range chain.Validators { + + cl := node.DockerClient + inspect, _ := cl.ContainerInspect(ctx, node.ContainerID()) + require.NoError(t, err) + + portBindings := inspect.HostConfig.PortBindings + hostPort := "" + + // find the host port mapped to 26657 on the container + for port, bindings := range portBindings { + if port == "26657/tcp" { + for _, binding := range bindings { + hostPort = binding.HostPort + break + } + break + } + } - rpc := fmt.Sprintf("http://%s", node.HostRPCPort) + rpc := fmt.Sprintf("http://127.0.0.1:%s", hostPort) addrs = append(addrs, rpc) } + // TODO: REMOVE ME t.Log(addrs) value := cfg.ProviderConfigs[chain.Config().ChainID].Value.(*cosmos.CosmosProviderConfig) - value.RPCAddr = addrs[0] - value.BackupRPCAddrs = addrs[1:] + value.RPCAddr = addrs[numVals-1] + value.BackupRPCAddrs = addrs[:numVals-1] cfgOutput.ProviderConfigs[chain.Config().ChainID] = &cmd.ProviderConfigWrapper{ Type: "cosmos", @@ -194,6 +200,25 @@ func TestBackupRpcs(t *testing.T) { Amount: transferAmount, } + t.Log("========== BEFORE SHUT DOWN LAST VALIDATOR ==========") + t.Log("========== BEFORE SHUT DOWN LAST VALIDATOR ==========") + t.Log("========== BEFORE SHUT DOWN LAST VALIDATOR ==========") + + // wait 10 blocks + require.NoError(t, testutil.WaitForBlocks(ctx, 10, chainA, chainB)) + + // turn off last node + val := chainA.(*interchaintestcosmos.CosmosChain).Validators[numVals-1] + err = val.PauseContainer(ctx) + require.NoError(t, err) + + // wait 10 blocks + require.NoError(t, testutil.WaitForBlocks(ctx, 10, chainA, chainB)) + + t.Log("========== AFTER SHUT DOWN LAST VALIDATOR ==========") + t.Log("========== AFTER SHUT DOWN LAST VALIDATOR ==========") + t.Log("========== AFTER SHUT DOWN LAST VALIDATOR ==========") + _, err = chainA.SendIBCTransfer(ctx, channel.ChannelID, userA.KeyName(), transferAB, ibc.TransferOptions{}) require.NoError(t, err) From e1e7e9950213b3a950adb096bde6a79dbdfeee50 Mon Sep 17 00:00:00 2001 From: Joel Smith Date: Fri, 7 Jun 2024 18:53:11 -0500 Subject: [PATCH 08/12] Populate config with CLI --- interchaintest/backup_rpc_test.go | 78 ++++++++----------------------- 1 file changed, 20 insertions(+), 58 deletions(-) diff --git a/interchaintest/backup_rpc_test.go b/interchaintest/backup_rpc_test.go index e2c787ebe..98bf83fb5 100644 --- a/interchaintest/backup_rpc_test.go +++ b/interchaintest/backup_rpc_test.go @@ -3,22 +3,21 @@ package interchaintest_test import ( "context" "fmt" + "strings" "testing" sdkmath "cosmossdk.io/math" transfertypes "github.com/cosmos/ibc-go/v8/modules/apps/transfer/types" - "github.com/cosmos/relayer/v2/cmd" relayertest "github.com/cosmos/relayer/v2/interchaintest" - "github.com/cosmos/relayer/v2/relayer/chains/cosmos" "github.com/strangelove-ventures/interchaintest/v8" interchaintestcosmos "github.com/strangelove-ventures/interchaintest/v8/chain/cosmos" "github.com/strangelove-ventures/interchaintest/v8/ibc" interchaintestrelayer "github.com/strangelove-ventures/interchaintest/v8/relayer" + "github.com/strangelove-ventures/interchaintest/v8/relayer/rly" "github.com/strangelove-ventures/interchaintest/v8/testreporter" "github.com/strangelove-ventures/interchaintest/v8/testutil" "github.com/stretchr/testify/require" "go.uber.org/zap/zaptest" - "gopkg.in/yaml.v3" ) // TestBackupRpcs tests the functionality of falling back to secondary RPCs when the primary node goes offline or becomes unresponsive. @@ -115,14 +114,7 @@ func TestBackupRpcs(t *testing.T) { require.NoError(t, err) require.True(t, initBal.Equal(userBBal)) - // Read relayer config from disk, configure memo limit, & write config back to disk. - relayer := r.(*relayertest.Relayer) - - cfg := relayer.Sys().MustGetConfig(t) - cfgOutput := new(cmd.ConfigOutputWrapper) - cfgOutput.ProviderConfigs = cmd.ProviderConfigs{} - cfgOutput.Global = cfg.Global - cfgOutput.Paths = cfg.Paths + rly := r.(*rly.CosmosRelayer) // Get all chains for _, chain := range [](*interchaintestcosmos.CosmosChain){chainA.(*interchaintestcosmos.CosmosChain), chainB.(*interchaintestcosmos.CosmosChain)} { @@ -131,48 +123,19 @@ func TestBackupRpcs(t *testing.T) { // loop through nodes to collect rpc addrs for _, node := range chain.Validators { - - cl := node.DockerClient - inspect, _ := cl.ContainerInspect(ctx, node.ContainerID()) - require.NoError(t, err) - - portBindings := inspect.HostConfig.PortBindings - hostPort := "" - - // find the host port mapped to 26657 on the container - for port, bindings := range portBindings { - if port == "26657/tcp" { - for _, binding := range bindings { - hostPort = binding.HostPort - break - } - break - } - } - - rpc := fmt.Sprintf("http://127.0.0.1:%s", hostPort) + rpc := fmt.Sprintf("http://%s:26657", node.Name()) addrs = append(addrs, rpc) } - // TODO: REMOVE ME - t.Log(addrs) + cmd := []string{"rly", "chains", "set-rpc-addr", chain.Config().Name, addrs[numVals-1], "--home", rly.HomeDir()} + res := r.Exec(ctx, eRep, cmd, nil) + require.NoError(t, res.Err) - value := cfg.ProviderConfigs[chain.Config().ChainID].Value.(*cosmos.CosmosProviderConfig) - value.RPCAddr = addrs[numVals-1] - value.BackupRPCAddrs = addrs[:numVals-1] - - cfgOutput.ProviderConfigs[chain.Config().ChainID] = &cmd.ProviderConfigWrapper{ - Type: "cosmos", - Value: value, - } + cmd = []string{"rly", "chains", "set-backup-rpc-addrs", chain.Config().Name, strings.Join(addrs[:numVals-1], ","), "--home", rly.HomeDir()} + res = r.Exec(ctx, eRep, cmd, nil) + require.NoError(t, res.Err) } - cfgBz, err := yaml.Marshal(cfgOutput) - require.NoError(t, err) - - err = relayer.Sys().WriteConfig(t, cfgBz) - require.NoError(t, err) - err = r.StartRelayer(ctx, eRep, pathName) require.NoError(t, err) @@ -200,29 +163,27 @@ func TestBackupRpcs(t *testing.T) { Amount: transferAmount, } - t.Log("========== BEFORE SHUT DOWN LAST VALIDATOR ==========") - t.Log("========== BEFORE SHUT DOWN LAST VALIDATOR ==========") - t.Log("========== BEFORE SHUT DOWN LAST VALIDATOR ==========") - // wait 10 blocks require.NoError(t, testutil.WaitForBlocks(ctx, 10, chainA, chainB)) - // turn off last node + // turn off last nodes on both chains val := chainA.(*interchaintestcosmos.CosmosChain).Validators[numVals-1] - err = val.PauseContainer(ctx) + err = val.StopContainer(ctx) + require.NoError(t, err) + + val = chainB.(*interchaintestcosmos.CosmosChain).Validators[numVals-1] + err = val.StopContainer(ctx) require.NoError(t, err) // wait 10 blocks require.NoError(t, testutil.WaitForBlocks(ctx, 10, chainA, chainB)) - t.Log("========== AFTER SHUT DOWN LAST VALIDATOR ==========") - t.Log("========== AFTER SHUT DOWN LAST VALIDATOR ==========") - t.Log("========== AFTER SHUT DOWN LAST VALIDATOR ==========") - + // send ibc tx from chain a to b _, err = chainA.SendIBCTransfer(ctx, channel.ChannelID, userA.KeyName(), transferAB, ibc.TransferOptions{}) require.NoError(t, err) - require.NoError(t, testutil.WaitForBlocks(ctx, 5, chainA, chainB)) + // wait 20 blocks so relayer can pick up the tx + require.NoError(t, testutil.WaitForBlocks(ctx, 20, chainA, chainB)) // Compose the ibc denom for balance assertions on the counterparty and assert balances. denom := transfertypes.GetPrefixedDenom( @@ -232,6 +193,7 @@ func TestBackupRpcs(t *testing.T) { ) trace := transfertypes.ParseDenomTrace(denom) + // validate user balances on both chains userABal, err = chainA.GetBalance(ctx, userA.FormattedAddress(), chainA.Config().Denom) require.NoError(t, err) require.True(t, userABal.Equal(initBal.Sub(transferAmount))) From dda253ee96917c4cb50a96abfb052e7056b23eba Mon Sep 17 00:00:00 2001 From: Joel Smith Date: Mon, 10 Jun 2024 10:03:06 -0500 Subject: [PATCH 09/12] fix nil pointer ref --- relayer/chains/cosmos/provider.go | 4 +++- relayer/chains/penumbra/provider.go | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/relayer/chains/cosmos/provider.go b/relayer/chains/cosmos/provider.go index c4e483f30..425fbe501 100644 --- a/relayer/chains/cosmos/provider.go +++ b/relayer/chains/cosmos/provider.go @@ -333,7 +333,9 @@ func (cc *CosmosProvider) startLivelinessChecks(ctx context.Context, timeout tim // exit routine if there is only one rpc client if len(rpcs) <= 1 { - cc.log.Debug("No backup RPCs defined", zap.String("chain", cc.ChainName())) + if cc.log != nil { + cc.log.Debug("No backup RPCs defined", zap.String("chain", cc.ChainName())) + } return } diff --git a/relayer/chains/penumbra/provider.go b/relayer/chains/penumbra/provider.go index e534d1e81..51d490292 100644 --- a/relayer/chains/penumbra/provider.go +++ b/relayer/chains/penumbra/provider.go @@ -286,7 +286,9 @@ func (cc *PenumbraProvider) startLivelinessChecks(ctx context.Context, timeout t // exit routine if there is only one rpc client if len(rpcs) <= 1 { - cc.log.Debug("No backup RPCs defined", zap.String("chain", cc.ChainName())) + if cc.log != nil { + cc.log.Debug("No backup RPCs defined", zap.String("chain", cc.ChainName())) + } return } From 1d9d303775f14df8a0527c46427e1510161a6b7b Mon Sep 17 00:00:00 2001 From: Joel Smith Date: Mon, 10 Jun 2024 10:22:10 -0500 Subject: [PATCH 10/12] retry primary rpc first --- relayer/chains/cosmos/provider.go | 2 +- relayer/chains/penumbra/provider.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/relayer/chains/cosmos/provider.go b/relayer/chains/cosmos/provider.go index 425fbe501..2f88097d6 100644 --- a/relayer/chains/cosmos/provider.go +++ b/relayer/chains/cosmos/provider.go @@ -329,7 +329,6 @@ func (cc *CosmosProvider) Init(ctx context.Context) error { func (cc *CosmosProvider) startLivelinessChecks(ctx context.Context, timeout time.Duration) { // list of rpcs & index to keep track of active rpc rpcs := append([]string{cc.PCfg.RPCAddr}, cc.PCfg.BackupRPCAddrs...) - index := 0 // exit routine if there is only one rpc client if len(rpcs) <= 1 { @@ -354,6 +353,7 @@ func (cc *CosmosProvider) startLivelinessChecks(ctx context.Context, timeout tim if err != nil { cc.log.Error("RPC client disconnected", zap.String("chain", cc.ChainName()), zap.Error(err)) + index := -1 attempts := 0 // attempt to connect to the backup RPC client diff --git a/relayer/chains/penumbra/provider.go b/relayer/chains/penumbra/provider.go index 51d490292..603a2a4fc 100644 --- a/relayer/chains/penumbra/provider.go +++ b/relayer/chains/penumbra/provider.go @@ -282,7 +282,6 @@ func (cc *PenumbraProvider) Init(ctx context.Context) error { func (cc *PenumbraProvider) startLivelinessChecks(ctx context.Context, timeout time.Duration) { // list of rpcs & index to keep track of active rpc rpcs := append([]string{cc.PCfg.RPCAddr}, cc.PCfg.BackupRPCAddrs...) - index := 0 // exit routine if there is only one rpc client if len(rpcs) <= 1 { @@ -307,6 +306,7 @@ func (cc *PenumbraProvider) startLivelinessChecks(ctx context.Context, timeout t if err != nil { cc.log.Error("RPC client disconnected", zap.String("chain", cc.ChainName()), zap.Error(err)) + index := -1 attempts := 0 // attempt to connect to the backup RPC client From 9cdbdcc748f781acab72268fb0dbc38b1b8c7f93 Mon Sep 17 00:00:00 2001 From: Joel Smith Date: Tue, 11 Jun 2024 14:14:22 -0500 Subject: [PATCH 11/12] fix loop variable --- cmd/chains.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/chains.go b/cmd/chains.go index c38422e97..aeba7323a 100644 --- a/cmd/chains.go +++ b/cmd/chains.go @@ -83,11 +83,12 @@ $ %s ch set-backup-rpc-addr ibc-0 https://abc.xyz.com:443,https://123.456.com:44 chainName := args[0] rpc_addresses := args[1] - // Split rpc_addresses by ',' + // split rpc_addresses by ',' rpc_addresses_list := strings.Split(rpc_addresses, ",") - // Loop through and ensure valid + // loop through and ensure valid for _, rpc_address := range rpc_addresses_list { + rpc_address := rpc_address if !isValidURL(rpc_address) { return invalidRpcAddr(rpc_address) } From 31ba93e1a38ee1aceac76eeaf8b72b4955dba47d Mon Sep 17 00:00:00 2001 From: Joel Smith Date: Tue, 11 Jun 2024 14:40:11 -0500 Subject: [PATCH 12/12] poll for acknowledgement --- interchaintest/backup_rpc_test.go | 37 ++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/interchaintest/backup_rpc_test.go b/interchaintest/backup_rpc_test.go index 98bf83fb5..4790dd470 100644 --- a/interchaintest/backup_rpc_test.go +++ b/interchaintest/backup_rpc_test.go @@ -8,11 +8,12 @@ import ( sdkmath "cosmossdk.io/math" transfertypes "github.com/cosmos/ibc-go/v8/modules/apps/transfer/types" + chantypes "github.com/cosmos/ibc-go/v8/modules/core/04-channel/types" relayertest "github.com/cosmos/relayer/v2/interchaintest" "github.com/strangelove-ventures/interchaintest/v8" - interchaintestcosmos "github.com/strangelove-ventures/interchaintest/v8/chain/cosmos" + "github.com/strangelove-ventures/interchaintest/v8/chain/cosmos" "github.com/strangelove-ventures/interchaintest/v8/ibc" - interchaintestrelayer "github.com/strangelove-ventures/interchaintest/v8/relayer" + icrelayer "github.com/strangelove-ventures/interchaintest/v8/relayer" "github.com/strangelove-ventures/interchaintest/v8/relayer/rly" "github.com/strangelove-ventures/interchaintest/v8/testreporter" "github.com/strangelove-ventures/interchaintest/v8/testutil" @@ -64,9 +65,9 @@ func TestBackupRpcs(t *testing.T) { rf := interchaintest.NewBuiltinRelayerFactory( ibc.CosmosRly, zaptest.NewLogger(t), - interchaintestrelayer.CustomDockerImage(image, "latest", "100:1000"), - interchaintestrelayer.ImagePull(false), - interchaintestrelayer.StartupFlags("--processor", "events", "--block-history", "100"), + icrelayer.CustomDockerImage(image, "latest", "100:1000"), + icrelayer.ImagePull(false), + icrelayer.StartupFlags("--processor", "events", "--block-history", "100"), ) r := rf.Build(t, client, network) @@ -117,7 +118,7 @@ func TestBackupRpcs(t *testing.T) { rly := r.(*rly.CosmosRelayer) // Get all chains - for _, chain := range [](*interchaintestcosmos.CosmosChain){chainA.(*interchaintestcosmos.CosmosChain), chainB.(*interchaintestcosmos.CosmosChain)} { + for _, chain := range [](*cosmos.CosmosChain){chainA.(*cosmos.CosmosChain), chainB.(*cosmos.CosmosChain)} { addrs := []string{} @@ -167,11 +168,11 @@ func TestBackupRpcs(t *testing.T) { require.NoError(t, testutil.WaitForBlocks(ctx, 10, chainA, chainB)) // turn off last nodes on both chains - val := chainA.(*interchaintestcosmos.CosmosChain).Validators[numVals-1] + val := chainA.(*cosmos.CosmosChain).Validators[numVals-1] err = val.StopContainer(ctx) require.NoError(t, err) - val = chainB.(*interchaintestcosmos.CosmosChain).Validators[numVals-1] + val = chainB.(*cosmos.CosmosChain).Validators[numVals-1] err = val.StopContainer(ctx) require.NoError(t, err) @@ -179,11 +180,25 @@ func TestBackupRpcs(t *testing.T) { require.NoError(t, testutil.WaitForBlocks(ctx, 10, chainA, chainB)) // send ibc tx from chain a to b - _, err = chainA.SendIBCTransfer(ctx, channel.ChannelID, userA.KeyName(), transferAB, ibc.TransferOptions{}) + tx, err := chainA.SendIBCTransfer(ctx, channel.ChannelID, userA.KeyName(), transferAB, ibc.TransferOptions{}) require.NoError(t, err) + require.NoError(t, tx.Validate()) - // wait 20 blocks so relayer can pick up the tx - require.NoError(t, testutil.WaitForBlocks(ctx, 20, chainA, chainB)) + // get chain b height + bHeight, err := chainB.Height(ctx) + require.NoError(t, err) + + // Poll for MsgRecvPacket on b chain + _, err = cosmos.PollForMessage[*chantypes.MsgRecvPacket](ctx, chainB.(*cosmos.CosmosChain), cosmos.DefaultEncoding().InterfaceRegistry, bHeight, bHeight+20, nil) + require.NoError(t, err) + + // get chain a height + aHeight, err := chainA.Height(ctx) + require.NoError(t, err) + + // poll for acknowledge on 'a' chain + _, err = testutil.PollForAck(ctx, chainA.(*cosmos.CosmosChain), aHeight, aHeight+30, tx.Packet) + require.NoError(t, err) // Compose the ibc denom for balance assertions on the counterparty and assert balances. denom := transfertypes.GetPrefixedDenom(