diff --git a/changelog.md b/changelog.md index 1fa1f6d681..e617224247 100644 --- a/changelog.md +++ b/changelog.md @@ -36,6 +36,7 @@ * [1942](https://github.com/zeta-chain/node/pull/1982) - support Bitcoin P2TR, P2WSH, P2SH, P2PKH addresses * [1935](https://github.com/zeta-chain/node/pull/1935) - add an operational authority group * [1954](https://github.com/zeta-chain/node/pull/1954) - add metric for concurrent keysigns +* [1998](https://github.com/zeta-chain/node/pull/1998) - zetaclient multiple rpc endpoint support ### Tests diff --git a/cmd/zetaclientd/debug.go b/cmd/zetaclientd/debug.go index dc5e69f8a4..43d03dce5b 100644 --- a/cmd/zetaclientd/debug.go +++ b/cmd/zetaclientd/debug.go @@ -104,7 +104,7 @@ func DebugCmd() *cobra.Command { coinType := coin.CoinType_Cmd for chain, evmConfig := range cfg.GetAllEVMConfigs() { if chainID == chain { - client, err = evm.NewEthClientFallback(&evmConfig, chainLogger) + client, err = evm.NewEthClientFallback(evmConfig, chainLogger) if err != nil { return err } diff --git a/cmd/zetaclientd/start_utils.go b/cmd/zetaclientd/start_utils.go index e0aa59002e..947789d4a9 100644 --- a/cmd/zetaclientd/start_utils.go +++ b/cmd/zetaclientd/start_utils.go @@ -73,14 +73,16 @@ func maskCfg(cfg config.Config) string { // Mask Sensitive data for _, chain := range maskedCfg.EVMChainConfigs { - if chain.Endpoint == "" { + if len(chain.Endpoint) == 0 { continue } - endpointURL, err := url.Parse(chain.Endpoint) - if err != nil { - continue + for i, endpoint := range chain.Endpoint { + endpointURL, err := url.Parse(endpoint) + if err != nil { + continue + } + chain.Endpoint[i] = endpointURL.Hostname() } - chain.Endpoint = endpointURL.Hostname() } maskedCfg.BitcoinConfig.RPCUsername = "" diff --git a/cmd/zetaclientd/utils.go b/cmd/zetaclientd/utils.go index a9fc63e1f8..d14c7c609d 100644 --- a/cmd/zetaclientd/utils.go +++ b/cmd/zetaclientd/utils.go @@ -70,8 +70,7 @@ func CreateSignerMap( mpiAddress := ethcommon.HexToAddress(evmChainParams.ConnectorContractAddress) erc20CustodyAddress := ethcommon.HexToAddress(evmChainParams.Erc20CustodyContractAddress) signer, err := evm.NewEVMSigner( - evmConfig.Chain, - evmConfig.Endpoint, + evmConfig, tss, config.GetConnectorABI(), config.GetERC20CustodyABI(), diff --git a/zetaclient/config/config_chain.go b/zetaclient/config/config_chain.go index f211f1030d..f1ed03cd0c 100644 --- a/zetaclient/config/config_chain.go +++ b/zetaclient/config/config_chain.go @@ -52,22 +52,22 @@ var evmChainsConfigs = map[int64]EVMConfig{ }, chains.GoerliChain().ChainId: { Chain: chains.GoerliChain(), - Endpoint: "", + Endpoint: []string{}, }, chains.SepoliaChain().ChainId: { Chain: chains.SepoliaChain(), - Endpoint: "", + Endpoint: []string{}, }, chains.BscTestnetChain().ChainId: { Chain: chains.BscTestnetChain(), - Endpoint: "", + Endpoint: []string{}, }, chains.MumbaiChain().ChainId: { Chain: chains.MumbaiChain(), - Endpoint: "", + Endpoint: []string{}, }, chains.GoerliLocalnetChain().ChainId: { Chain: chains.GoerliLocalnetChain(), - Endpoint: "http://eth:8545", + Endpoint: []string{"http://eth:8545"}, }, } diff --git a/zetaclient/evm/evm_client.go b/zetaclient/evm/evm_client.go index ba452bb305..7bf2c57437 100644 --- a/zetaclient/evm/evm_client.go +++ b/zetaclient/evm/evm_client.go @@ -69,7 +69,7 @@ var _ interfaces.ChainClient = &ChainClient{} // Filled with above constants depending on chain type ChainClient struct { chain chains.Chain - evmClient EthClientFallbackInterface + evmClient interfaces.EthClientFallback zetaClient interfaces.ZetaCoreBridger Tss interfaces.TSSSigner lastBlockScanned uint64 @@ -129,7 +129,7 @@ func NewEVMChainClient( ob.outTXConfirmedTransactions = make(map[string]*ethtypes.Transaction) // Initialize evm client - client, err := NewEthClientFallback(&evmCfg, chainLogger) + client, err := NewEthClientFallback(evmCfg, chainLogger) if err != nil { return nil, err } diff --git a/zetaclient/evm/evm_rpc_fallback.go b/zetaclient/evm/evm_rpc_fallback.go index b55317b1d5..dd4fe0738f 100644 --- a/zetaclient/evm/evm_rpc_fallback.go +++ b/zetaclient/evm/evm_rpc_fallback.go @@ -3,6 +3,8 @@ package evm import ( "context" "errors" + "math/big" + "github.com/ethereum/go-ethereum" ethcommon "github.com/ethereum/go-ethereum/common" ethtypes "github.com/ethereum/go-ethereum/core/types" @@ -12,35 +14,27 @@ import ( "github.com/zeta-chain/zetacore/zetaclient/common" "github.com/zeta-chain/zetacore/zetaclient/config" "github.com/zeta-chain/zetacore/zetaclient/interfaces" - "math/big" ) -// EthClientFallbackInterface consolidates interfaces to external chain clients -type EthClientFallbackInterface interface { - // EVMRPCClient EVMJSONRPCClient - Need to implement both interfaces to support newer data types - interfaces.EVMRPCClient - interfaces.EVMJSONRPCClient -} - -var _ EthClientFallbackInterface = &EthClientFallback{} +var _ interfaces.EthClientFallback = &EthClientFallback{} // EthClientFallback is a decorator combining client interfaces used by evm chains. Also encapsulates list of clients // defined by endpoints from the config. type EthClientFallback struct { - evmCfg *config.EVMConfig + evmCfg config.EVMConfig ethClients *common.ClientQueue - jsonRpcClients *common.ClientQueue + jsonRPCClients *common.ClientQueue logger zerolog.Logger } // NewEthClientFallback creates new instance of eth client used by evm chain client. -func NewEthClientFallback(evmCfg *config.EVMConfig, logger zerolog.Logger) (*EthClientFallback, error) { +func NewEthClientFallback(evmCfg config.EVMConfig, logger zerolog.Logger) (*EthClientFallback, error) { if len(evmCfg.Endpoint) == 0 { return nil, errors.New("invalid endpoints") } ethClientFallback := EthClientFallback{} ethClientFallback.ethClients = common.NewClientQueue() - ethClientFallback.jsonRpcClients = common.NewClientQueue() + ethClientFallback.jsonRPCClients = common.NewClientQueue() // Initialize clients for _, endpoint := range evmCfg.Endpoint { @@ -53,8 +47,8 @@ func NewEthClientFallback(evmCfg *config.EVMConfig, logger zerolog.Logger) (*Eth ethClientFallback.ethClients.Append(client) //Initialize jsonRPC clients from https://github.com/onrik/ethrpc - jsonRpcClient := ethrpc.NewEthRPC(endpoint) - ethClientFallback.jsonRpcClients.Append(jsonRpcClient) + jsonRPCClient := ethrpc.NewEthRPC(endpoint) + ethClientFallback.jsonRPCClients.Append(jsonRPCClient) } ethClientFallback.evmCfg = evmCfg @@ -346,6 +340,21 @@ func (e *EthClientFallback) TransactionSender(ctx context.Context, tx *ethtypes. return res, err } +func (e *EthClientFallback) ChainID(ctx context.Context) (res *big.Int, err error) { + for i := 0; i < e.ethClients.Length(); i++ { + if client := e.ethClients.First(); client != nil { + rpcClient := client.(ethclient.Client) + res, err = rpcClient.ChainID(ctx) + } + if err != nil { + e.ethClients.Next() + continue + } + break + } + return +} + // Implementation of interface for jsonRPC eth client - https://github.com/onrik/ethrpc // EthGetBlockByNumber implementation of interfaces.EVMJSONRPCClient @@ -353,13 +362,13 @@ func (e *EthClientFallback) EthGetBlockByNumber(number int, withTransactions boo var res *ethrpc.Block var err error - for i := 0; i < e.jsonRpcClients.Length(); i++ { - if client := e.jsonRpcClients.First(); client != nil { + for i := 0; i < e.jsonRPCClients.Length(); i++ { + if client := e.jsonRPCClients.First(); client != nil { res, err = client.(interfaces.EVMJSONRPCClient).EthGetBlockByNumber(number, withTransactions) } if err != nil { e.logger.Debug().Err(err).Msg("client endpoint failed attempting fallback client") - e.jsonRpcClients.Next() + e.jsonRPCClients.Next() continue } break @@ -372,13 +381,13 @@ func (e *EthClientFallback) EthGetTransactionByHash(hash string) (*ethrpc.Transa var res *ethrpc.Transaction var err error - for i := 0; i < e.jsonRpcClients.Length(); i++ { - if client := e.jsonRpcClients.First(); client != nil { + for i := 0; i < e.jsonRPCClients.Length(); i++ { + if client := e.jsonRPCClients.First(); client != nil { res, err = client.(interfaces.EVMJSONRPCClient).EthGetTransactionByHash(hash) } if err != nil { e.logger.Debug().Err(err).Msg("client endpoint failed attempting fallback client") - e.jsonRpcClients.Next() + e.jsonRPCClients.Next() continue } break diff --git a/zetaclient/evm/evm_signer.go b/zetaclient/evm/evm_signer.go index e077d19fe2..eb87449a8d 100644 --- a/zetaclient/evm/evm_signer.go +++ b/zetaclient/evm/evm_signer.go @@ -11,12 +11,13 @@ import ( "sync" "time" + "github.com/zeta-chain/zetacore/zetaclient/config" + sdk "github.com/cosmos/cosmos-sdk/types" "github.com/ethereum/go-ethereum/accounts/abi" ethcommon "github.com/ethereum/go-ethereum/common" ethtypes "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethclient" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/zeta-chain/protocol-contracts/pkg/contracts/evm/erc20custody.sol" @@ -38,7 +39,7 @@ import ( // Signer deals with the signing EVM transactions and implements the ChainSigner interface type Signer struct { - client interfaces.EVMRPCClient + client interfaces.EthClientFallback chain *chains.Chain tssSigner interfaces.TSSSigner ethSigner ethtypes.Signer @@ -58,8 +59,7 @@ type Signer struct { var _ interfaces.ChainSigner = &Signer{} func NewEVMSigner( - chain chains.Chain, - endpoint string, + evmCfg config.EVMConfig, tssSigner interfaces.TSSSigner, zetaConnectorABI string, erc20CustodyABI string, @@ -69,7 +69,11 @@ func NewEVMSigner( loggers clientcommon.ClientLogger, ts *metrics.TelemetryServer, ) (*Signer, error) { - client, ethSigner, err := getEVMRPC(endpoint) + logger := clientcommon.ClientLogger{ + Std: loggers.Std.With().Str("chain", evmCfg.Chain.ChainName.String()).Str("module", "EVMSigner").Logger(), + Compliance: loggers.Compliance, + } + client, ethSigner, err := getEVMRPC(evmCfg, logger) if err != nil { return nil, err } @@ -83,19 +87,16 @@ func NewEVMSigner( } return &Signer{ - client: client, - chain: &chain, - tssSigner: tssSigner, - ethSigner: ethSigner, - zetaConnectorABI: connectorABI, - erc20CustodyABI: custodyABI, - zetaConnectorAddress: zetaConnectorAddress, - er20CustodyAddress: erc20CustodyAddress, - coreContext: coreContext, - logger: clientcommon.ClientLogger{ - Std: loggers.Std.With().Str("chain", chain.ChainName.String()).Str("module", "EVMSigner").Logger(), - Compliance: loggers.Compliance, - }, + client: client, + chain: &evmCfg.Chain, + tssSigner: tssSigner, + ethSigner: ethSigner, + zetaConnectorABI: connectorABI, + erc20CustodyABI: custodyABI, + zetaConnectorAddress: zetaConnectorAddress, + er20CustodyAddress: erc20CustodyAddress, + coreContext: coreContext, + logger: logger, ts: ts, mu: &sync.Mutex{}, outTxHashBeingReported: make(map[string]bool), @@ -643,7 +644,7 @@ func (signer *Signer) SignWhitelistTx( func (signer *Signer) GetReportedTxList() *map[string]bool { return &signer.outTxHashBeingReported } -func (signer *Signer) EvmClient() interfaces.EVMRPCClient { +func (signer *Signer) EvmClient() interfaces.EthClientFallback { return signer.client } func (signer *Signer) EvmSigner() ethtypes.Signer { @@ -653,15 +654,15 @@ func (signer *Signer) EvmSigner() ethtypes.Signer { // ________________________ // getEVMRPC is a helper function to set up the client and signer, also initializes a mock client for unit tests -func getEVMRPC(endpoint string) (interfaces.EVMRPCClient, ethtypes.Signer, error) { - if endpoint == stub.EVMRPCEnabled { +func getEVMRPC(cfg config.EVMConfig, logger clientcommon.ClientLogger) (interfaces.EthClientFallback, ethtypes.Signer, error) { + if cfg.Endpoint[0] == stub.EVMRPCEnabled { chainID := big.NewInt(chains.BscMainnetChain().ChainId) ethSigner := ethtypes.NewLondonSigner(chainID) client := &stub.MockEvmClient{} return client, ethSigner, nil } - client, err := ethclient.Dial(endpoint) + client, err := NewEthClientFallback(cfg, logger.Std) if err != nil { return nil, nil, err } diff --git a/zetaclient/interfaces/interfaces.go b/zetaclient/interfaces/interfaces.go index 1c01b7e486..2992ae283a 100644 --- a/zetaclient/interfaces/interfaces.go +++ b/zetaclient/interfaces/interfaces.go @@ -151,3 +151,10 @@ type EVMJSONRPCClient interface { EthGetBlockByNumber(number int, withTransactions bool) (*ethrpc.Block, error) EthGetTransactionByHash(hash string) (*ethrpc.Transaction, error) } + +// EthClientFallback consolidates interfaces to external chain clients +type EthClientFallback interface { + // EVMRPCClient EVMJSONRPCClient - Need to implement both interfaces to support newer data types + EVMRPCClient + EVMJSONRPCClient +} diff --git a/zetaclient/supplychecker/zeta_supply_checker.go b/zetaclient/supplychecker/zeta_supply_checker.go index 14b4c056d2..a30ca50a4e 100644 --- a/zetaclient/supplychecker/zeta_supply_checker.go +++ b/zetaclient/supplychecker/zeta_supply_checker.go @@ -54,7 +54,8 @@ func NewZetaSupplyChecker(appContext *appcontext.AppContext, zetaClient *zetabri if evmConfig.Chain.IsZetaChain() { continue } - client, err := ethclient.Dial(evmConfig.Endpoint) + // This function doesn't seem to be used, defaulting to first endpoint + client, err := ethclient.Dial(evmConfig.Endpoint[0]) if err != nil { return zetaSupplyChecker, err } diff --git a/zetaclient/testutils/stub/evm_rpc.go b/zetaclient/testutils/stub/evm_rpc.go index dd1fd5fedf..52afaa204e 100644 --- a/zetaclient/testutils/stub/evm_rpc.go +++ b/zetaclient/testutils/stub/evm_rpc.go @@ -4,10 +4,12 @@ import ( "errors" "math/big" + "github.com/onrik/ethrpc" + "github.com/zeta-chain/zetacore/zetaclient/interfaces" + "github.com/ethereum/go-ethereum" ethcommon "github.com/ethereum/go-ethereum/common" ethtypes "github.com/ethereum/go-ethereum/core/types" - "github.com/zeta-chain/zetacore/zetaclient/interfaces" "golang.org/x/net/context" ) @@ -27,12 +29,20 @@ func (s subscription) Err() <-chan error { } // EvmClient interface -var _ interfaces.EVMRPCClient = &MockEvmClient{} +var _ interfaces.EthClientFallback = &MockEvmClient{} type MockEvmClient struct { Receipts []*ethtypes.Receipt } +func (e *MockEvmClient) EthGetBlockByNumber(_ int, _ bool) (*ethrpc.Block, error) { + return nil, nil +} + +func (e *MockEvmClient) EthGetTransactionByHash(_ string) (*ethrpc.Transaction, error) { + return nil, nil +} + func NewMockEvmClient() *MockEvmClient { client := &MockEvmClient{} return client.Reset()