From 6de5bc85c8b7c298e285602b57a4d2f8ff2d21cd Mon Sep 17 00:00:00 2001 From: Lucas Bertrand Date: Fri, 19 Jan 2024 11:43:28 -0800 Subject: [PATCH 01/10] refactor(`hotfix`): support retrying sending inbound vote with higher gas limit (#1601) * chore: v12.0.0 changelog (#1578) * fix: skip unsupported chain parameters (#1581) Co-authored-by: Lucas Bertrand * remove debug zetaclient1 * remove comments * tx resend * complete renaming * change log level and create constants * increase retry value * test change gas limit for outbound * refactor vote inbound outbound into separate files * add monitoring capability for outbound txs * changelog --------- Co-authored-by: Charlie Chen <34498985+ws4charlie@users.noreply.github.com> --- changelog.md | 5 + .../scripts/start-zetaclientd-genesis.sh | 2 +- contrib/localnet/scripts/start-zetaclientd.sh | 2 +- zetaclient/bitcoin_client.go | 6 +- zetaclient/broadcast.go | 8 +- zetaclient/evm_client.go | 32 +-- zetaclient/inbound_tracker.go | 16 +- zetaclient/interfaces.go | 4 +- zetaclient/tx.go | 202 ++---------------- zetaclient/tx_vote_inbound.go | 153 +++++++++++++ zetaclient/tx_vote_outbound.go | 154 +++++++++++++ zetaclient/voter_test.go | 12 +- zetaclient/zetacore_bridge.go | 15 +- zetaclient/zetacore_observer_test.go | 8 +- 14 files changed, 384 insertions(+), 235 deletions(-) create mode 100644 zetaclient/tx_vote_inbound.go create mode 100644 zetaclient/tx_vote_outbound.go diff --git a/changelog.md b/changelog.md index 3c8acacc3c..67f62a348a 100644 --- a/changelog.md +++ b/changelog.md @@ -2,6 +2,10 @@ ## Unreleased +### Features + +* [1591](https://github.com/zeta-chain/node/pull/1591) - support lower gas limit for voting on inbound and outbound transactions + ## Version: v12.0.0 ### Breaking Changes @@ -35,6 +39,7 @@ Getting the correct TSS address for Bitcoin now requires proviidng the Bitcoin c ### Fixes +* [1575](https://github.com/zeta-chain/node/issues/1575) - Skip unsupported chain parameters by IsSupported flag * [1554](https://github.com/zeta-chain/node/pull/1554) - Screen out unconfirmed UTXOs that are not created by TSS itself * [1560](https://github.com/zeta-chain/node/issues/1560) - Zetaclient post evm-chain outtx hashes only when receipt is available * [1516](https://github.com/zeta-chain/node/issues/1516) - Unprivileged outtx tracker removal diff --git a/contrib/localnet/scripts/start-zetaclientd-genesis.sh b/contrib/localnet/scripts/start-zetaclientd-genesis.sh index 7c23cd0673..5f2cadceab 100755 --- a/contrib/localnet/scripts/start-zetaclientd-genesis.sh +++ b/contrib/localnet/scripts/start-zetaclientd-genesis.sh @@ -36,6 +36,6 @@ else SEED=$(curl --retry 10 --retry-delay 5 --retry-connrefused -s zetaclient0:8123/p2p) done rm ~/.tss/* - zetaclientd init --peer /ip4/172.20.0.21/tcp/6668/p2p/"$SEED" --zetacore-url "$node" --chain-id athens_101-1 --operator "$operatorAddress" --log-format=text --public-ip "$MYIP" --log-level 0 --keyring-backend "$BACKEND" + zetaclientd init --peer /ip4/172.20.0.21/tcp/6668/p2p/"$SEED" --zetacore-url "$node" --chain-id athens_101-1 --operator "$operatorAddress" --log-format=text --public-ip "$MYIP" --log-level 1 --keyring-backend "$BACKEND" zetaclientd start fi diff --git a/contrib/localnet/scripts/start-zetaclientd.sh b/contrib/localnet/scripts/start-zetaclientd.sh index 14de0fada6..eecb8b8e5d 100644 --- a/contrib/localnet/scripts/start-zetaclientd.sh +++ b/contrib/localnet/scripts/start-zetaclientd.sh @@ -23,6 +23,6 @@ else zetaclientd init \ --peer /ip4/172.20.0.21/tcp/6668/p2p/$SEED \ --pre-params ~/preParams.json --zetacore-url $node \ - --chain-id athens_101-1 --operator zeta1lz2fqwzjnk6qy48fgj753h48444fxtt7hekp52 --log-level 0 --hotkey=val_grantee_observer + --chain-id athens_101-1 --operator zeta1lz2fqwzjnk6qy48fgj753h48444fxtt7hekp52 --log-level 1 --hotkey=val_grantee_observer zetaclientd start fi diff --git a/zetaclient/bitcoin_client.go b/zetaclient/bitcoin_client.go index a316e3b4ac..2c91707f32 100644 --- a/zetaclient/bitcoin_client.go +++ b/zetaclient/bitcoin_client.go @@ -385,12 +385,12 @@ func (ob *BitcoinChainClient) observeInTx() error { // post inbound vote message to zetacore for _, inTx := range inTxs { msg := ob.GetInboundVoteMessageFromBtcEvent(inTx) - zetaHash, ballot, err := ob.zetaClient.PostSend(PostSendEVMGasLimit, msg) + zetaHash, ballot, err := ob.zetaClient.PostVoteInbound(PostVoteInboundGasLimit, PostVoteInboundExecutionGasLimit, msg) if err != nil { ob.logger.WatchInTx.Error().Err(err).Msgf("observeInTxBTC: error posting to zeta core for tx %s", inTx.TxHash) return err // we have to re-scan this block next time } else if zetaHash != "" { - ob.logger.WatchInTx.Info().Msgf("observeInTxBTC: BTC deposit detected and reported: PostSend zeta tx: %s ballot %s", zetaHash, ballot) + ob.logger.WatchInTx.Info().Msgf("observeInTxBTC: BTC deposit detected and reported: PostVoteInbound zeta tx: %s ballot %s", zetaHash, ballot) } } @@ -470,7 +470,7 @@ func (ob *BitcoinChainClient) IsSendOutTxProcessed(sendHash string, nonce uint64 } logger.Debug().Msgf("Bitcoin outTx confirmed: txid %s, amount %s\n", res.TxID, amountInSat.String()) - zetaHash, ballot, err := ob.zetaClient.PostReceiveConfirmation( + zetaHash, ballot, err := ob.zetaClient.PostVoteOutbound( sendHash, res.TxID, // #nosec G701 always positive diff --git a/zetaclient/broadcast.go b/zetaclient/broadcast.go index f015f2df85..c4fee8c1c9 100644 --- a/zetaclient/broadcast.go +++ b/zetaclient/broadcast.go @@ -53,7 +53,6 @@ func (b *ZetaCoreBridge) Broadcast(gaslimit uint64, authzWrappedMsg sdktypes.Msg b.seqNumber[authzSigner.KeyType] = seqNumber } } - //b.logger.Info().Uint64("account_number", b.accountNumber).Uint64("sequence_number", b.seqNumber).Msg("account info") flags := flag.NewFlagSet("zetacore", 0) @@ -69,12 +68,13 @@ func (b *ZetaCoreBridge) Broadcast(gaslimit uint64, authzWrappedMsg sdktypes.Msg if err != nil { return "", err } + builder.SetGasLimit(gaslimit) + // #nosec G701 always in range fee := sdktypes.NewCoins(sdktypes.NewCoin(config.BaseDenom, cosmos.NewInt(int64(gaslimit)).Mul(cosmos.NewInt(baseGasPrice)))) builder.SetFeeAmount(fee) - //fmt.Printf("signing from name: %s\n", ctx.GetFromName()) err = b.SignTx(factory, ctx.GetFromName(), builder, true, ctx.TxConfig) if err != nil { return "", err @@ -90,6 +90,7 @@ func (b *ZetaCoreBridge) Broadcast(gaslimit uint64, authzWrappedMsg sdktypes.Msg b.logger.Error().Err(err).Msgf("fail to broadcast tx %s", err.Error()) return "", err } + // Code will be the tendermint ABICode , it start at 1 , so if it is an error , code will not be zero if commit.Code > 0 { if commit.Code == 32 { @@ -114,11 +115,8 @@ func (b *ZetaCoreBridge) Broadcast(gaslimit uint64, authzWrappedMsg sdktypes.Msg } return commit.TxHash, fmt.Errorf("fail to broadcast to zetachain,code:%d, log:%s", commit.Code, commit.RawLog) } - //b.logger.Debug().Msgf("Received a TxHash of %v from the metachain, Code %d, log %s", commit.TxHash, commit.Code, commit.Logs) // increment seqNum - //seq := b.seqNumber[authzSigner.KeyType] - //atomic.AddUint64(&seq, 1) b.seqNumber[authzSigner.KeyType] = b.seqNumber[authzSigner.KeyType] + 1 return commit.TxHash, nil diff --git a/zetaclient/evm_client.go b/zetaclient/evm_client.go index 8a68ac121e..263e40a251 100644 --- a/zetaclient/evm_client.go +++ b/zetaclient/evm_client.go @@ -318,7 +318,7 @@ func (ob *EVMChainClient) IsSendOutTxProcessed(sendHash string, nonce uint64, co if receipt.Status == 1 { recvStatus = common.ReceiveStatus_Success } - zetaTxHash, ballot, err := ob.zetaClient.PostReceiveConfirmation( + zetaTxHash, ballot, err := ob.zetaClient.PostVoteOutbound( sendHash, receipt.TxHash.Hex(), receipt.BlockNumber.Uint64(), @@ -340,7 +340,7 @@ func (ob *EVMChainClient) IsSendOutTxProcessed(sendHash string, nonce uint64, co } else if cointype == common.CoinType_Gas { // the outbound is a regular Ether/BNB/Matic transfer; no need to check events if receipt.Status == 1 { - zetaTxHash, ballot, err := ob.zetaClient.PostReceiveConfirmation( + zetaTxHash, ballot, err := ob.zetaClient.PostVoteOutbound( sendHash, receipt.TxHash.Hex(), receipt.BlockNumber.Uint64(), @@ -361,7 +361,7 @@ func (ob *EVMChainClient) IsSendOutTxProcessed(sendHash string, nonce uint64, co return true, true, nil } else if receipt.Status == 0 { // the same as below events flow logger.Info().Msgf("Found (failed tx) sendHash %s on chain %s txhash %s", sendHash, ob.chain.String(), receipt.TxHash.Hex()) - zetaTxHash, ballot, err := ob.zetaClient.PostReceiveConfirmation( + zetaTxHash, ballot, err := ob.zetaClient.PostVoteOutbound( sendHash, receipt.TxHash.Hex(), receipt.BlockNumber.Uint64(), @@ -375,7 +375,7 @@ func (ob *EVMChainClient) IsSendOutTxProcessed(sendHash string, nonce uint64, co common.CoinType_Gas, ) if err != nil { - logger.Error().Err(err).Msgf("PostReceiveConfirmation error in WatchTxHashWithTimeout; zeta tx hash %s cctx %s nonce %d", zetaTxHash, sendHash, nonce) + logger.Error().Err(err).Msgf("PostVoteOutbound error in WatchTxHashWithTimeout; zeta tx hash %s cctx %s nonce %d", zetaTxHash, sendHash, nonce) } else if zetaTxHash != "" { logger.Info().Msgf("Zeta tx hash: %s cctx %s nonce %d ballot %s", zetaTxHash, sendHash, nonce, ballot) } @@ -405,7 +405,7 @@ func (ob *EVMChainClient) IsSendOutTxProcessed(sendHash string, nonce uint64, co sendhash := vLog.Topics[3].Hex() //var rxAddress string = ethcommon.HexToAddress(vLog.Topics[1].Hex()).Hex() mMint := receivedLog.ZetaValue - zetaTxHash, ballot, err := ob.zetaClient.PostReceiveConfirmation( + zetaTxHash, ballot, err := ob.zetaClient.PostVoteOutbound( sendhash, vLog.TxHash.Hex(), vLog.BlockNumber, @@ -442,7 +442,7 @@ func (ob *EVMChainClient) IsSendOutTxProcessed(sendHash string, nonce uint64, co } sendhash := vLog.Topics[2].Hex() mMint := revertedLog.RemainingZetaValue - zetaTxHash, ballot, err := ob.zetaClient.PostReceiveConfirmation( + zetaTxHash, ballot, err := ob.zetaClient.PostVoteOutbound( sendhash, vLog.TxHash.Hex(), vLog.BlockNumber, @@ -470,7 +470,7 @@ func (ob *EVMChainClient) IsSendOutTxProcessed(sendHash string, nonce uint64, co } else if receipt.Status == 0 { //FIXME: check nonce here by getTransaction RPC logger.Info().Msgf("Found (failed tx) sendHash %s on chain %s txhash %s", sendHash, ob.chain.String(), receipt.TxHash.Hex()) - zetaTxHash, ballot, err := ob.zetaClient.PostReceiveConfirmation( + zetaTxHash, ballot, err := ob.zetaClient.PostVoteOutbound( sendHash, receipt.TxHash.Hex(), receipt.BlockNumber.Uint64(), @@ -510,7 +510,7 @@ func (ob *EVMChainClient) IsSendOutTxProcessed(sendHash string, nonce uint64, co } if confHeight <= ob.GetLastBlockHeight() { logger.Info().Msg("Confirmed! Sending PostConfirmation to zetacore...") - zetaTxHash, ballot, err := ob.zetaClient.PostReceiveConfirmation( + zetaTxHash, ballot, err := ob.zetaClient.PostVoteOutbound( sendHash, vLog.TxHash.Hex(), vLog.BlockNumber, @@ -537,7 +537,7 @@ func (ob *EVMChainClient) IsSendOutTxProcessed(sendHash string, nonce uint64, co } } else { logger.Info().Msgf("Found (failed tx) sendHash %s on chain %s txhash %s", sendHash, ob.chain.String(), receipt.TxHash.Hex()) - zetaTxHash, ballot, err := ob.zetaClient.PostReceiveConfirmation( + zetaTxHash, ballot, err := ob.zetaClient.PostVoteOutbound( sendHash, receipt.TxHash.Hex(), receipt.BlockNumber.Uint64(), @@ -551,7 +551,7 @@ func (ob *EVMChainClient) IsSendOutTxProcessed(sendHash string, nonce uint64, co common.CoinType_ERC20, ) if err != nil { - logger.Error().Err(err).Msgf("PostReceiveConfirmation error in WatchTxHashWithTimeout; zeta tx hash %s", zetaTxHash) + logger.Error().Err(err).Msgf("PostVoteOutbound error in WatchTxHashWithTimeout; zeta tx hash %s", zetaTxHash) } else if zetaTxHash != "" { logger.Info().Msgf("Zeta tx hash: %s cctx %s nonce %d ballot %s", zetaTxHash, sendHash, nonce, ballot) } @@ -971,7 +971,7 @@ func (ob *EVMChainClient) observeZetaSent(startBlock, toBlock uint64) uint64 { "observeZetaSent: error getting inbound vote msg for tx %s chain %d", event.Raw.TxHash.Hex(), ob.chain.ChainId) continue } - zetaHash, ballot, err := ob.zetaClient.PostSend(PostSendNonEVMGasLimit, &msg) + zetaHash, ballot, err := ob.zetaClient.PostVoteInbound(PostVoteInboundGasLimit, PostVoteInboundMessagePassingExecutionGasLimit, &msg) if err != nil { ob.logger.ExternalChainWatcher.Error().Err(err).Msgf( "observeZetaSent: error posting event to zeta core for tx %s at height %d for chain %d", @@ -979,7 +979,7 @@ func (ob *EVMChainClient) observeZetaSent(startBlock, toBlock uint64) uint64 { return beingScanned - 1 // we have to re-scan from this block next time } else if zetaHash != "" { ob.logger.ExternalChainWatcher.Info().Msgf( - "observeZetaSent: event detected in tx %s at height %d for chain %d, PostSend zeta tx: %s ballot %s", + "observeZetaSent: event detected in tx %s at height %d for chain %d, PostVoteInbound zeta tx: %s ballot %s", event.Raw.TxHash.Hex(), event.Raw.BlockNumber, ob.chain.ChainId, zetaHash, ballot) } } @@ -1050,7 +1050,7 @@ func (ob *EVMChainClient) observeERC20Deposited(startBlock, toBlock uint64) uint "observeERC20Deposited: error getting inbound vote msg for tx %s chain %d", event.Raw.TxHash.Hex(), ob.chain.ChainId) continue } - zetaHash, ballot, err := ob.zetaClient.PostSend(PostSendEVMGasLimit, &msg) + zetaHash, ballot, err := ob.zetaClient.PostVoteInbound(PostVoteInboundGasLimit, PostVoteInboundExecutionGasLimit, &msg) if err != nil { ob.logger.ExternalChainWatcher.Error().Err(err).Msgf( "observeERC20Deposited: error posting event to zeta core for tx %s at height %d for chain %d", @@ -1058,7 +1058,7 @@ func (ob *EVMChainClient) observeERC20Deposited(startBlock, toBlock uint64) uint return beingScanned - 1 // we have to re-scan from this block next time } else if zetaHash != "" { ob.logger.ExternalChainWatcher.Info().Msgf( - "observeERC20Deposited: event detected in tx %s at height %d for chain %d, PostSend zeta tx: %s ballot %s", + "observeERC20Deposited: event detected in tx %s at height %d for chain %d, PostVoteInbound zeta tx: %s ballot %s", event.Raw.TxHash.Hex(), event.Raw.BlockNumber, ob.chain.ChainId, zetaHash, ballot) } } @@ -1133,14 +1133,14 @@ func (ob *EVMChainClient) observeTssRecvd(startBlock, toBlock uint64, flags obse if msg == nil { continue } - zetaHash, ballot, err := ob.zetaClient.PostSend(PostSendEVMGasLimit, msg) + zetaHash, ballot, err := ob.zetaClient.PostVoteInbound(PostVoteInboundGasLimit, PostVoteInboundExecutionGasLimit, msg) if err != nil { ob.logger.ExternalChainWatcher.Error().Err(err).Msgf( "observeTssRecvd: error posting to zeta core for tx %s at height %d for chain %d", tx.Hash().Hex(), bn, ob.chain.ChainId) return startBlock - 1 // we have to re-scan this block next time } else if zetaHash != "" { ob.logger.ExternalChainWatcher.Info().Msgf( - "observeTssRecvd: gas asset deposit detected in tx %s at height %d for chain %d, PostSend zeta tx: %s ballot %s", + "observeTssRecvd: gas asset deposit detected in tx %s at height %d for chain %d, PostVoteInbound zeta tx: %s ballot %s", tx.Hash().Hex(), bn, ob.chain.ChainId, zetaHash, ballot) } } diff --git a/zetaclient/inbound_tracker.go b/zetaclient/inbound_tracker.go index 8a7ebd1147..6137f9f20a 100644 --- a/zetaclient/inbound_tracker.go +++ b/zetaclient/inbound_tracker.go @@ -114,12 +114,12 @@ func (ob *BitcoinChainClient) CheckReceiptForBtcTxHash(txHash string, vote bool) if !vote { return msg.Digest(), nil } - zetaHash, ballot, err := ob.zetaClient.PostSend(PostSendEVMGasLimit, msg) + zetaHash, ballot, err := ob.zetaClient.PostVoteInbound(PostVoteInboundGasLimit, PostVoteInboundExecutionGasLimit, msg) if err != nil { ob.logger.WatchInTx.Error().Err(err).Msg("error posting to zeta core") return "", err } else if ballot == "" { - ob.logger.WatchInTx.Info().Msgf("BTC deposit detected and reported: PostSend zeta tx: %s ballot %s", zetaHash, ballot) + ob.logger.WatchInTx.Info().Msgf("BTC deposit detected and reported: PostVoteInbound zeta tx: %s ballot %s", zetaHash, ballot) } return msg.Digest(), nil } @@ -192,12 +192,12 @@ func (ob *EVMChainClient) CheckReceiptForCoinTypeZeta(txHash string, vote bool) return msg.Digest(), nil } - zetaHash, ballot, err := ob.zetaClient.PostSend(PostSendNonEVMGasLimit, &msg) + zetaHash, ballot, err := ob.zetaClient.PostVoteInbound(PostVoteInboundGasLimit, PostVoteInboundMessagePassingExecutionGasLimit, &msg) if err != nil { ob.logger.ExternalChainWatcher.Error().Err(err).Msg("error posting to zeta core") return "", err } else if zetaHash != "" { - ob.logger.ExternalChainWatcher.Info().Msgf("ZetaSent event detected and reported: PostSend zeta tx: %s ballot %s", zetaHash, ballot) + ob.logger.ExternalChainWatcher.Info().Msgf("ZetaSent event detected and reported: PostVoteInbound zeta tx: %s ballot %s", zetaHash, ballot) } return msg.Digest(), nil @@ -240,12 +240,12 @@ func (ob *EVMChainClient) CheckReceiptForCoinTypeERC20(txHash string, vote bool) return msg.Digest(), nil } - zetaHash, ballot, err := ob.zetaClient.PostSend(PostSendEVMGasLimit, &msg) + zetaHash, ballot, err := ob.zetaClient.PostVoteInbound(PostVoteInboundGasLimit, PostVoteInboundExecutionGasLimit, &msg) if err != nil { ob.logger.ExternalChainWatcher.Error().Err(err).Msg("error posting to zeta core") return "", err } else if zetaHash != "" { - ob.logger.ExternalChainWatcher.Info().Msgf("ERC20 Deposit event detected and reported: PostSend zeta tx: %s ballot %s", zetaHash, ballot) + ob.logger.ExternalChainWatcher.Info().Msgf("ERC20 Deposit event detected and reported: PostVoteInbound zeta tx: %s ballot %s", zetaHash, ballot) } return msg.Digest(), nil @@ -297,12 +297,12 @@ func (ob *EVMChainClient) CheckReceiptForCoinTypeGas(txHash string, vote bool) ( return msg.Digest(), nil } - zetaHash, ballot, err := ob.zetaClient.PostSend(PostSendEVMGasLimit, msg) + zetaHash, ballot, err := ob.zetaClient.PostVoteInbound(PostVoteInboundGasLimit, PostVoteInboundExecutionGasLimit, msg) if err != nil { ob.logger.ExternalChainWatcher.Error().Err(err).Msg("error posting to zeta core") return "", err } else if zetaHash != "" { - ob.logger.ExternalChainWatcher.Info().Msgf("Gas deposit detected and reported: PostSend zeta tx: %s ballot %s", zetaHash, ballot) + ob.logger.ExternalChainWatcher.Info().Msgf("Gas deposit detected and reported: PostVoteInbound zeta tx: %s ballot %s", zetaHash, ballot) } return msg.Digest(), nil diff --git a/zetaclient/interfaces.go b/zetaclient/interfaces.go index fe2d0824c1..6ca41f59fd 100644 --- a/zetaclient/interfaces.go +++ b/zetaclient/interfaces.go @@ -48,8 +48,8 @@ type ChainSigner interface { // ZetaCoreBridger is the interface to interact with ZetaCore type ZetaCoreBridger interface { - PostSend(zetaGasLimit uint64, msg *crosschaintypes.MsgVoteOnObservedInboundTx) (string, string, error) - PostReceiveConfirmation( + PostVoteInbound(gasLimit, retryGasLimit uint64, msg *crosschaintypes.MsgVoteOnObservedInboundTx) (string, string, error) + PostVoteOutbound( sendHash string, outTxHash string, outBlockHeight uint64, diff --git a/zetaclient/tx.go b/zetaclient/tx.go index 38a02052ed..15847c3f80 100644 --- a/zetaclient/tx.go +++ b/zetaclient/tx.go @@ -2,15 +2,10 @@ package zetaclient import ( "fmt" - "math/big" - "strings" "time" - "cosmossdk.io/math" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/authz" - "github.com/pkg/errors" - "github.com/zeta-chain/go-tss/blame" "github.com/zeta-chain/zetacore/common" "github.com/zeta-chain/zetacore/x/crosschain/types" @@ -19,55 +14,27 @@ import ( ) const ( - PostGasPriceGasLimit = 1_500_000 + // DefaultGasLimit is the default gas limit used for broadcasting txs + DefaultGasLimit = 200_000 + + // PostGasPriceGasLimit is the gas limit for voting new gas price + PostGasPriceGasLimit = 1_500_000 + + // AddTxHashToOutTxTrackerGasLimit is the gas limit for adding tx hash to out tx tracker AddTxHashToOutTxTrackerGasLimit = 200_000 - PostNonceGasLimit = 200_000 - PostSendEVMGasLimit = 4_000_000 // likely emit a lot of logs, so costly - PostSendNonEVMGasLimit = 1_000_000 - PostReceiveConfirmationGasLimit = 400_000 - PostBlameDataGasLimit = 200_000 - DefaultGasLimit = 200_000 - PostProveOutboundTxGasLimit = 400_000 - DefaultRetryCount = 5 - ExtendedRetryCount = 15 - DefaultRetryInterval = 5 -) -// GetInBoundVoteMessage returns a new MsgVoteOnObservedInboundTx -func GetInBoundVoteMessage( - sender string, - senderChain int64, - txOrigin string, - receiver string, - receiverChain int64, - amount math.Uint, - message string, - inTxHash string, - inBlockHeight uint64, - gasLimit uint64, - coinType common.CoinType, - asset string, - signerAddress string, - eventIndex uint, -) *types.MsgVoteOnObservedInboundTx { - msg := types.NewMsgVoteOnObservedInboundTx( - signerAddress, - sender, - senderChain, - txOrigin, - receiver, - receiverChain, - amount, - message, - inTxHash, - inBlockHeight, - gasLimit, - coinType, - asset, - eventIndex, - ) - return msg -} + // PostBlameDataGasLimit is the gas limit for voting on blames + PostBlameDataGasLimit = 200_000 + + // DefaultRetryCount is the number of retries for broadcasting a tx + DefaultRetryCount = 5 + + // ExtendedRetryCount is an extended number of retries for broadcasting a tx, used in keygen operations + ExtendedRetryCount = 15 + + // DefaultRetryInterval is the interval between retries in seconds + DefaultRetryInterval = 5 +) func (b *ZetaCoreBridge) WrapMessageWithAuthz(msg sdk.Msg) (sdk.Msg, AuthZSigner, error) { msgURL := sdk.MsgTypeURL(msg) @@ -126,100 +93,6 @@ func (b *ZetaCoreBridge) AddTxHashToOutTxTracker( return zetaTxHash, nil } -func (b *ZetaCoreBridge) PostSend(zetaGasLimit uint64, msg *types.MsgVoteOnObservedInboundTx) (string, string, error) { - authzMsg, authzSigner, err := b.WrapMessageWithAuthz(msg) - if err != nil { - return "", "", err - } - - // don't post send if has already voted before - ballotIndex := msg.Digest() - hasVoted, err := b.HasVoted(ballotIndex, msg.Creator) - if err != nil { - return "", ballotIndex, errors.Wrapf(err, "PostSend: unable to check if already voted for ballot %s voter %s", ballotIndex, msg.Creator) - } - if hasVoted { - return "", ballotIndex, nil - } - - for i := 0; i < DefaultRetryCount; i++ { - zetaTxHash, err := b.Broadcast(zetaGasLimit, authzMsg, authzSigner) - if err == nil { - // monitor the result of the transaction - go b.MonitorTxResult(zetaTxHash, true) - - return zetaTxHash, ballotIndex, nil - } - b.logger.Debug().Err(err).Msgf("PostSend broadcast fail | Retry count : %d", i+1) - time.Sleep(DefaultRetryInterval * time.Second) - } - return "", ballotIndex, fmt.Errorf("post send failed after %d retries", DefaultRetryInterval) -} - -func (b *ZetaCoreBridge) PostReceiveConfirmation( - sendHash string, - outTxHash string, - outBlockHeight uint64, - outTxGasUsed uint64, - outTxEffectiveGasPrice *big.Int, - outTxEffectiveGasLimit uint64, - amount *big.Int, - status common.ReceiveStatus, - chain common.Chain, - nonce uint64, - coinType common.CoinType, -) (string, string, error) { - signerAddress := b.keys.GetOperatorAddress().String() - msg := types.NewMsgVoteOnObservedOutboundTx( - signerAddress, - sendHash, - outTxHash, - outBlockHeight, - outTxGasUsed, - math.NewIntFromBigInt(outTxEffectiveGasPrice), - outTxEffectiveGasLimit, - math.NewUintFromBigInt(amount), - status, - chain.ChainId, - nonce, - coinType, - ) - - authzMsg, authzSigner, err := b.WrapMessageWithAuthz(msg) - if err != nil { - return "", "", err - } - - // don't post confirmation if has already voted before - ballotIndex := msg.Digest() - hasVoted, err := b.HasVoted(ballotIndex, msg.Creator) - if err != nil { - return "", ballotIndex, errors.Wrapf(err, "PostReceiveConfirmation: unable to check if already voted for ballot %s voter %s", ballotIndex, msg.Creator) - } - if hasVoted { - return "", ballotIndex, nil - } - - // FIXME: remove this gas limit stuff; in the special ante handler with no gas limit, add - // NewMsgReceiveConfirmation to it. - var gasLimit uint64 = PostReceiveConfirmationGasLimit - if status == common.ReceiveStatus_Failed { - gasLimit = PostSendEVMGasLimit - } - for i := 0; i < DefaultRetryCount; i++ { - zetaTxHash, err := b.Broadcast(gasLimit, authzMsg, authzSigner) - if err == nil { - // monitor the result of the transaction - go b.MonitorTxResult(zetaTxHash, true) - - return zetaTxHash, ballotIndex, nil - } - b.logger.Debug().Err(err).Msgf("PostReceive broadcast fail | Retry count : %d", i+1) - time.Sleep(DefaultRetryInterval * time.Second) - } - return "", ballotIndex, fmt.Errorf("post receive failed after %d retries", DefaultRetryCount) -} - func (b *ZetaCoreBridge) SetTSS(tssPubkey string, keyGenZetaHeight int64, status common.ReceiveStatus) (string, error) { signerAddress := b.keys.GetOperatorAddress().String() msg := types.NewMsgCreateTSSVoter(signerAddress, tssPubkey, keyGenZetaHeight, status) @@ -308,40 +181,3 @@ func (b *ZetaCoreBridge) PostAddBlockHeader(chainID int64, blockHash []byte, hei } return "", fmt.Errorf("post add block header failed after %d retries", DefaultRetryCount) } - -// MonitorTxResult monitors the result of a tx (used for inbound and outbound vote txs) -func (b *ZetaCoreBridge) MonitorTxResult(zetaTxHash string, isInbound bool) { - var lastErr error - ticker := 5 * time.Second - retry := 10 - prefix := "MonitorOutboundTxResult" - if isInbound { - prefix = "MonitorInboundTxResult" - } - - for i := 0; i < retry; i++ { - time.Sleep(ticker) - txResult, err := b.QueryTxResult(zetaTxHash) - if err == nil { - // the inbound vote tx shouldn't fail to execute - if strings.Contains(txResult.RawLog, "failed to execute message") { - b.logger.Error().Msgf( - "%s: failed to execute vote, txHash: %s, txResult %s", - prefix, - zetaTxHash, - txResult.String(), - ) - } - return - } - lastErr = err - } - - b.logger.Error().Err(lastErr).Msgf( - "%s: unable to query tx result for txHash %s, err %s", - prefix, - zetaTxHash, - lastErr.Error(), - ) - return -} diff --git a/zetaclient/tx_vote_inbound.go b/zetaclient/tx_vote_inbound.go new file mode 100644 index 0000000000..1e5807474f --- /dev/null +++ b/zetaclient/tx_vote_inbound.go @@ -0,0 +1,153 @@ +package zetaclient + +import ( + "fmt" + "strings" + "time" + + "cosmossdk.io/math" + "github.com/zeta-chain/zetacore/common" + + "github.com/pkg/errors" + + "github.com/zeta-chain/zetacore/x/crosschain/types" +) + +const ( + // PostVoteInboundGasLimit is the gas limit for voting on observed inbound tx + PostVoteInboundGasLimit = 400_000 + + // PostVoteInboundExecutionGasLimit is the gas limit for voting on observed inbound tx and executing it + PostVoteInboundExecutionGasLimit = 4_000_000 + + // PostVoteInboundMessagePassingExecutionGasLimit is the gas limit for voting on, and executing ,observed inbound tx related to message passing (coin_type == zeta) + PostVoteInboundMessagePassingExecutionGasLimit = 1_000_000 + + // MonitorVoteInboundTxResultInterval is the interval between retries for monitoring tx result in seconds + MonitorVoteInboundTxResultInterval = 5 + + // MonitorVoteInboundTxResultRetryCount is the number of retries to fetch monitoring tx result + MonitorVoteInboundTxResultRetryCount = 20 +) + +// GetInBoundVoteMessage returns a new MsgVoteOnObservedInboundTx +func GetInBoundVoteMessage( + sender string, + senderChain int64, + txOrigin string, + receiver string, + receiverChain int64, + amount math.Uint, + message string, + inTxHash string, + inBlockHeight uint64, + gasLimit uint64, + coinType common.CoinType, + asset string, + signerAddress string, + eventIndex uint, +) *types.MsgVoteOnObservedInboundTx { + msg := types.NewMsgVoteOnObservedInboundTx( + signerAddress, + sender, + senderChain, + txOrigin, + receiver, + receiverChain, + amount, + message, + inTxHash, + inBlockHeight, + gasLimit, + coinType, + asset, + eventIndex, + ) + return msg +} + +// PostVoteInbound posts a vote on an observed inbound tx +// retryGasLimit is the gas limit used to resend the tx if it fails because of insufficient gas +// it is used when the ballot is finalized and the inbound tx needs to be processed +func (b *ZetaCoreBridge) PostVoteInbound(gasLimit, retryGasLimit uint64, msg *types.MsgVoteOnObservedInboundTx) (string, string, error) { + authzMsg, authzSigner, err := b.WrapMessageWithAuthz(msg) + if err != nil { + return "", "", err + } + + // don't post send if has already voted before + ballotIndex := msg.Digest() + hasVoted, err := b.HasVoted(ballotIndex, msg.Creator) + if err != nil { + return "", ballotIndex, errors.Wrapf(err, "PostVoteInbound: unable to check if already voted for ballot %s voter %s", ballotIndex, msg.Creator) + } + if hasVoted { + return "", ballotIndex, nil + } + + for i := 0; i < DefaultRetryCount; i++ { + zetaTxHash, err := b.Broadcast(gasLimit, authzMsg, authzSigner) + if err == nil { + // monitor the result of the transaction and resend if necessary + go b.MonitorVoteInboundTxResult(zetaTxHash, retryGasLimit, msg) + + return zetaTxHash, ballotIndex, nil + } + b.logger.Debug().Err(err).Msgf("PostVoteInbound broadcast fail | Retry count : %d", i+1) + time.Sleep(DefaultRetryInterval * time.Second) + } + return "", ballotIndex, fmt.Errorf("post send failed after %d retries", DefaultRetryInterval) +} + +// MonitorVoteInboundTxResult monitors the result of a vote inbound tx +// retryGasLimit is the gas limit used to resend the tx if it fails because of insufficient gas +// if retryGasLimit is 0, the tx is not resent +func (b *ZetaCoreBridge) MonitorVoteInboundTxResult(zetaTxHash string, retryGasLimit uint64, msg *types.MsgVoteOnObservedInboundTx) { + var lastErr error + + for i := 0; i < MonitorVoteInboundTxResultRetryCount; i++ { + time.Sleep(MonitorVoteInboundTxResultInterval * time.Second) + + // query tx result from ZetaChain + txResult, err := b.QueryTxResult(zetaTxHash) + + if err == nil { + if strings.Contains(txResult.RawLog, "failed to execute message") { + // the inbound vote tx shouldn't fail to execute + // this shouldn't happen + b.logger.Error().Msgf( + "MonitorInboundTxResult: failed to execute vote, txHash: %s, log %s", zetaTxHash, txResult.RawLog, + ) + } else if strings.Contains(txResult.RawLog, "out of gas") { + // if the tx fails with an out of gas error, resend the tx with more gas if retryGasLimit > 0 + b.logger.Debug().Msgf( + "MonitorInboundTxResult: out of gas, txHash: %s, log %s", zetaTxHash, txResult.RawLog, + ) + if retryGasLimit > 0 { + // new retryGasLimit set to 0 to prevent reentering this function + _, _, err := b.PostVoteInbound(retryGasLimit, 0, msg) + if err != nil { + b.logger.Error().Err(err).Msgf( + "MonitorInboundTxResult: failed to resend tx, txHash: %s, log %s", zetaTxHash, txResult.RawLog, + ) + } else { + b.logger.Info().Msgf( + "MonitorInboundTxResult: successfully resent tx, txHash: %s, log %s", zetaTxHash, txResult.RawLog, + ) + } + } + } else { + b.logger.Debug().Msgf( + "MonitorInboundTxResult: successful txHash %s, log %s", zetaTxHash, txResult.RawLog, + ) + } + return + } + lastErr = err + } + + b.logger.Error().Err(lastErr).Msgf( + "MonitorInboundTxResult: unable to query tx result for txHash %s, err %s", zetaTxHash, lastErr.Error(), + ) + return +} diff --git a/zetaclient/tx_vote_outbound.go b/zetaclient/tx_vote_outbound.go new file mode 100644 index 0000000000..bcf79c69f6 --- /dev/null +++ b/zetaclient/tx_vote_outbound.go @@ -0,0 +1,154 @@ +package zetaclient + +import ( + "fmt" + "math/big" + "strings" + "time" + + "cosmossdk.io/math" + "github.com/pkg/errors" + "github.com/zeta-chain/zetacore/common" + "github.com/zeta-chain/zetacore/x/crosschain/types" +) + +const ( + // PostVoteOutboundGasLimit is the gas limit for voting on observed outbound tx + PostVoteOutboundGasLimit = 400_000 + + // PostVoteOutboundRevertGasLimit is the gas limit for voting on observed outbound tx for revert (when outbound fails) + // The value needs to be higher because reverting implies interacting with the EVM to perform swaps for the gas token + PostVoteOutboundRevertGasLimit = 1_500_000 + + // MonitorVoteOutboundTxResultInterval is the interval between retries for monitoring tx result in seconds + MonitorVoteOutboundTxResultInterval = 5 + + // MonitorVoteOutboundTxResultRetryCount is the number of retries to fetch monitoring tx result + MonitorVoteOutboundTxResultRetryCount = 20 +) + +// PostVoteOutbound posts a vote on an observed outbound tx +func (b *ZetaCoreBridge) PostVoteOutbound( + sendHash string, + outTxHash string, + outBlockHeight uint64, + outTxGasUsed uint64, + outTxEffectiveGasPrice *big.Int, + outTxEffectiveGasLimit uint64, + amount *big.Int, + status common.ReceiveStatus, + chain common.Chain, + nonce uint64, + coinType common.CoinType, +) (string, string, error) { + signerAddress := b.keys.GetOperatorAddress().String() + msg := types.NewMsgVoteOnObservedOutboundTx( + signerAddress, + sendHash, + outTxHash, + outBlockHeight, + outTxGasUsed, + math.NewIntFromBigInt(outTxEffectiveGasPrice), + outTxEffectiveGasLimit, + math.NewUintFromBigInt(amount), + status, + chain.ChainId, + nonce, + coinType, + ) + + // when an outbound fails and a revert is required, the gas limit needs to be higher + // this is because the revert tx needs to interact with the EVM to perform swaps for the gas token + // the higher gas limit is only necessary when the vote is finalized and the outbound is processed + // therefore we use a retryGasLimit with a higher value to resend the tx if it fails (when the vote is finalized) + retryGasLimit := uint64(0) + if msg.Status == common.ReceiveStatus_Failed { + retryGasLimit = PostVoteOutboundRevertGasLimit + } + + return b.PostVoteOutboundFromMsg(PostVoteOutboundGasLimit, retryGasLimit, msg) +} + +// PostVoteOutboundFromMsg posts a vote on an observed outbound tx from a MsgVoteOnObservedOutboundTx +func (b *ZetaCoreBridge) PostVoteOutboundFromMsg(gasLimit, retryGasLimit uint64, msg *types.MsgVoteOnObservedOutboundTx) (string, string, error) { + authzMsg, authzSigner, err := b.WrapMessageWithAuthz(msg) + if err != nil { + return "", "", err + } + + // don't post confirmation if has already voted before + ballotIndex := msg.Digest() + hasVoted, err := b.HasVoted(ballotIndex, msg.Creator) + if err != nil { + return "", ballotIndex, errors.Wrapf(err, "PostVoteOutbound: unable to check if already voted for ballot %s voter %s", ballotIndex, msg.Creator) + } + if hasVoted { + return "", ballotIndex, nil + } + for i := 0; i < DefaultRetryCount; i++ { + zetaTxHash, err := b.Broadcast(gasLimit, authzMsg, authzSigner) + if err == nil { + // monitor the result of the transaction and resend if necessary + go b.MonitorVoteOutboundTxResult(zetaTxHash, retryGasLimit, msg) + + return zetaTxHash, ballotIndex, nil + } + b.logger.Debug().Err(err).Msgf("PostVoteOutbound broadcast fail | Retry count : %d", i+1) + time.Sleep(DefaultRetryInterval * time.Second) + } + return "", ballotIndex, fmt.Errorf("post receive failed after %d retries", DefaultRetryCount) +} + +// MonitorVoteOutboundTxResult monitors the result of a vote outbound tx +// retryGasLimit is the gas limit used to resend the tx if it fails because of insufficient gas +// if retryGasLimit is 0, the tx is not resent +func (b *ZetaCoreBridge) MonitorVoteOutboundTxResult(zetaTxHash string, retryGasLimit uint64, msg *types.MsgVoteOnObservedOutboundTx) { + var lastErr error + + for i := 0; i < MonitorVoteOutboundTxResultRetryCount; i++ { + time.Sleep(MonitorVoteOutboundTxResultInterval * time.Second) + + // query tx result from ZetaChain + txResult, err := b.QueryTxResult(zetaTxHash) + + if err == nil { + if strings.Contains(txResult.RawLog, "failed to execute message") { + // the inbound vote tx shouldn't fail to execute + // this shouldn't happen + b.logger.Error().Msgf( + "MonitorVoteOutboundTxResult: failed to execute vote, txHash: %s, log %s", zetaTxHash, txResult.RawLog, + ) + } else if strings.Contains(txResult.RawLog, "out of gas") { + // if the tx fails with an out of gas error, resend the tx with more gas if retryGasLimit > 0 + b.logger.Debug().Msgf( + "MonitorVoteOutboundTxResult: out of gas, txHash: %s, log %s", zetaTxHash, txResult.RawLog, + ) + if retryGasLimit > 0 { + // new retryGasLimit set to 0 to prevent reentering this function + _, _, err := b.PostVoteOutboundFromMsg(retryGasLimit, 0, msg) + + if err != nil { + b.logger.Error().Err(err).Msgf( + "MonitorVoteOutboundTxResult: failed to resend tx, txHash: %s, log %s", zetaTxHash, txResult.RawLog, + ) + } else { + b.logger.Info().Msgf( + "MonitorVoteOutboundTxResult: successfully resent tx, txHash: %s, log %s", zetaTxHash, txResult.RawLog, + ) + } + } + } else { + b.logger.Debug().Msgf( + "MonitorVoteOutboundTxResult: successful txHash %s, log %s", zetaTxHash, txResult.RawLog, + ) + } + return + } + lastErr = err + } + + b.logger.Error().Err(lastErr).Msgf( + "MonitorVoteOutboundTxResult: unable to query tx result for txHash %s, err %s", zetaTxHash, lastErr.Error(), + ) + return +} diff --git a/zetaclient/voter_test.go b/zetaclient/voter_test.go index 5df01de806..853c17eea9 100644 --- a/zetaclient/voter_test.go +++ b/zetaclient/voter_test.go @@ -85,20 +85,20 @@ func (s *VoterSuite) SetUpTest(c *C) { func (s *VoterSuite) TestSendVoter(c *C) { b1 := s.bridge1 b2 := s.bridge2 - metaHash, err := b1.PostSend("0xfrom", "Ethereum", "0xfrom", "0xto", "BSC", "123456", "23245", "little message", + metaHash, err := b1.PostVoteInbound("0xfrom", "Ethereum", "0xfrom", "0xto", "BSC", "123456", "23245", "little message", "0xtxhash", 123123, "0xtoken") c.Assert(err, IsNil) - log.Info().Msgf("PostSend metaHash %s", metaHash) + log.Info().Msgf("PostVoteInbound metaHash %s", metaHash) // wait for the next block timer1 := time.NewTimer(2 * time.Second) <-timer1.C - metaHash, err = b2.PostSend("0xfrom", "Ethereum", "0xfrom", "0xto", "BSC", "123456", "23245", "little message", + metaHash, err = b2.PostVoteInbound("0xfrom", "Ethereum", "0xfrom", "0xto", "BSC", "123456", "23245", "little message", "0xtxhash", 123123, "0xtoken") c.Assert(err, IsNil) - log.Info().Msgf("Second PostSend metaHash %s", metaHash) + log.Info().Msgf("Second PostVoteInbound metaHash %s", metaHash) // wait for the next block timer2 := time.NewTimer(2 * time.Second) @@ -111,13 +111,13 @@ func (s *VoterSuite) TestSendVoter(c *C) { send := sends[0] - metaHash, err = b1.PostReceiveConfirmation(send.Index, "0xoutHash", 2123, "23245") + metaHash, err = b1.PostVoteOutbound(send.Index, "0xoutHash", 2123, "23245") c.Assert(err, IsNil) timer3 := time.NewTimer(2 * time.Second) <-timer3.C - metaHash, err = b2.PostReceiveConfirmation(send.Index, "0xoutHash", 2123, "23245") + metaHash, err = b2.PostVoteOutbound(send.Index, "0xoutHash", 2123, "23245") c.Assert(err, IsNil) receives, err := b2.GetAllReceive() diff --git a/zetaclient/zetacore_bridge.go b/zetaclient/zetacore_bridge.go index 0a2b3b08c4..bf4466565c 100644 --- a/zetaclient/zetacore_bridge.go +++ b/zetaclient/zetacore_bridge.go @@ -196,7 +196,8 @@ func (b *ZetaCoreBridge) UpdateConfigFromCore(cfg *config.Config, init bool) err return err } if plan != nil && bn == plan.Height-1 { // stop zetaclients; notify operator to upgrade and restart - b.logger.Warn().Msgf("Active upgrade plan detected and upgrade height reached: %s at height %d; ZetaClient is stopped; please kill this process, replace zetaclientd binary with upgraded version, and restart zetaclientd", plan.Name, plan.Height) + b.logger.Warn().Msgf("Active upgrade plan detected and upgrade height reached: %s at height %d; ZetaClient is stopped;"+ + "please kill this process, replace zetaclientd binary with upgraded version, and restart zetaclientd", plan.Name, plan.Height) b.pause <- struct{}{} // notify CoreObserver to stop ChainClients, Signers, and CoreObservder itself } @@ -212,14 +213,16 @@ func (b *ZetaCoreBridge) UpdateConfigFromCore(cfg *config.Config, init bool) err for _, chainParam := range chainParams { err := config.ValidateChainParams(chainParam) if err != nil { - b.logger.Debug().Err(err).Msgf( - "Invalid core params for chain %s", - common.GetChainFromChainID(chainParam.ChainId).ChainName, - ) + b.logger.Warn().Err(err).Msgf("Invalid chain params for chain %d", chainParam.ChainId) + continue + } + if !chainParam.GetIsSupported() { + b.logger.Info().Msgf("Chain %d is not supported yet", chainParam.ChainId) + continue } if common.IsBitcoinChain(chainParam.ChainId) { newBTCParams = chainParam - } else { + } else if common.IsEVMChain(chainParam.ChainId) { newEVMParams[chainParam.ChainId] = chainParam } } diff --git a/zetaclient/zetacore_observer_test.go b/zetaclient/zetacore_observer_test.go index afa8114764..919450d1cb 100644 --- a/zetaclient/zetacore_observer_test.go +++ b/zetaclient/zetacore_observer_test.go @@ -117,18 +117,18 @@ func (s *COSuite) SetUpTest(c *C) { func (s *COSuite) TestSendFlow(c *C) { b1 := s.bridge1 b2 := s.bridge2 - metaHash, err := b1.PostSend(TEST_SENDER, "Ethereum", TEST_SENDER, TEST_RECEIVER, "BSC", "1337", "0", "treat or trick", + metaHash, err := b1.PostVoteInbound(TEST_SENDER, "Ethereum", TEST_SENDER, TEST_RECEIVER, "BSC", "1337", "0", "treat or trick", "0xtxhash", 123123, "0xtoken") c.Assert(err, IsNil) - c.Logf("PostSend metaHash %s", metaHash) + c.Logf("PostVoteInbound metaHash %s", metaHash) timer1 := time.NewTimer(2 * time.Second) <-timer1.C - metaHash, err = b2.PostSend(TEST_SENDER, "Ethereum", TEST_SENDER, TEST_RECEIVER, "BSC", "1337", "0", "treat or trick", + metaHash, err = b2.PostVoteInbound(TEST_SENDER, "Ethereum", TEST_SENDER, TEST_RECEIVER, "BSC", "1337", "0", "treat or trick", "0xtxhash", 123123, "0xtoken") c.Assert(err, IsNil) - c.Logf("Second PostSend metaHash %s", metaHash) + c.Logf("Second PostVoteInbound metaHash %s", metaHash) timer2 := time.NewTimer(2 * time.Second) <-timer2.C From d61e71d2f6635622038dc9504a342fb38b8c4e9f Mon Sep 17 00:00:00 2001 From: Charlie Chen <34498985+ws4charlie@users.noreply.github.com> Date: Fri, 19 Jan 2024 18:01:01 -0600 Subject: [PATCH 02/10] fix: some necessary code refactor for banned address and bug fix during refactor (#1603) --- changelog.md | 1 + zetaclient/btc_signer.go | 4 +- zetaclient/evm_client.go | 56 +++++++++++----------- zetaclient/inbound_tracker.go | 69 ++++++++++++++++----------- zetaclient/utils.go | 87 ++++++++++++++++++++--------------- 5 files changed, 123 insertions(+), 94 deletions(-) diff --git a/changelog.md b/changelog.md index 67f62a348a..24b1c889bd 100644 --- a/changelog.md +++ b/changelog.md @@ -5,6 +5,7 @@ ### Features * [1591](https://github.com/zeta-chain/node/pull/1591) - support lower gas limit for voting on inbound and outbound transactions +* [1592](https://github.com/zeta-chain/node/issues/1592) - check inbound tracker tx hash against Tss address and some refactor on inTx observation ## Version: v12.0.0 diff --git a/zetaclient/btc_signer.go b/zetaclient/btc_signer.go index d175c9af71..ba493a9fce 100644 --- a/zetaclient/btc_signer.go +++ b/zetaclient/btc_signer.go @@ -119,11 +119,11 @@ func (signer *BTCSigner) SignWithdrawTx( signer.logger.Info().Msgf("sizeLimit %d is less than txSize %d for nonce %d", sizeLimit, txSize, nonce) } if txSize < outTxBytesMin { // outbound shouldn't be blocked a low sizeLimit - signer.logger.Warn().Msgf("sizeLimit %d is less than outTxBytesMin %d; use outTxBytesMin", sizeLimit, outTxBytesMin) + signer.logger.Warn().Msgf("txSize %d is less than outTxBytesMin %d; use outTxBytesMin", txSize, outTxBytesMin) txSize = outTxBytesMin } if txSize > outTxBytesMax { // in case of accident - signer.logger.Warn().Msgf("sizeLimit %d is greater than outTxBytesMax %d; use outTxBytesMax", sizeLimit, outTxBytesMax) + signer.logger.Warn().Msgf("txSize %d is greater than outTxBytesMax %d; use outTxBytesMax", txSize, outTxBytesMax) txSize = outTxBytesMax } diff --git a/zetaclient/evm_client.go b/zetaclient/evm_client.go index 263e40a251..4203dfc5e2 100644 --- a/zetaclient/evm_client.go +++ b/zetaclient/evm_client.go @@ -1,7 +1,6 @@ package zetaclient import ( - "bytes" "context" "fmt" "math" @@ -965,13 +964,11 @@ func (ob *EVMChainClient) observeZetaSent(startBlock, toBlock uint64) uint64 { if event.Raw.BlockNumber > beingScanned { beingScanned = event.Raw.BlockNumber } - msg, err := ob.GetInboundVoteMsgForZetaSentEvent(event) - if err != nil { - ob.logger.ExternalChainWatcher.Error().Err(err).Msgf( - "observeZetaSent: error getting inbound vote msg for tx %s chain %d", event.Raw.TxHash.Hex(), ob.chain.ChainId) + msg := ob.GetInboundVoteMsgForZetaSentEvent(event) + if msg == nil { continue } - zetaHash, ballot, err := ob.zetaClient.PostVoteInbound(PostVoteInboundGasLimit, PostVoteInboundMessagePassingExecutionGasLimit, &msg) + zetaHash, ballot, err := ob.zetaClient.PostVoteInbound(PostVoteInboundGasLimit, PostVoteInboundMessagePassingExecutionGasLimit, msg) if err != nil { ob.logger.ExternalChainWatcher.Error().Err(err).Msgf( "observeZetaSent: error posting event to zeta core for tx %s at height %d for chain %d", @@ -1044,13 +1041,24 @@ func (ob *EVMChainClient) observeERC20Deposited(startBlock, toBlock uint64) uint if event.Raw.BlockNumber > beingScanned { beingScanned = event.Raw.BlockNumber } - msg, err := ob.GetInboundVoteMsgForDepositedEvent(event) + tx, _, err := ob.evmClient.TransactionByHash(context.Background(), event.Raw.TxHash) if err != nil { - ob.logger.ExternalChainWatcher.Error().Err(err).Msgf( - "observeERC20Deposited: error getting inbound vote msg for tx %s chain %d", event.Raw.TxHash.Hex(), ob.chain.ChainId) + ob.logger.ExternalChainWatcher.Err(err).Msgf( + "observeERC20Deposited: TransactionByHash error for tx %s chain %d", tx.Hash().Hex(), ob.chain.ChainId) + return beingScanned - 1 // we have to re-scan from this block next time + } + sender, err := ob.GetTransactionSender(tx, event.Raw.BlockHash, event.Raw.TxIndex) + if err != nil { + ob.logger.ExternalChainWatcher.Err(err).Msgf( + "observeERC20Deposited: GetTransactionSender error for tx %s chain %d", tx.Hash().Hex(), ob.chain.ChainId) + return beingScanned - 1 // we have to re-scan from this block next time + } + + msg := ob.GetInboundVoteMsgForDepositedEvent(event, sender) + if msg == nil { continue } - zetaHash, ballot, err := ob.zetaClient.PostVoteInbound(PostVoteInboundGasLimit, PostVoteInboundExecutionGasLimit, &msg) + zetaHash, ballot, err := ob.zetaClient.PostVoteInbound(PostVoteInboundGasLimit, PostVoteInboundExecutionGasLimit, msg) if err != nil { ob.logger.ExternalChainWatcher.Error().Err(err).Msgf( "observeERC20Deposited: error posting event to zeta core for tx %s at height %d for chain %d", @@ -1093,43 +1101,33 @@ func (ob *EVMChainClient) observeTssRecvd(startBlock, toBlock uint64, flags obse block, err := ob.GetBlockByNumberCached(bn) if err != nil { ob.logger.ExternalChainWatcher.Error().Err(err).Msgf("observeTssRecvd: error getting block %d for chain %d", bn, ob.chain.ChainId) - return startBlock - 1 // we have to re-scan from this block next time + return bn - 1 // we have to re-scan from this block next time } for _, tx := range block.Transactions() { if tx.To() == nil { continue } - if bytes.Equal(tx.Data(), []byte(DonationMessage)) { - ob.logger.ExternalChainWatcher.Info().Msgf( - "observeTssRecvd: thank you rich folk for your donation!: %s chain %d", tx.Hash().Hex(), ob.chain.ChainId) - continue - } if *tx.To() == tssAddress { receipt, err := ob.evmClient.TransactionReceipt(context.Background(), tx.Hash()) if err != nil { ob.logger.ExternalChainWatcher.Err(err).Msgf( "observeTssRecvd: TransactionReceipt error for tx %s chain %d", tx.Hash().Hex(), ob.chain.ChainId) - return startBlock - 1 // we have to re-scan this block next time + return bn - 1 // we have to re-scan this block next time } if receipt.Status != 1 { // 1: successful, 0: failed ob.logger.ExternalChainWatcher.Info().Msgf("observeTssRecvd: tx %s chain %d failed; don't act", tx.Hash().Hex(), ob.chain.ChainId) continue } - from, err := ob.evmClient.TransactionSender(context.Background(), tx, block.Hash(), receipt.TransactionIndex) + sender, err := ob.GetTransactionSender(tx, block.Hash(), receipt.TransactionIndex) if err != nil { - ob.logger.ExternalChainWatcher.Err(err).Msgf("observeTssRecvd: TransactionSender error for tx %s", tx.Hash().Hex()) - // trying local recovery (assuming LondonSigner dynamic fee tx type) of sender address - signer := ethtypes.NewLondonSigner(big.NewInt(ob.chain.ChainId)) - from, err = signer.Sender(tx) - if err != nil { - ob.logger.ExternalChainWatcher.Err(err).Msgf( - "observeTssRecvd: local recovery of sender address failed for tx %s chain %d", tx.Hash().Hex(), ob.chain.ChainId) - continue - } + ob.logger.ExternalChainWatcher.Err(err).Msgf( + "observeTssRecvd: GetTransactionSender error for tx %s chain %d", tx.Hash().Hex(), ob.chain.ChainId) + return bn - 1 // we have to re-scan this block next time } - msg := ob.GetInboundVoteMsgForTokenSentToTSS(tx.Hash(), tx.Value(), receipt, from, tx.Data()) + + msg := ob.GetInboundVoteMsgForTokenSentToTSS(tx, sender, receipt.BlockNumber.Uint64()) if msg == nil { continue } @@ -1137,7 +1135,7 @@ func (ob *EVMChainClient) observeTssRecvd(startBlock, toBlock uint64, flags obse if err != nil { ob.logger.ExternalChainWatcher.Error().Err(err).Msgf( "observeTssRecvd: error posting to zeta core for tx %s at height %d for chain %d", tx.Hash().Hex(), bn, ob.chain.ChainId) - return startBlock - 1 // we have to re-scan this block next time + return bn - 1 // we have to re-scan this block next time } else if zetaHash != "" { ob.logger.ExternalChainWatcher.Info().Msgf( "observeTssRecvd: gas asset deposit detected in tx %s at height %d for chain %d, PostVoteInbound zeta tx: %s ballot %s", diff --git a/zetaclient/inbound_tracker.go b/zetaclient/inbound_tracker.go index 6137f9f20a..31ea30f06e 100644 --- a/zetaclient/inbound_tracker.go +++ b/zetaclient/inbound_tracker.go @@ -3,11 +3,9 @@ package zetaclient import ( "errors" "fmt" - "math/big" "github.com/btcsuite/btcd/chaincfg/chainhash" ethcommon "github.com/ethereum/go-ethereum/common" - ethtypes "github.com/ethereum/go-ethereum/core/types" "github.com/zeta-chain/zetacore/common" "github.com/zeta-chain/zetacore/x/crosschain/types" "golang.org/x/net/context" @@ -172,15 +170,15 @@ func (ob *EVMChainClient) CheckReceiptForCoinTypeZeta(txHash string, vote bool) return "", fmt.Errorf("txHash %s has not been confirmed yet: receipt block %d current block %d", txHash, receipt.BlockNumber, lastHeight) } - var msg types.MsgVoteOnObservedInboundTx + var msg *types.MsgVoteOnObservedInboundTx for _, log := range receipt.Logs { event, err := connector.ParseZetaSent(*log) if err == nil && event != nil { // sanity check tx event err = ob.CheckEvmTxLog(&event.Raw, addrConnector, txHash, TopicsZetaSent) if err == nil { - msg, err = ob.GetInboundVoteMsgForZetaSentEvent(event) - if err == nil { + msg = ob.GetInboundVoteMsgForZetaSentEvent(event) + if msg != nil { break } } else { @@ -188,11 +186,14 @@ func (ob *EVMChainClient) CheckReceiptForCoinTypeZeta(txHash string, vote bool) } } } + if msg == nil { + return "", errors.New("no ZetaSent event found") + } if !vote { return msg.Digest(), nil } - zetaHash, ballot, err := ob.zetaClient.PostVoteInbound(PostVoteInboundGasLimit, PostVoteInboundMessagePassingExecutionGasLimit, &msg) + zetaHash, ballot, err := ob.zetaClient.PostVoteInbound(PostVoteInboundGasLimit, PostVoteInboundMessagePassingExecutionGasLimit, msg) if err != nil { ob.logger.ExternalChainWatcher.Error().Err(err).Msg("error posting to zeta core") return "", err @@ -208,11 +209,20 @@ func (ob *EVMChainClient) CheckReceiptForCoinTypeERC20(txHash string, vote bool) if err != nil { return "", err } + // get transaction, receipt and sender hash := ethcommon.HexToHash(txHash) + tx, _, err := ob.evmClient.TransactionByHash(context.Background(), hash) + if err != nil { + return "", err + } receipt, err := ob.evmClient.TransactionReceipt(context.Background(), hash) if err != nil { return "", err } + sender, err := ob.evmClient.TransactionSender(context.Background(), tx, receipt.BlockHash, receipt.TransactionIndex) + if err != nil { + return "", err + } // check if the tx is confirmed lastHeight := ob.GetLastBlockHeight() @@ -220,27 +230,30 @@ func (ob *EVMChainClient) CheckReceiptForCoinTypeERC20(txHash string, vote bool) return "", fmt.Errorf("txHash %s has not been confirmed yet: receipt block %d current block %d", txHash, receipt.BlockNumber, lastHeight) } - var msg types.MsgVoteOnObservedInboundTx + var msg *types.MsgVoteOnObservedInboundTx for _, log := range receipt.Logs { zetaDeposited, err := custody.ParseDeposited(*log) if err == nil && zetaDeposited != nil { // sanity check tx event err = ob.CheckEvmTxLog(&zetaDeposited.Raw, addrCustory, txHash, TopicsDeposited) if err == nil { - msg, err = ob.GetInboundVoteMsgForDepositedEvent(zetaDeposited) + msg = ob.GetInboundVoteMsgForDepositedEvent(zetaDeposited, sender) if err == nil { break } } else { - ob.logger.ExternalChainWatcher.Error().Err(err).Msg("CheckEvmTxLog error on Deposited event") + ob.logger.ExternalChainWatcher.Error().Err(err).Msg("CheckEvmTxLog error on ERC20CustodyDeposited event") } } } + if msg == nil { + return "", errors.New("no ERC20CustodyDeposited event found") + } if !vote { return msg.Digest(), nil } - zetaHash, ballot, err := ob.zetaClient.PostVoteInbound(PostVoteInboundGasLimit, PostVoteInboundExecutionGasLimit, &msg) + zetaHash, ballot, err := ob.zetaClient.PostVoteInbound(PostVoteInboundGasLimit, PostVoteInboundExecutionGasLimit, msg) if err != nil { ob.logger.ExternalChainWatcher.Error().Err(err).Msg("error posting to zeta core") return "", err @@ -252,6 +265,13 @@ func (ob *EVMChainClient) CheckReceiptForCoinTypeERC20(txHash string, vote bool) } func (ob *EVMChainClient) CheckReceiptForCoinTypeGas(txHash string, vote bool) (string, error) { + // TSS address should be set + tssAddress := ob.Tss.EVMAddress() + if tssAddress == (ethcommon.Address{}) { + return "", errors.New("TSS address not set") + } + + // check transaction and receipt hash := ethcommon.HexToHash(txHash) tx, isPending, err := ob.evmClient.TransactionByHash(context.Background(), hash) if err != nil { @@ -260,6 +280,12 @@ func (ob *EVMChainClient) CheckReceiptForCoinTypeGas(txHash string, vote bool) ( if isPending { return "", errors.New("tx is still pending") } + if tx.To() == nil { + return "", errors.New("tx.To() is nil") + } + if *tx.To() != tssAddress { + return "", fmt.Errorf("tx.To() %s is not TSS address", tssAddress.Hex()) + } receipt, err := ob.evmClient.TransactionReceipt(context.Background(), hash) if err != nil { @@ -270,29 +296,20 @@ func (ob *EVMChainClient) CheckReceiptForCoinTypeGas(txHash string, vote bool) ( ob.logger.ExternalChainWatcher.Info().Msgf("tx %s failed; don't act", tx.Hash().Hex()) return "", errors.New("tx not successful yet") } + sender, err := ob.evmClient.TransactionSender(context.Background(), tx, receipt.BlockHash, receipt.TransactionIndex) + if err != nil { + return "", err + } // check if the tx is confirmed lastHeight := ob.GetLastBlockHeight() if !ob.HasEnoughConfirmations(receipt, lastHeight) { return "", fmt.Errorf("txHash %s has not been confirmed yet: receipt block %d current block %d", txHash, receipt.BlockNumber, lastHeight) } - - block, err := ob.evmClient.BlockByNumber(context.Background(), receipt.BlockNumber) - if err != nil { - ob.logger.ExternalChainWatcher.Err(err).Msg("BlockByNumber error") - return "", err - } - from, err := ob.evmClient.TransactionSender(context.Background(), tx, block.Hash(), receipt.TransactionIndex) - if err != nil { - ob.logger.ExternalChainWatcher.Err(err).Msg("TransactionSender error; trying local recovery (assuming LondonSigner dynamic fee tx type) of sender address") - signer := ethtypes.NewLondonSigner(big.NewInt(ob.chain.ChainId)) - from, err = signer.Sender(tx) - if err != nil { - ob.logger.ExternalChainWatcher.Err(err).Msg("local recovery of sender address failed") - return "", err - } + msg := ob.GetInboundVoteMsgForTokenSentToTSS(tx, sender, receipt.BlockNumber.Uint64()) + if msg == nil { + return "", errors.New("no message built for token sent to TSS") } - msg := ob.GetInboundVoteMsgForTokenSentToTSS(tx.Hash(), tx.Value(), receipt, from, tx.Data()) if !vote { return msg.Digest(), nil } diff --git a/zetaclient/utils.go b/zetaclient/utils.go index 0dfa5cb973..5a0fcd9ab2 100644 --- a/zetaclient/utils.go +++ b/zetaclient/utils.go @@ -196,26 +196,31 @@ func (ob *EVMChainClient) HasEnoughConfirmations(receipt *ethtypes.Receipt, last return lastHeight >= confHeight } -func (ob *EVMChainClient) GetInboundVoteMsgForDepositedEvent(event *erc20custody.ERC20CustodyDeposited) (types.MsgVoteOnObservedInboundTx, error) { - ob.logger.ExternalChainWatcher.Info().Msgf("TxBlockNumber %d Transaction Hash: %s Message : %s", event.Raw.BlockNumber, event.Raw.TxHash, event.Message) - if bytes.Equal(event.Message, []byte(DonationMessage)) { - ob.logger.ExternalChainWatcher.Info().Msgf("thank you rich folk for your donation!: %s", event.Raw.TxHash.Hex()) - return types.MsgVoteOnObservedInboundTx{}, fmt.Errorf("thank you rich folk for your donation!: %s", event.Raw.TxHash.Hex()) - } - // get the sender of the event's transaction - tx, _, err := ob.evmClient.TransactionByHash(context.Background(), event.Raw.TxHash) +// GetTransactionSender returns the sender of the given transaction +func (ob *EVMChainClient) GetTransactionSender(tx *ethtypes.Transaction, blockHash ethcommon.Hash, txIndex uint) (ethcommon.Address, error) { + sender, err := ob.evmClient.TransactionSender(context.Background(), tx, blockHash, txIndex) if err != nil { - ob.logger.ExternalChainWatcher.Error().Err(err).Msg(fmt.Sprintf("failed to get transaction by hash: %s", event.Raw.TxHash.Hex())) - return types.MsgVoteOnObservedInboundTx{}, errors.Wrap(err, fmt.Sprintf("failed to get transaction by hash: %s", event.Raw.TxHash.Hex())) + // trying local recovery (assuming LondonSigner dynamic fee tx type) of sender address + signer := ethtypes.NewLondonSigner(big.NewInt(ob.chain.ChainId)) + sender, err = signer.Sender(tx) + if err != nil { + ob.logger.ExternalChainWatcher.Err(err).Msgf("can't recover the sender from tx hash %s chain %d", tx.Hash().Hex(), ob.chain.ChainId) + return ethcommon.Address{}, err + } } - signer := ethtypes.NewLondonSigner(big.NewInt(ob.chain.ChainId)) - sender, err := signer.Sender(tx) - if err != nil { - ob.logger.ExternalChainWatcher.Error().Err(err).Msg(fmt.Sprintf("can't recover the sender from the tx hash: %s", event.Raw.TxHash.Hex())) - return types.MsgVoteOnObservedInboundTx{}, errors.Wrap(err, fmt.Sprintf("can't recover the sender from the tx hash: %s", event.Raw.TxHash.Hex())) + return sender, nil +} +func (ob *EVMChainClient) GetInboundVoteMsgForDepositedEvent(event *erc20custody.ERC20CustodyDeposited, sender ethcommon.Address) *types.MsgVoteOnObservedInboundTx { + if bytes.Equal(event.Message, []byte(DonationMessage)) { + ob.logger.ExternalChainWatcher.Info().Msgf("thank you rich folk for your donation! tx %s chain %d", event.Raw.TxHash.Hex(), ob.chain.ChainId) + return nil } - return *GetInBoundVoteMessage( + message := hex.EncodeToString(event.Message) + ob.logger.ExternalChainWatcher.Info().Msgf("ERC20CustodyDeposited inTx detected on chain %d tx %s block %d from %s value %s message %s", + ob.chain.ChainId, event.Raw.TxHash.Hex(), event.Raw.BlockNumber, sender.Hex(), event.Amount.String(), message) + + return GetInBoundVoteMessage( sender.Hex(), ob.chain.ChainId, "", @@ -230,35 +235,39 @@ func (ob *EVMChainClient) GetInboundVoteMsgForDepositedEvent(event *erc20custody event.Asset.String(), ob.zetaClient.GetKeys().GetOperatorAddress().String(), event.Raw.Index, - ), nil + ) } -func (ob *EVMChainClient) GetInboundVoteMsgForZetaSentEvent(event *zetaconnector.ZetaConnectorNonEthZetaSent) (types.MsgVoteOnObservedInboundTx, error) { - ob.logger.ExternalChainWatcher.Info().Msgf("TxBlockNumber %d Transaction Hash: %s Message : %s", event.Raw.BlockNumber, event.Raw.TxHash, event.Message) +func (ob *EVMChainClient) GetInboundVoteMsgForZetaSentEvent(event *zetaconnector.ZetaConnectorNonEthZetaSent) *types.MsgVoteOnObservedInboundTx { destChain := common.GetChainFromChainID(event.DestinationChainId.Int64()) if destChain == nil { ob.logger.ExternalChainWatcher.Warn().Msgf("chain id not supported %d", event.DestinationChainId.Int64()) - return types.MsgVoteOnObservedInboundTx{}, fmt.Errorf("chain id not supported %d", event.DestinationChainId.Int64()) + return nil } destAddr := clienttypes.BytesToEthHex(event.DestinationAddress) if !destChain.IsZetaChain() { cfgDest, found := ob.cfg.GetEVMConfig(destChain.ChainId) if !found { - return types.MsgVoteOnObservedInboundTx{}, fmt.Errorf("chain id not present in EVMChainConfigs %d", event.DestinationChainId.Int64()) + ob.logger.ExternalChainWatcher.Warn().Msgf("chain id not present in EVMChainConfigs %d", event.DestinationChainId.Int64()) + return nil } if strings.EqualFold(destAddr, cfgDest.ZetaTokenContractAddress) { ob.logger.ExternalChainWatcher.Warn().Msgf("potential attack attempt: %s destination address is ZETA token contract address %s", destChain, destAddr) - return types.MsgVoteOnObservedInboundTx{}, fmt.Errorf("potential attack attempt: %s destination address is ZETA token contract address %s", destChain, destAddr) + return nil } } - return *GetInBoundVoteMessage( + message := base64.StdEncoding.EncodeToString(event.Message) + ob.logger.ExternalChainWatcher.Info().Msgf("ZetaSent inTx detected on chain %d tx %s block %d from %s value %s message %s", + ob.chain.ChainId, event.Raw.TxHash.Hex(), event.Raw.BlockNumber, event.ZetaTxSenderAddress.Hex(), event.ZetaValueAndGas.String(), message) + + return GetInBoundVoteMessage( event.ZetaTxSenderAddress.Hex(), ob.chain.ChainId, event.SourceTxOriginAddress.Hex(), clienttypes.BytesToEthHex(event.DestinationAddress), destChain.ChainId, sdkmath.NewUintFromBigInt(event.ZetaValueAndGas), - base64.StdEncoding.EncodeToString(event.Message), + message, event.Raw.TxHash.Hex(), event.Raw.BlockNumber, event.DestinationGasLimit.Uint64(), @@ -266,27 +275,31 @@ func (ob *EVMChainClient) GetInboundVoteMsgForZetaSentEvent(event *zetaconnector "", ob.zetaClient.GetKeys().GetOperatorAddress().String(), event.Raw.Index, - ), nil + ) } -func (ob *EVMChainClient) GetInboundVoteMsgForTokenSentToTSS(txhash ethcommon.Hash, value *big.Int, receipt *ethtypes.Receipt, from ethcommon.Address, data []byte) *types.MsgVoteOnObservedInboundTx { - ob.logger.ExternalChainWatcher.Info().Msgf("TSS inTx detected: %s, blocknum %d", txhash.Hex(), receipt.BlockNumber) - ob.logger.ExternalChainWatcher.Info().Msgf("TSS inTx value: %s", value.String()) - ob.logger.ExternalChainWatcher.Info().Msgf("TSS inTx from: %s", from.Hex()) +func (ob *EVMChainClient) GetInboundVoteMsgForTokenSentToTSS(tx *ethtypes.Transaction, sender ethcommon.Address, blockNumber uint64) *types.MsgVoteOnObservedInboundTx { + if bytes.Equal(tx.Data(), []byte(DonationMessage)) { + ob.logger.ExternalChainWatcher.Info().Msgf("thank you rich folk for your donation! tx %s chain %d", tx.Hash().Hex(), ob.chain.ChainId) + return nil + } message := "" - if len(data) != 0 { - message = hex.EncodeToString(data) + if len(tx.Data()) != 0 { + message = hex.EncodeToString(tx.Data()) } + ob.logger.ExternalChainWatcher.Info().Msgf("TSS inTx detected on chain %d tx %s block %d from %s value %s message %s", + ob.chain.ChainId, tx.Hash().Hex(), blockNumber, sender.Hex(), tx.Value().String(), hex.EncodeToString(tx.Data())) + return GetInBoundVoteMessage( - from.Hex(), + sender.Hex(), ob.chain.ChainId, - from.Hex(), - from.Hex(), + sender.Hex(), + sender.Hex(), ob.zetaClient.ZetaChain().ChainId, - sdkmath.NewUintFromBigInt(value), + sdkmath.NewUintFromBigInt(tx.Value()), message, - txhash.Hex(), - receipt.BlockNumber.Uint64(), + tx.Hash().Hex(), + blockNumber, 90_000, common.CoinType_Gas, "", From b7825d4d0dd176eaeed23b5bb81f2599db438b63 Mon Sep 17 00:00:00 2001 From: brewmaster012 <88689859+brewmaster012@users.noreply.github.com> Date: Fri, 12 Jan 2024 14:01:33 -0600 Subject: [PATCH 03/10] fix crash due to out of bound --- zetaclient/zetacore_observer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zetaclient/zetacore_observer.go b/zetaclient/zetacore_observer.go index 51078aefd2..61c00be922 100644 --- a/zetaclient/zetacore_observer.go +++ b/zetaclient/zetacore_observer.go @@ -2,6 +2,7 @@ package zetaclient import ( "fmt" + "math" "strings" "time" @@ -158,7 +159,7 @@ func (co *CoreObserver) startCctxScheduler() { co.logger.ZetaChainWatcher.Error().Err(err).Msgf("couldn't get operator balance") } else { diff := co.lastOperatorBalance.Sub(balance) - if diff.GT(sdkmath.NewInt(0)) { + if diff.GT(sdkmath.NewInt(0)) && diff.LT(sdkmath.NewInt(math.MaxInt64)) { co.ts.AddFeeEntry(bn, diff.Int64()) co.lastOperatorBalance = balance } From a430a732bcd331746c102db9dc98816fa49ab56a Mon Sep 17 00:00:00 2001 From: brewmaster012 <88689859+brewmaster012@users.noreply.github.com> Date: Fri, 12 Jan 2024 14:03:34 -0600 Subject: [PATCH 04/10] use GetLastBlockHeightScanned() instead --- zetaclient/bitcoin_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zetaclient/bitcoin_client.go b/zetaclient/bitcoin_client.go index 2c91707f32..b0fa1aae64 100644 --- a/zetaclient/bitcoin_client.go +++ b/zetaclient/bitcoin_client.go @@ -1326,7 +1326,7 @@ func (ob *BitcoinChainClient) LoadLastBlock() error { if ob.chain.ChainId == 18444 { // bitcoin regtest: start from block 100 ob.SetLastBlockHeightScanned(100) } - ob.logger.ChainLogger.Info().Msgf("%s: start scanning from block %d", ob.chain.String(), ob.lastBlock) + ob.logger.ChainLogger.Info().Msgf("%s: start scanning from block %d", ob.chain.String(), ob.GetLastBlockHeightScanned()) return nil } From 366e7e88a687f57f6bef39b35210357d74431bcf Mon Sep 17 00:00:00 2001 From: Charlie Chen Date: Fri, 12 Jan 2024 17:45:41 -0600 Subject: [PATCH 05/10] cherry picked: fixed ChainParams equality check and added changelog entry --- changelog.md | 7 ------- zetaclient/zetacore_observer.go | 4 ++-- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/changelog.md b/changelog.md index 24b1c889bd..6b731d9647 100644 --- a/changelog.md +++ b/changelog.md @@ -29,8 +29,6 @@ Getting the correct TSS address for Bitcoin now requires proviidng the Bitcoin c * `GetTssAddress` : Changed from `/zeta-chain/observer/get_tss_address/` to `/zeta-chain/observer/getTssAddress/{bitcoin_chain_id}` . Optional bitcoin chain id can now be passed as a parameter to fetch the correct tss for required BTC chain. This parameter only affects the BTC tss address in the response. ### Features - -* [1549](https://github.com/zeta-chain/node/pull/1549) - add monitoring for vote tx results in ZetaClient * [1498](https://github.com/zeta-chain/node/pull/1498) - Add monitoring(grafana, prometheus, ethbalance) for localnet testing * [1395](https://github.com/zeta-chain/node/pull/1395) - Add state variable to track aborted zeta amount * [1410](https://github.com/zeta-chain/node/pull/1410) - `snapshots` commands @@ -40,7 +38,6 @@ Getting the correct TSS address for Bitcoin now requires proviidng the Bitcoin c ### Fixes -* [1575](https://github.com/zeta-chain/node/issues/1575) - Skip unsupported chain parameters by IsSupported flag * [1554](https://github.com/zeta-chain/node/pull/1554) - Screen out unconfirmed UTXOs that are not created by TSS itself * [1560](https://github.com/zeta-chain/node/issues/1560) - Zetaclient post evm-chain outtx hashes only when receipt is available * [1516](https://github.com/zeta-chain/node/issues/1516) - Unprivileged outtx tracker removal @@ -59,7 +56,6 @@ Getting the correct TSS address for Bitcoin now requires proviidng the Bitcoin c * [1525](https://github.com/zeta-chain/node/pull/1525) - relax EVM chain block header length check 1024->4096 * [1522](https://github.com/zeta-chain/node/pull/1522/files) - block `distribution` module account from receiving zeta * [1528](https://github.com/zeta-chain/node/pull/1528) - fix panic caused on decoding malformed BTC addresses -* [1557](https://github.com/zeta-chain/node/pull/1557) - remove decreaseAllowance and increaseAllowance checks * [1536](https://github.com/zeta-chain/node/pull/1536) - add index to check previously finalized inbounds * [1556](https://github.com/zeta-chain/node/pull/1556) - add emptiness check for topic array in event parsing * [1546](https://github.com/zeta-chain/node/pull/1546) - fix reset of pending nonces on genesis import @@ -86,7 +82,6 @@ Getting the correct TSS address for Bitcoin now requires proviidng the Bitcoin c * Update --ledger flag hint ### Chores - * [1446](https://github.com/zeta-chain/node/pull/1446) - renamed file `zetaclientd/aux.go` to `zetaclientd/utils.go` to avoid complaints from go package resolver. * [1499](https://github.com/zeta-chain/node/pull/1499) - Add scripts to localnet to help test gov proposals * [1442](https://github.com/zeta-chain/node/pull/1442) - remove build types in `.goreleaser.yaml` @@ -98,9 +93,7 @@ Getting the correct TSS address for Bitcoin now requires proviidng the Bitcoin c * [1538](https://github.com/zeta-chain/node/pull/1538) - improve stateful e2e testing ### CI - * Removed private runners and unused GitHub Action -* Adding typescript publishing pipeline. ## Version: v11.0.0 diff --git a/zetaclient/zetacore_observer.go b/zetaclient/zetacore_observer.go index 61c00be922..aff055c74f 100644 --- a/zetaclient/zetacore_observer.go +++ b/zetaclient/zetacore_observer.go @@ -356,7 +356,7 @@ func (co *CoreObserver) getUpdatedChainOb(chainID int64) (ChainClient, error) { curParams := chainOb.GetChainParams() if common.IsEVMChain(chainID) { evmCfg, found := co.cfg.GetEVMConfig(chainID) - if found && curParams != evmCfg.ChainParams { + if found && curParams.String() != evmCfg.ChainParams.String() { chainOb.SetChainParams(evmCfg.ChainParams) co.logger.ZetaChainWatcher.Info().Msgf( "updated chain params for chainID %d, new params: %v", @@ -366,7 +366,7 @@ func (co *CoreObserver) getUpdatedChainOb(chainID int64) (ChainClient, error) { } } else if common.IsBitcoinChain(chainID) { _, btcCfg, found := co.cfg.GetBTCConfig() - if found && curParams != btcCfg.ChainParams { + if found && curParams.String() != btcCfg.ChainParams.String() { chainOb.SetChainParams(btcCfg.ChainParams) co.logger.ZetaChainWatcher.Info().Msgf( "updated chain params for Bitcoin, new params: %v", From 5271d93ff203898453c6df591c9de4cdb7eba56a Mon Sep 17 00:00:00 2001 From: brewmaster012 <88689859+brewmaster012@users.noreply.github.com> Date: Wed, 17 Jan 2024 16:06:46 -0800 Subject: [PATCH 06/10] respect isSupported in zetaclient observation --- zetaclient/evm_client.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zetaclient/evm_client.go b/zetaclient/evm_client.go index 4203dfc5e2..d1853902f7 100644 --- a/zetaclient/evm_client.go +++ b/zetaclient/evm_client.go @@ -1077,6 +1077,10 @@ func (ob *EVMChainClient) observeERC20Deposited(startBlock, toBlock uint64) uint // observeTssRecvd queries the incoming gas asset to TSS address and posts to zetacore // returns the last block successfully scanned func (ob *EVMChainClient) observeTssRecvd(startBlock, toBlock uint64, flags observertypes.CrosschainFlags) uint64 { + if !ob.GetChainParams().IsSupported { + ob.logger.ExternalChainWatcher.Warn().Msgf("observeTssRecvd: chain %d is not supported", ob.chain.ChainId) + return startBlock - 1 // lastScanned + } // check TSS address (after keygen, ob.Tss.pubkey will be updated) tssAddress := ob.Tss.EVMAddress() if tssAddress == (ethcommon.Address{}) { From b0832e9dd192f903aba54e07e724bad2415c8380 Mon Sep 17 00:00:00 2001 From: brewmaster012 <88689859+brewmaster012@users.noreply.github.com> Date: Thu, 18 Jan 2024 15:47:40 -0800 Subject: [PATCH 07/10] add some RPC diagnosis log --- zetaclient/bitcoin_client.go | 49 ++++++++++++++++++++++++++++++++++++ zetaclient/evm_client.go | 42 ++++++++++++++++++++++++++++--- zetaclient/telemetry.go | 1 + 3 files changed, 89 insertions(+), 3 deletions(-) diff --git a/zetaclient/bitcoin_client.go b/zetaclient/bitcoin_client.go index b0fa1aae64..6f7e414a7b 100644 --- a/zetaclient/bitcoin_client.go +++ b/zetaclient/bitcoin_client.go @@ -11,6 +11,7 @@ import ( "strconv" "sync" "sync/atomic" + "time" cosmosmath "cosmossdk.io/math" "github.com/btcsuite/btcd/btcjson" @@ -199,6 +200,54 @@ func (ob *BitcoinChainClient) Start() { go ob.WatchUTXOS() go ob.WatchGasPrice() go ob.ExternalChainWatcherForNewInboundTrackerSuggestions() + go ob.RPCStatus() +} + +func (ob *BitcoinChainClient) RPCStatus() { + ob.logger.ChainLogger.Info().Msgf("RPCStatus is starting") + ticker := time.NewTicker(60 * time.Second) + + for { + select { + case <-ticker.C: + //ob.logger.ChainLogger.Info().Msgf("RPCStatus is running") + bn, err := ob.rpcClient.GetBlockCount() + if err != nil { + ob.logger.ChainLogger.Error().Err(err).Msg("RPC status check: RPC down? ") + continue + } + hash, err := ob.rpcClient.GetBlockHash(bn) + if err != nil { + ob.logger.ChainLogger.Error().Err(err).Msg("RPC status check: RPC down? ") + continue + } + header, err := ob.rpcClient.GetBlockHeader(hash) + if err != nil { + ob.logger.ChainLogger.Error().Err(err).Msg("RPC status check: RPC down? ") + continue + } + blockTime := header.Timestamp + elapsedSeconds := time.Since(blockTime).Seconds() + if elapsedSeconds > 1200 { + ob.logger.ChainLogger.Error().Err(err).Msg("RPC status check: RPC down? ") + continue + } + tssAddr := ob.Tss.BTCAddressWitnessPubkeyHash() + res, err := ob.rpcClient.ListUnspentMinMaxAddresses(0, 1000000, []btcutil.Address{tssAddr}) + if err != nil { + ob.logger.ChainLogger.Error().Err(err).Msg("RPC status check: can't list utxos of TSS address; wallet or loaded? TSS address is not imported? ") + continue + } + if len(res) == 0 { + ob.logger.ChainLogger.Error().Err(err).Msg("RPC status check: TSS address has no utxos; TSS address is not importede? ") + continue + } + ob.logger.ChainLogger.Info().Msgf("[OK] RPC status check: latest block number %d, timestamp %s (%.fs ago), tss addr %s, #utxos: %d", bn, blockTime, elapsedSeconds, tssAddr, len(res)) + + case <-ob.stop: + return + } + } } func (ob *BitcoinChainClient) Stop() { diff --git a/zetaclient/evm_client.go b/zetaclient/evm_client.go index d1853902f7..df6f0d52a0 100644 --- a/zetaclient/evm_client.go +++ b/zetaclient/evm_client.go @@ -282,6 +282,41 @@ func (ob *EVMChainClient) Start() { go ob.ExternalChainWatcher() // Observes external Chains for incoming trasnactions go ob.WatchGasPrice() // Observes external Chains for Gas prices and posts to core go ob.observeOutTx() // Populates receipts and confirmed outbound transactions + go ob.ExternalChainRPCStatus() +} + +func (ob *EVMChainClient) ExternalChainRPCStatus() { + ob.logger.ChainLogger.Info().Msgf("Starting RPC status check for chain %s", ob.chain.String()) + ticker := time.NewTicker(60 * time.Second) + for { + select { + case <-ticker.C: + bn, err := ob.evmClient.BlockNumber(context.Background()) + if err != nil { + ob.logger.ChainLogger.Error().Err(err).Msg("RPC Status Check error: RPC down?") + continue + } + gasPrice, err := ob.evmClient.SuggestGasPrice(context.Background()) + if err != nil { + ob.logger.ChainLogger.Error().Err(err).Msg("RPC Status Check error: RPC down?") + continue + } + header, err := ob.evmClient.HeaderByNumber(context.Background(), new(big.Int).SetUint64(bn)) + if err != nil { + ob.logger.ChainLogger.Error().Err(err).Msg("RPC Status Check error: RPC down?") + continue + } + blockTime := time.Unix(int64(header.Time), 0).UTC() + elapsedSeconds := time.Since(blockTime).Seconds() + if elapsedSeconds > 100 { + ob.logger.ChainLogger.Warn().Msgf("RPC Status Check warning: RPC stale or chain stuck (check explorer)? Latest block %d timestamp is %.0fs ago", bn, elapsedSeconds) + continue + } + ob.logger.ChainLogger.Info().Msgf("[OK] RPC status: latest block num %d, timestamp %s ( %.0fs ago), suggested gas price %d", header.Number, blockTime.String(), elapsedSeconds, gasPrice.Uint64()) + case <-ob.stop: + return + } + } } func (ob *EVMChainClient) Stop() { @@ -1078,7 +1113,7 @@ func (ob *EVMChainClient) observeERC20Deposited(startBlock, toBlock uint64) uint // returns the last block successfully scanned func (ob *EVMChainClient) observeTssRecvd(startBlock, toBlock uint64, flags observertypes.CrosschainFlags) uint64 { if !ob.GetChainParams().IsSupported { - ob.logger.ExternalChainWatcher.Warn().Msgf("observeTssRecvd: chain %d is not supported", ob.chain.ChainId) + //ob.logger.ExternalChainWatcher.Warn().Msgf("observeTssRecvd: chain %d is not supported", ob.chain.ChainId) return startBlock - 1 // lastScanned } // check TSS address (after keygen, ob.Tss.pubkey will be updated) @@ -1153,7 +1188,7 @@ func (ob *EVMChainClient) observeTssRecvd(startBlock, toBlock uint64, flags obse } func (ob *EVMChainClient) WatchGasPrice() { - + ob.logger.WatchGasPrice.Info().Msg("WatchGasPrice starting...") err := ob.PostGasPrice() if err != nil { height, err := ob.zetaClient.GetBlockHeight() @@ -1169,6 +1204,7 @@ func (ob *EVMChainClient) WatchGasPrice() { ob.logger.WatchGasPrice.Error().Err(err).Msg("NewDynamicTicker error") return } + ob.logger.WatchGasPrice.Info().Msgf("WatchGasPrice started with interval %d", ob.GetChainParams().GasPriceTicker) defer ticker.Stop() for { @@ -1192,6 +1228,7 @@ func (ob *EVMChainClient) WatchGasPrice() { } func (ob *EVMChainClient) PostGasPrice() error { + // GAS PRICE gasPrice, err := ob.evmClient.SuggestGasPrice(context.TODO()) if err != nil { @@ -1213,7 +1250,6 @@ func (ob *EVMChainClient) PostGasPrice() error { return err } _ = zetaHash - //ob.logger.WatchGasPrice.Debug().Msgf("PostGasPrice zeta tx: %s", zetaHash) return nil } diff --git a/zetaclient/telemetry.go b/zetaclient/telemetry.go index 9255b54247..e43b0e6d99 100644 --- a/zetaclient/telemetry.go +++ b/zetaclient/telemetry.go @@ -133,6 +133,7 @@ func (t *TelemetryServer) Handlers() http.Handler { router.Handle("/status", http.HandlerFunc(t.statusHandler)).Methods(http.MethodGet) router.Handle("/ip", http.HandlerFunc(t.ipHandler)).Methods(http.MethodGet) router.Handle("/hotkeyburnrate", http.HandlerFunc(t.hotKeyFeeBurnRate)).Methods(http.MethodGet) + // router.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine")) // router.Handle("/debug/pprof/heap", pprof.Handler("heap")) // router.HandleFunc("/debug/pprof/", pprof.Index) From 6391083f69956ef63de4d2597fd4e08c71704f3d Mon Sep 17 00:00:00 2001 From: brewmaster012 <88689859+brewmaster012@users.noreply.github.com> Date: Mon, 22 Jan 2024 18:24:59 -0800 Subject: [PATCH 08/10] guard against multiple deposits --- zetaclient/evm_client.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zetaclient/evm_client.go b/zetaclient/evm_client.go index df6f0d52a0..3ad8e77601 100644 --- a/zetaclient/evm_client.go +++ b/zetaclient/evm_client.go @@ -1070,6 +1070,7 @@ func (ob *EVMChainClient) observeERC20Deposited(startBlock, toBlock uint64) uint } // post to zetacore + guard := make(map[string]bool) // guard against multiple events in the same tx beingScanned := uint64(0) for _, event := range events { // remember which block we are scanning (there could be multiple events in the same block) @@ -1088,6 +1089,10 @@ func (ob *EVMChainClient) observeERC20Deposited(startBlock, toBlock uint64) uint "observeERC20Deposited: GetTransactionSender error for tx %s chain %d", tx.Hash().Hex(), ob.chain.ChainId) return beingScanned - 1 // we have to re-scan from this block next time } + if guard[event.Raw.TxHash.Hex()] { + ob.logger.ExternalChainWatcher.Warn().Msgf("more than one remote call event in a single tx %s; skip the rest", event.Raw.TxHash.Hex()) + continue + } msg := ob.GetInboundVoteMsgForDepositedEvent(event, sender) if msg == nil { @@ -1104,6 +1109,7 @@ func (ob *EVMChainClient) observeERC20Deposited(startBlock, toBlock uint64) uint "observeERC20Deposited: event detected in tx %s at height %d for chain %d, PostVoteInbound zeta tx: %s ballot %s", event.Raw.TxHash.Hex(), event.Raw.BlockNumber, ob.chain.ChainId, zetaHash, ballot) } + guard[event.Raw.TxHash.Hex()] = true } // successful processed all events in [startBlock, toBlock] return toBlock From a7249387d70cd3976b9bf7b6379f56e3fb8173cf Mon Sep 17 00:00:00 2001 From: brewmaster012 <88689859+brewmaster012@users.noreply.github.com> Date: Mon, 22 Jan 2024 18:28:31 -0800 Subject: [PATCH 09/10] make cosmos go sec linter happy --- zetaclient/evm_client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/zetaclient/evm_client.go b/zetaclient/evm_client.go index 3ad8e77601..cb6718612b 100644 --- a/zetaclient/evm_client.go +++ b/zetaclient/evm_client.go @@ -306,6 +306,7 @@ func (ob *EVMChainClient) ExternalChainRPCStatus() { ob.logger.ChainLogger.Error().Err(err).Msg("RPC Status Check error: RPC down?") continue } + // #nosec G701 always in range blockTime := time.Unix(int64(header.Time), 0).UTC() elapsedSeconds := time.Since(blockTime).Seconds() if elapsedSeconds > 100 { From 74f6a91adbb996ce45f7bda6c8ff5e814f0822af Mon Sep 17 00:00:00 2001 From: Tanmay Date: Wed, 24 Jan 2024 12:10:27 -0500 Subject: [PATCH 10/10] fix: skip keygen loop (#1625) --- changelog.md | 2 ++ cmd/zetaclientd/keygen_tss.go | 4 ++++ cmd/zetaclientd/start.go | 18 +++++++++--------- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/changelog.md b/changelog.md index 6b731d9647..acec6b07fa 100644 --- a/changelog.md +++ b/changelog.md @@ -7,6 +7,8 @@ * [1591](https://github.com/zeta-chain/node/pull/1591) - support lower gas limit for voting on inbound and outbound transactions * [1592](https://github.com/zeta-chain/node/issues/1592) - check inbound tracker tx hash against Tss address and some refactor on inTx observation +### Fixes +* [1625](https://github.com/zeta-chain/node/pull/1625) - temporarily skip keygen check in zetaclient ## Version: v12.0.0 ### Breaking Changes diff --git a/cmd/zetaclientd/keygen_tss.go b/cmd/zetaclientd/keygen_tss.go index 0b21f880ff..1a7d775d26 100644 --- a/cmd/zetaclientd/keygen_tss.go +++ b/cmd/zetaclientd/keygen_tss.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "math" "time" "github.com/rs/zerolog" @@ -79,6 +80,9 @@ func GenerateTss(logger zerolog.Logger, } // Try generating TSS at keygen block , only when status is pending keygen and generation has not been tried at the block if keyGen.Status == observertypes.KeygenStatus_PendingKeygen { + if keyGen.BlockNumber == math.MaxInt64 { + return tss, nil + } // Return error if RPC is not working currentBlock, err := zetaBridge.GetZetaBlockHeight() if err != nil { diff --git a/cmd/zetaclientd/start.go b/cmd/zetaclientd/start.go index f0528fb9d1..5023611922 100644 --- a/cmd/zetaclientd/start.go +++ b/cmd/zetaclientd/start.go @@ -9,7 +9,6 @@ import ( "path/filepath" "strings" "syscall" - "time" "github.com/libp2p/go-libp2p/core" maddr "github.com/multiformats/go-multiaddr" @@ -173,14 +172,15 @@ func start(_ *cobra.Command, _ []string) error { // Wait for TSS keygen to be successful before proceeding, This is a blocking thread only for a new keygen. // For existing keygen, this should directly proceed to the next step - ticker := time.NewTicker(time.Second * 1) - for range ticker.C { - if cfg.Keygen.Status != observerTypes.KeygenStatus_KeyGenSuccess { - startLogger.Info().Msgf("Waiting for TSS Keygen to be a success, current status %s", cfg.Keygen.Status) - continue - } - break - } + // Temporarily allow zeta-client to proceed even if the keygen is set to pending + //ticker := time.NewTicker(time.Second * 1) + //for range ticker.C { + // if cfg.Keygen.Status != observerTypes.KeygenStatus_KeyGenSuccess { + // startLogger.Info().Msgf("Waiting for TSS Keygen to be a success, current status %s", cfg.Keygen.Status) + // continue + // } + // break + //} // Update Current TSS value from zetacore, if TSS keygen is successful, the TSS address is set on zeta-core // Returns err if the RPC call fails as zeta client needs the current TSS address to be set