fix: tss metrics (#1163)

cmd/zetaclientd/keygen_tss.go

@@ -15,14 +15,15 @@ import (
 	observerTypes "github.com/zeta-chain/zetacore/x/observer/types"
 	mc "github.com/zeta-chain/zetacore/zetaclient"
 	"github.com/zeta-chain/zetacore/zetaclient/config"
+	"github.com/zeta-chain/zetacore/zetaclient/metrics"
 	tsscommon "gitlab.com/thorchain/tss/go-tss/common"
 	"gitlab.com/thorchain/tss/go-tss/keygen"
 	"gitlab.com/thorchain/tss/go-tss/p2p"
 )
 
-func GenerateTss(logger zerolog.Logger, cfg *config.Config, zetaBridge *mc.ZetaCoreBridge, peers p2p.AddrList, priKey secp256k1.PrivKey, ts *mc.TelemetryServer, tssHistoricalList []types.TSS) (*mc.TSS, error) {
+func GenerateTss(logger zerolog.Logger, cfg *config.Config, zetaBridge *mc.ZetaCoreBridge, peers p2p.AddrList, priKey secp256k1.PrivKey, ts *mc.TelemetryServer, tssHistoricalList []types.TSS, metrics *metrics.Metrics) (*mc.TSS, error) {
 	keygenLogger := logger.With().Str("module", "keygen").Logger()
-	tss, err := mc.NewTSS(peers, priKey, preParams, cfg, zetaBridge, tssHistoricalList)
+	tss, err := mc.NewTSS(peers, priKey, preParams, cfg, zetaBridge, tssHistoricalList, metrics)
 	if err != nil {
 		keygenLogger.Error().Err(err).Msg("NewTSS error")
 		return nil, err
@@ -69,7 +70,7 @@ func GenerateTss(logger zerolog.Logger, cfg *config.Config, zetaBridge *mc.ZetaC
 				if currentBlock != keyGen.BlockNumber {
 					if currentBlock > lastBlock {
 						lastBlock = currentBlock
-						keygenLogger.Info().Msgf("Waiting For Keygen Block to arrive or new keygen block to be set. Keygen Block : %d Current Block : %d", keyGen.BlockNumber, currentBlock)
+						keygenLogger.Info().Msgf("Waiting For Keygen Block to arrive or new keygen block to be set. Keygen Block : %d Current Block : %d ChainID %s ", keyGen.BlockNumber, currentBlock, cfg.ChainID)
 					}
 					continue
 				}

cmd/zetaclientd/start.go

@@ -81,6 +81,7 @@ func start(_ *cobra.Command, _ []string) error {
 	if strings.Compare(res.GetDefaultNodeInfo().Network, cfg.ChainID) != 0 {
 		startLogger.Warn().Msgf("chain id mismatch, zeta-core chain id %s, zeta client chain id %s; reset zeta client chain id", res.GetDefaultNodeInfo().Network, cfg.ChainID)
 		cfg.ChainID = res.GetDefaultNodeInfo().Network
+		zetaBridge.UpdateChainID(cfg.ChainID)
 	}
 
 	// CreateAuthzSigner : which is used to sign all authz messages . All votes broadcast to zetacore are wrapped in authz exec .
@@ -135,14 +136,22 @@ func start(_ *cobra.Command, _ []string) error {
 			startLogger.Error().Err(err).Msg("telemetryServer error")
 		}
 	}()
+
+	metrics, err := metrics2.NewMetrics()
+	if err != nil {
+		log.Error().Err(err).Msg("NewMetrics")
+		return err
+	}
+	metrics.Start()
+
 	var tssHistoricalList []types.TSS
 	tssHistoricalList, err = zetaBridge.GetTssHistory()
 	if err != nil {
 		startLogger.Error().Err(err).Msg("GetTssHistory error")
 	}
 
 	telemetryServer.SetIPAddress(cfg.PublicIP)
-	tss, err := GenerateTss(masterLogger, cfg, zetaBridge, peers, priKey, telemetryServer, tssHistoricalList)
+	tss, err := GenerateTss(masterLogger, cfg, zetaBridge, peers, priKey, telemetryServer, tssHistoricalList, metrics)
 	if err != nil {
 		return err
 	}
@@ -204,23 +213,9 @@ func start(_ *cobra.Command, _ []string) error {
 		return err
 	}
 
-	metrics, err := metrics2.NewMetrics()
-	if err != nil {
-		log.Error().Err(err).Msg("NewMetrics")
-		return err
-	}
-	metrics.Start()
-
 	userDir, _ := os.UserHomeDir()
 	dbpath := filepath.Join(userDir, ".zetaclient/chainobserver")
 
-	// Register zetaclient.TSS prometheus metrics
-	err = tss.RegisterMetrics(metrics)
-	if err != nil {
-		startLogger.Err(err).Msg("tss.RegisterMetrics")
-		return err
-	}
-
 	// CreateChainClientMap : This creates a map of all chain clients . Each chain client is responsible for listening to events on the chain and processing them
 	chainClientMap, err := CreateChainClientMap(zetaBridge, tss, dbpath, metrics, masterLogger, cfg, telemetryServer)
 	if err != nil {
@@ -244,10 +239,10 @@ func start(_ *cobra.Command, _ []string) error {
 	// stop zetacore observer
 	for _, chain := range cfg.GetEnabledChains() {
 		// zeta chain does not have a chain client
-		if chain.IsZetaChain() {
-			continue
+		if chain.IsExternalChain() {
+			(chainClientMap)[chain].Stop()
 		}
-		(chainClientMap)[chain].Stop()
+
 	}
 	zetaBridge.Stop()
 

zetaclient/query.go

@@ -123,10 +123,9 @@ func (b *ZetaCoreBridge) GetCctxByNonce(chainID int64, nonce uint64) (*types.Cro
 
 func (b *ZetaCoreBridge) GetObserverList(chain common.Chain) ([]string, error) {
 	client := zetaObserverTypes.NewQueryClient(b.grpcConn)
-	resp, err := client.ObserversByChain(context.Background(), &zetaObserverTypes.QueryObserversByChainRequest{
-		ObservationChain: chain.ChainName.String(),
-	})
+	err := error(nil)
 	for i := 0; i <= DefaultRetryCount; i++ {
+		resp, err := client.ObserversByChain(context.Background(), &zetaObserverTypes.QueryObserversByChainRequest{ObservationChain: chain.ChainName.String()})
 		if err == nil {
 			return resp.Observers, nil
 		}
@@ -166,8 +165,9 @@ func (b *ZetaCoreBridge) GetLatestZetaBlock() (*tmtypes.Block, error) {
 
 func (b *ZetaCoreBridge) GetNodeInfo() (*tmservice.GetNodeInfoResponse, error) {
 	client := tmservice.NewServiceClient(b.grpcConn)
-	res, err := client.GetNodeInfo(context.Background(), &tmservice.GetNodeInfoRequest{})
+	err := error(nil)
 	for i := 0; i <= DefaultRetryCount; i++ {
+		res, err := client.GetNodeInfo(context.Background(), &tmservice.GetNodeInfoRequest{})
 		if err == nil {
 			return res, nil
 		}
@@ -215,11 +215,16 @@ func (b *ZetaCoreBridge) GetAllNodeAccounts() ([]*zetaObserverTypes.NodeAccount,
 
 func (b *ZetaCoreBridge) GetKeyGen() (*zetaObserverTypes.Keygen, error) {
 	client := zetaObserverTypes.NewQueryClient(b.grpcConn)
-	resp, err := client.Keygen(context.Background(), &zetaObserverTypes.QueryGetKeygenRequest{})
-	if err != nil {
-		return nil, err
+	err := error(nil)
+	for i := 0; i <= ExtendedRetryCount; i++ {
+		resp, err := client.Keygen(context.Background(), &zetaObserverTypes.QueryGetKeygenRequest{})
+		if err == nil {
+			return resp.Keygen, nil
+		}
+		time.Sleep(DefaultRetryInterval * time.Second)
 	}
-	return resp.Keygen, nil
+	return nil, fmt.Errorf("failed to get keygen | err %s", err.Error())
+
 }
 
 func (b *ZetaCoreBridge) GetCurrentTss() (*types.TSS, error) {

zetaclient/tss_signer.go

@@ -365,7 +365,7 @@ func getKeyAddrBTCWitnessPubkeyHash(tssPubkey string) (*btcutil.AddressWitnessPu
 	return addr, nil
 }
 
-func NewTSS(peer p2p.AddrList, privkey tmcrypto.PrivKey, preParams *keygen.LocalPreParams, cfg *config.Config, bridge *ZetaCoreBridge, tssHistoricalList []types.TSS) (*TSS, error) {
+func NewTSS(peer p2p.AddrList, privkey tmcrypto.PrivKey, preParams *keygen.LocalPreParams, cfg *config.Config, bridge *ZetaCoreBridge, tssHistoricalList []types.TSS, metrics *metrics.Metrics) (*TSS, error) {
 	server, err := SetupTSSServer(peer, privkey, preParams, cfg)
 	if err != nil {
 		return nil, fmt.Errorf("SetupTSSServer error: %w", err)
@@ -390,6 +390,12 @@ func NewTSS(peer p2p.AddrList, privkey tmcrypto.PrivKey, preParams *keygen.Local
 	if err != nil {
 		bridge.logger.Error().Err(err).Msg("VerifyKeysharesForPubkeys fail")
 	}
+	err = newTss.RegisterMetrics(metrics)
+	if err != nil {
+		bridge.logger.Err(err).Msg("tss.RegisterMetrics")
+		return nil, err
+	}
+
 	return &newTss, nil
 }
 

zetaclient/tx.go

@@ -27,6 +27,7 @@ const (
 	DefaultGasLimit                 = 200_000
 	PostProveOutboundTxGasLimit     = 400_000
 	DefaultRetryCount               = 5
+	ExtendedRetryCount              = 15
 	DefaultRetryInterval            = 5
 )
 

zetaclient/zetabridge.go

@@ -116,6 +116,12 @@ func MakeLegacyCodec() *codec.LegacyAmino {
 	return cdc
 }
 
+func (b *ZetaCoreBridge) UpdateChainID(chainID string) {
+	if b.zetaChainID != chainID {
+		b.zetaChainID = chainID
+	}
+}
+
 func (b *ZetaCoreBridge) Stop() {
 	b.logger.Info().Msgf("ZetaBridge is stopping")
 	close(b.stop) // this notifies all configupdater to stop
@@ -146,7 +152,7 @@ func (b *ZetaCoreBridge) WaitForCoreToCreateBlocks() {
 		}
 		retryCount++
 		b.logger.Debug().Msgf("Failed to get latest Block , Retry : %d/%d", retryCount, DefaultRetryCount)
-		if retryCount > DefaultRetryCount {
+		if retryCount > ExtendedRetryCount {
 			panic(fmt.Sprintf("ZetaCore is not ready , Waited for %d seconds", DefaultRetryCount*DefaultRetryInterval))
 		}
 		time.Sleep(DefaultRetryInterval * time.Second)