feat: V16 - add metric for concurrent key signs

changelog.md

@@ -27,6 +27,7 @@
 * [1755](https://github.com/zeta-chain/node/issues/1755) - use evm JSON RPC for inbound tx (including blob tx) observation.
 * [1815](https://github.com/zeta-chain/node/pull/1815) - add authority module for authorized actions
 * [1884](https://github.com/zeta-chain/node/pull/1884) - added zetatool cmd, added subcommand to filter deposits
+* [1954](https://github.com/zeta-chain/node/pull/1954) - add metric for concurrent keysigns
 
 ### Tests
 

zetaclient/metrics/metrics.go

@@ -78,6 +78,12 @@ var (
 		Name:      "last_start_timestamp_seconds",
 		Help:      "Start time in Unix time",
 	})
+
+	NumActiveMsgSigns = promauto.NewGauge(prometheus.GaugeOpts{
+		Namespace: ZetaClientNamespace,
+		Name:      "num_active_message_signs",
+		Help:      "Number of concurrent key signs",
+	})
 )
 
 func NewMetrics() (*Metrics, error) {

zetaclient/tss/tss_keysigns_tracker.go

@@ -0,0 +1,49 @@
+package tss
+
+import (
+	"sync"
+
+	"github.com/rs/zerolog"
+	"github.com/zeta-chain/zetacore/zetaclient/metrics"
+)
+
+// ConcurrentKeysignsTracker keeps track of concurrent keysigns performed by go-tss
+type ConcurrentKeysignsTracker struct {
+	numActiveMsgSigns int64
+	mu                sync.Mutex
+	Logger            zerolog.Logger
+}
+
+// NewKeysignsTracker - constructor
+func NewKeysignsTracker(logger zerolog.Logger) *ConcurrentKeysignsTracker {
+	return &ConcurrentKeysignsTracker{
+		numActiveMsgSigns: 0,
+		mu:                sync.Mutex{},
+		Logger:            logger.With().Str("submodule", "ConcurrentKeysignsTracker").Logger(),
+	}
+}
+
+// StartMsgSign is incrementing the number of active signing ceremonies as well as updating the prometheus metric
+func (k *ConcurrentKeysignsTracker) StartMsgSign() {
+	k.mu.Lock()
+	defer k.mu.Unlock()
+	k.numActiveMsgSigns++
+	metrics.NumActiveMsgSigns.Inc()
+	k.Logger.Debug().Msgf("Start TSS message sign, numActiveMsgSigns: %d", k.numActiveMsgSigns)
+}
+
+// EndMsgSign is decrementing the number of active signing ceremonies as well as updating the prometheus metric
+func (k *ConcurrentKeysignsTracker) EndMsgSign() {
+	k.mu.Lock()
+	defer k.mu.Unlock()
+	if k.numActiveMsgSigns > 0 {
+		k.numActiveMsgSigns--
+		metrics.NumActiveMsgSigns.Dec()
+	}
+	k.Logger.Debug().Msgf("End TSS message sign, numActiveMsgSigns: %d", k.numActiveMsgSigns)
+}
+
+// GetNumActiveMessageSigns gets the current number of active signing ceremonies
+func (k *ConcurrentKeysignsTracker) GetNumActiveMessageSigns() int64 {
+	return k.numActiveMsgSigns
+}

zetaclient/tss/tss_keysigns_tracker_test.go

@@ -0,0 +1,27 @@
+package tss
+
+import (
+	"testing"
+
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/require"
+)
+
+func TestKeySignManager_StartMsgSign(t *testing.T) {
+	ksman := NewKeysignsTracker(zerolog.Logger{})
+	ksman.StartMsgSign()
+	ksman.StartMsgSign()
+	ksman.StartMsgSign()
+	ksman.StartMsgSign()
+	require.Equal(t, int64(4), ksman.GetNumActiveMessageSigns())
+}
+
+func TestKeySignManager_EndMsgSign(t *testing.T) {
+	ksman := NewKeysignsTracker(zerolog.Logger{})
+	ksman.StartMsgSign()
+	ksman.StartMsgSign()
+	ksman.EndMsgSign()
+	ksman.EndMsgSign()
+	ksman.EndMsgSign()
+	require.Equal(t, int64(0), ksman.GetNumActiveMessageSigns())
+}

zetaclient/tss/tss_signer.go

@@ -68,12 +68,13 @@ var _ interfaces.TSSSigner = (*TSS)(nil)
 
 // TSS is a struct that holds the server and the keys for TSS
 type TSS struct {
-	Server        *tss.TssServer
-	Keys          map[string]*Key // PubkeyInBech32 => TSSKey
-	CurrentPubkey string
-	logger        zerolog.Logger
-	Signers       []string
-	CoreBridge    interfaces.ZetaCoreBridger
+	Server          *tss.TssServer
+	Keys            map[string]*Key // PubkeyInBech32 => TSSKey
+	CurrentPubkey   string
+	logger          zerolog.Logger
+	Signers         []string
+	CoreBridge      interfaces.ZetaCoreBridger
+	KeysignsTracker *ConcurrentKeysignsTracker
 
 	// TODO: support multiple Bitcoin network, not just one network
 	// https://github.com/zeta-chain/node/issues/1397
@@ -92,17 +93,19 @@ func NewTSS(
 	tssPassword string,
 	hotkeyPassword string,
 ) (*TSS, error) {
+	logger := log.With().Str("module", "tss_signer").Logger()
 	server, err := SetupTSSServer(peer, privkey, preParams, appContext.Config(), tssPassword)
 	if err != nil {
 		return nil, fmt.Errorf("SetupTSSServer error: %w", err)
 	}
 	newTss := TSS{
-		Server:         server,
-		Keys:           make(map[string]*Key),
-		CurrentPubkey:  appContext.ZetaCoreContext().GetCurrentTssPubkey(),
-		logger:         log.With().Str("module", "tss_signer").Logger(),
-		CoreBridge:     bridge,
-		BitcoinChainID: bitcoinChainID,
+		Server:          server,
+		Keys:            make(map[string]*Key),
+		CurrentPubkey:   appContext.ZetaCoreContext().GetCurrentTssPubkey(),
+		logger:          logger,
+		CoreBridge:      bridge,
+		KeysignsTracker: NewKeysignsTracker(logger),
+		BitcoinChainID:  bitcoinChainID,
 	}
 
 	err = newTss.LoadTssFilesFromDirectory(appContext.Config().TssPath)
@@ -121,9 +124,12 @@ func NewTSS(
 	if err != nil {
 		return nil, err
 	}
+	// Initialize metrics
 	for _, key := range keygenRes.GranteePubkeys {
 		metrics.TssNodeBlamePerPubKey.WithLabelValues(key).Inc()
 	}
+	metrics.NumActiveMsgSigns.Set(0)
+
 	return &newTss, nil
 }
 
@@ -203,7 +209,9 @@ func (tss *TSS) Sign(digest []byte, height uint64, nonce uint64, chain *common.C
 	}
 	// #nosec G701 always in range
 	keysignReq := keysign.NewRequest(tssPubkey, []string{base64.StdEncoding.EncodeToString(H)}, int64(height), nil, "0.14.0")
+	tss.KeysignsTracker.StartMsgSign()
 	ksRes, err := tss.Server.KeySign(keysignReq)
+	tss.KeysignsTracker.EndMsgSign()
 	if err != nil {
 		log.Warn().Msg("keysign fail")
 	}
@@ -271,8 +279,9 @@ func (tss *TSS) SignBatch(digests [][]byte, height uint64, nonce uint64, chain *
 	}
 	// #nosec G701 always in range
 	keysignReq := keysign.NewRequest(tssPubkey, digestBase64, int64(height), nil, "0.14.0")
-
+	tss.KeysignsTracker.StartMsgSign()
 	ksRes, err := tss.Server.KeySign(keysignReq)
+	tss.KeysignsTracker.EndMsgSign()
 	if err != nil {
 		log.Warn().Err(err).Msg("keysign fail")
 	}