diff --git a/signer/cosigner_nonce_cache.go b/signer/cosigner_nonce_cache.go index b061e611..4c9e06df 100644 --- a/signer/cosigner_nonce_cache.go +++ b/signer/cosigner_nonce_cache.go @@ -12,6 +12,7 @@ import ( const defaultGetNoncesInterval = 3 * time.Second const defaultGetNoncesTimeout = 4 * time.Second +const cachePreSize = 10000 type CosignerNonceCache struct { logger cometlog.Logger @@ -61,7 +62,7 @@ type NonceCache struct { func NewNonceCache() NonceCache { return NonceCache{ - cache: make(map[uuid.UUID]*CachedNonce, 10000), + cache: make(map[uuid.UUID]*CachedNonce, cachePreSize), } } @@ -98,6 +99,9 @@ type CachedNonce struct { // UUID identifying this collection of nonces UUID uuid.UUID + // Expiration time of this nonce + Expiration time.Time + // Cached nonces, cosigners which have this nonce in their metadata, ready to sign Nonces []CosignerNoncesRel } @@ -130,6 +134,9 @@ func (cnc *CosignerNonceCache) getUuids(n int) []uuid.UUID { } func (cnc *CosignerNonceCache) reconcile(ctx context.Context) { + // prune expired nonces + cnc.pruneNonces() + if !cnc.leader.IsLeader() { return } @@ -191,6 +198,9 @@ func (cnc *CosignerNonceCache) LoadN(ctx context.Context, n int) { nonces := make([]*CachedNonceSingle, len(cnc.cosigners)) var wg sync.WaitGroup wg.Add(len(cnc.cosigners)) + + expiration := time.Now().Add(cnc.getNoncesInterval) + for i, p := range cnc.cosigners { i := i p := p @@ -223,7 +233,8 @@ func (cnc *CosignerNonceCache) LoadN(ctx context.Context, n int) { added := 0 for i, u := range uuids { nonce := CachedNonce{ - UUID: u, + UUID: u, + Expiration: expiration, } num := uint8(0) for _, n := range nonces { @@ -302,6 +313,16 @@ CheckNoncesLoop: return nil, fmt.Errorf("no nonces found involving cosigners %+v", cosignerInts) } +func (cnc *CosignerNonceCache) pruneNonces() { + cnc.cache.mu.Lock() + defer cnc.cache.mu.Unlock() + for u, cn := range cnc.cache.cache { + if time.Now().After(cn.Expiration) { + delete(cnc.cache.cache, u) + } + } +} + func (cnc *CosignerNonceCache) clearNonce(uuid uuid.UUID) { cnc.cache.mu.Lock() defer cnc.cache.mu.Unlock() @@ -334,5 +355,5 @@ func (cnc *CosignerNonceCache) ClearNonces(cosigner Cosigner) { func (cnc *CosignerNonceCache) ClearAllNonces() { cnc.cache.mu.Lock() defer cnc.cache.mu.Unlock() - cnc.cache.cache = make(map[uuid.UUID]*CachedNonce, 10000) + cnc.cache.cache = make(map[uuid.UUID]*CachedNonce, cachePreSize) } diff --git a/signer/cosigner_nonce_cache_test.go b/signer/cosigner_nonce_cache_test.go index 393eeb4d..c7ac2586 100644 --- a/signer/cosigner_nonce_cache_test.go +++ b/signer/cosigner_nonce_cache_test.go @@ -28,7 +28,7 @@ func TestNonceCacheDemand(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) - nonceCache.LoadN(ctx, 1000) + nonceCache.LoadN(ctx, 500) go nonceCache.Start(ctx) diff --git a/signer/local_cosigner.go b/signer/local_cosigner.go index 09d4ee3f..38404a30 100644 --- a/signer/local_cosigner.go +++ b/signer/local_cosigner.go @@ -16,6 +16,8 @@ import ( var _ Cosigner = &LocalCosigner{} +const nonceExpiration = 10 * time.Second + // LocalCosigner responds to sign requests. // It maintains a high watermark to avoid double-signing. // Signing is thread safe. @@ -27,7 +29,7 @@ type LocalCosigner struct { address string pendingDiskWG sync.WaitGroup - nonces map[uuid.UUID][]Nonces + nonces map[uuid.UUID]*NoncesWithExpiration // protects the nonces map noncesMu sync.RWMutex } @@ -43,7 +45,7 @@ func NewLocalCosigner( config: config, security: security, address: address, - nonces: make(map[uuid.UUID][]Nonces), + nonces: make(map[uuid.UUID]*NoncesWithExpiration), } } @@ -55,6 +57,31 @@ type ChainState struct { signer ThresholdSigner } +// StartNoncePruner periodically prunes nonces that have expired. +func (cosigner *LocalCosigner) StartNoncePruner(ctx context.Context) { + ticker := time.NewTicker(nonceExpiration / 4) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + cosigner.pruneNonces() + } + } +} + +// pruneNonces removes nonces that have expired. +func (cosigner *LocalCosigner) pruneNonces() { + cosigner.noncesMu.Lock() + defer cosigner.noncesMu.Unlock() + now := time.Now() + for uuid, nonces := range cosigner.nonces { + if now.After(nonces.Expiration) { + delete(cosigner.nonces, uuid) + } + } +} + func (cosigner *LocalCosigner) combinedNonces(myID int, threshold uint8, uuid uuid.UUID) ([]Nonce, error) { cosigner.noncesMu.RLock() defer cosigner.noncesMu.RUnlock() @@ -67,7 +94,7 @@ func (cosigner *LocalCosigner) combinedNonces(myID int, threshold uint8, uuid uu combinedNonces := make([]Nonce, 0, threshold) // calculate secret and public keys - for _, c := range nonces { + for _, c := range nonces.Nonces { if len(c.Shares) == 0 || len(c.Shares[myID-1]) == 0 { continue } @@ -289,6 +316,7 @@ func (cosigner *LocalCosigner) LoadSignStateIfNecessary(chainID string) error { return nil } +// GetNonces returns the nonces for the given UUIDs, generating if necessary. func (cosigner *LocalCosigner) GetNonces( _ context.Context, uuids []uuid.UUID, @@ -355,7 +383,7 @@ func (cosigner *LocalCosigner) GetNonces( return res, nil } -func (cosigner *LocalCosigner) generateNoncesIfNecessary(uuid uuid.UUID) ([]Nonces, error) { +func (cosigner *LocalCosigner) generateNoncesIfNecessary(uuid uuid.UUID) (*NoncesWithExpiration, error) { // protects the meta map cosigner.noncesMu.Lock() defer cosigner.noncesMu.Unlock() @@ -369,8 +397,13 @@ func (cosigner *LocalCosigner) generateNoncesIfNecessary(uuid uuid.UUID) ([]Nonc return nil, err } - cosigner.nonces[uuid] = newNonces - return newNonces, nil + res := NoncesWithExpiration{ + Nonces: newNonces, + Expiration: time.Now().Add(nonceExpiration), + } + + cosigner.nonces[uuid] = &res + return &res, nil } // Get the ephemeral secret part for an ephemeral share @@ -388,7 +421,7 @@ func (cosigner *LocalCosigner) getNonce( return zero, err } - ourCosignerMeta := meta[id-1] + ourCosignerMeta := meta.Nonces[id-1] nonce, err := cosigner.security.EncryptAndSign(peerID, ourCosignerMeta.PubKey, ourCosignerMeta.Shares[peerID-1]) if err != nil { return zero, err @@ -414,7 +447,7 @@ func (cosigner *LocalCosigner) setNonce(uuid uuid.UUID, nonce CosignerNonce) err cosigner.noncesMu.Lock() defer cosigner.noncesMu.Unlock() - nonces, ok := cosigner.nonces[uuid] + n, ok := cosigner.nonces[uuid] // generate metadata placeholder if !ok { return fmt.Errorf( @@ -424,11 +457,11 @@ func (cosigner *LocalCosigner) setNonce(uuid uuid.UUID, nonce CosignerNonce) err } // set slot - if nonces[nonce.SourceID-1].Shares == nil { - nonces[nonce.SourceID-1].Shares = make([][]byte, len(cosigner.config.Config.ThresholdModeConfig.Cosigners)) + if n.Nonces[nonce.SourceID-1].Shares == nil { + n.Nonces[nonce.SourceID-1].Shares = make([][]byte, len(cosigner.config.Config.ThresholdModeConfig.Cosigners)) } - nonces[nonce.SourceID-1].Shares[cosigner.GetID()-1] = nonceShare - nonces[nonce.SourceID-1].PubKey = noncePub + n.Nonces[nonce.SourceID-1].Shares[cosigner.GetID()-1] = nonceShare + n.Nonces[nonce.SourceID-1].PubKey = noncePub return nil } diff --git a/signer/threshold_signer.go b/signer/threshold_signer.go index b404ad10..f8e9ef34 100644 --- a/signer/threshold_signer.go +++ b/signer/threshold_signer.go @@ -1,5 +1,7 @@ package signer +import "time" + // Interface for the local signer whether it's a soft sign or HSM type ThresholdSigner interface { // PubKey returns the public key bytes for the combination of all cosigners. @@ -18,6 +20,11 @@ type Nonces struct { Shares [][]byte } +type NoncesWithExpiration struct { + Expiration time.Time + Nonces []Nonces +} + // Nonce is the ephemeral information from another cosigner destined for this cosigner. type Nonce struct { ID int diff --git a/signer/threshold_validator.go b/signer/threshold_validator.go index 6d4c5bcb..11eb920d 100644 --- a/signer/threshold_validator.go +++ b/signer/threshold_validator.go @@ -106,6 +106,8 @@ func (pv *ThresholdValidator) Start(ctx context.Context) error { go pv.nonceCache.Start(ctx) + go pv.myCosigner.StartNoncePruner(ctx) + return nil }