Skip to content

Commit

Permalink
Merge branch 'main' into andrew/mitigate_unexpected_state
Browse files Browse the repository at this point in the history
  • Loading branch information
@agouin
agouin authored Dec 12, 2023
2 parents 67dbdd1 + 60760f2 commit 407fab7
Show file tree
Hide file tree
Showing 2 changed files with 211 additions and 43 deletions.
51 changes: 27 additions & 24 deletions signer/cosigner_nonce_cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ type CosignerNonceCache struct {

threshold uint8

cache NonceCache
cache *NonceCache

pruner NonceCachePruner

Expand Down Expand Up @@ -112,6 +112,30 @@ func (nc *NonceCache) Delete(index int) {
nc.cache = append(nc.cache[:index], nc.cache[index+1:]...)
}

func (nc *NonceCache) PruneNonces() int {
nc.mu.Lock()
defer nc.mu.Unlock()
nonExpiredIndex := -1
for i := 0; i < len(nc.cache); i++ {
if time.Now().Before(nc.cache[i].Expiration) {
nonExpiredIndex = i
break
}
}

var deleteCount int
if nonExpiredIndex == -1 {
// No non-expired nonces, delete everything
deleteCount = len(nc.cache)
nc.cache = nil
} else {
// Prune everything up to the non-expired nonce
deleteCount = nonExpiredIndex
nc.cache = nc.cache[nonExpiredIndex:]
}
return deleteCount
}

type CosignerNoncesRel struct {
Cosigner Cosigner
Nonces CosignerNonces
Expand Down Expand Up @@ -152,13 +176,14 @@ func NewCosignerNonceCache(
nonceExpiration: nonceExpiration,
threshold: threshold,
pruner: pruner,
cache: new(NonceCache),
// buffer up to 1000 empty events so that we don't ever block
empty: make(chan struct{}, 1000),
movingAverage: newMovingAverage(4 * getNoncesInterval), // weighted average over 4 intervals
}
// the only time pruner is expected to be non-nil is during tests, otherwise we use the cache logic.
if pruner == nil {
cnc.pruner = cnc
cnc.pruner = cnc.cache
}

return cnc
Expand Down Expand Up @@ -371,28 +396,6 @@ CheckNoncesLoop:
return nil, fmt.Errorf("no nonces found involving cosigners %+v", cosignerInts)
}

func (cnc *CosignerNonceCache) PruneNonces() int {
cnc.cache.mu.Lock()
defer cnc.cache.mu.Unlock()
nonExpiredIndex := len(cnc.cache.cache) - 1
for i := len(cnc.cache.cache) - 1; i >= 0; i-- {
if time.Now().Before(cnc.cache.cache[i].Expiration) {
nonExpiredIndex = i
break
}
if i == 0 {
deleteCount := len(cnc.cache.cache)
cnc.cache.cache = nil
return deleteCount
}
}
deleteCount := len(cnc.cache.cache) - nonExpiredIndex - 1
if nonExpiredIndex != len(cnc.cache.cache)-1 {
cnc.cache.cache = cnc.cache.cache[:nonExpiredIndex+1]
}
return deleteCount
}

func (cnc *CosignerNonceCache) ClearNonces(cosigner Cosigner) {
cnc.cache.mu.Lock()
defer cnc.cache.mu.Unlock()
Expand Down
203 changes: 184 additions & 19 deletions signer/cosigner_nonce_cache_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ func TestClearNonces(t *testing.T) {

cnc := CosignerNonceCache{
threshold: 2,
cache: new(NonceCache),
}

for i := 0; i < 10; i++ {
Expand Down Expand Up @@ -118,14 +119,14 @@ func TestClearNonces(t *testing.T) {
}

type mockPruner struct {
cnc *CosignerNonceCache
cache *NonceCache
count int
pruned int
mu sync.Mutex
}

func (mp *mockPruner) PruneNonces() int {
pruned := mp.cnc.PruneNonces()
pruned := mp.cache.PruneNonces()
mp.mu.Lock()
defer mp.mu.Unlock()
mp.count++
Expand Down Expand Up @@ -159,7 +160,7 @@ func TestNonceCacheDemand(t *testing.T) {
mp,
)

mp.cnc = nonceCache
mp.cache = nonceCache.cache

ctx, cancel := context.WithCancel(context.Background())

Expand All @@ -184,8 +185,8 @@ func TestNonceCacheDemand(t *testing.T) {

count, pruned := mp.Result()

require.Greater(t, count, 0)
require.Equal(t, 0, pruned)
require.Greater(t, count, 0, "count of pruning calls must be greater than 0")
require.Equal(t, 0, pruned, "no nonces should have been pruned")
}

func TestNonceCacheExpiration(t *testing.T) {
Expand All @@ -197,42 +198,206 @@ func TestNonceCacheExpiration(t *testing.T) {

mp := &mockPruner{}

noncesExpiration := 1000 * time.Millisecond
getNoncesInterval := noncesExpiration / 5
getNoncesTimeout := 10 * time.Millisecond
nonceCache := NewCosignerNonceCache(
cometlog.NewTMLogger(cometlog.NewSyncWriter(os.Stdout)),
cosigners,
&MockLeader{id: 1, leader: &ThresholdValidator{myCosigner: lcs[0]}},
250*time.Millisecond,
10*time.Millisecond,
500*time.Millisecond,
getNoncesInterval,
getNoncesTimeout,
noncesExpiration,
2,
mp,
)

mp.cnc = nonceCache
mp.cache = nonceCache.cache

ctx, cancel := context.WithCancel(context.Background())

const loadN = 500

const loadN = 100
// Load first set of 100 nonces
nonceCache.LoadN(ctx, loadN)

go nonceCache.Start(ctx)

time.Sleep(1 * time.Second)
// Sleep for 1/2 nonceExpiration, no nonces should have expired yet
time.Sleep(noncesExpiration / 2)

// Load second set of 100 nonces
nonceCache.LoadN(ctx, loadN)

// Wait for first set of nonces to expire + wait for the interval to have run
time.Sleep((noncesExpiration / 2) + getNoncesInterval)

count, pruned := mp.Result()

// we should have pruned at least three times after
// waiting for a second with a reconcile interval of 250ms
require.GreaterOrEqual(t, count, 3)
// we should have pruned at least 5 times after
// waiting for 1200ms with a reconcile interval of 200ms
require.GreaterOrEqual(t, count, 5)

// we should have pruned at least the number of nonces we loaded and knew would expire
require.GreaterOrEqual(t, pruned, loadN)
// we should have pruned only the first set of nonces
// The second set of nonces should not have expired yet and we should not have load any more
require.Equal(t, pruned, loadN)

cancel()

// the cache should be empty or 1 since no nonces are being consumed.
require.LessOrEqual(t, nonceCache.cache.Size(), 1)
// the cache should be 100 (loadN) as the second set should not have expired.
require.LessOrEqual(t, nonceCache.cache.Size(), loadN)
}

func TestNonceCachePrune(t *testing.T) {
type testCase struct {
name string
nonces []*CachedNonce
expected []*CachedNonce
}

now := time.Now()

testCases := []testCase{
{
name: "no nonces",
nonces: nil,
expected: nil,
},
{
name: "no expired nonces",
nonces: []*CachedNonce{
{
UUID: uuid.MustParse("d6ef381f-6234-432d-b204-d8957fe60360"),
Expiration: now.Add(1 * time.Second),
},
{
UUID: uuid.MustParse("cdc3673d-7946-459a-b458-cbbde0eecd04"),
Expiration: now.Add(2 * time.Second),
},
{
UUID: uuid.MustParse("38c6a201-0b8b-46eb-ab69-c7b2716d408e"),
Expiration: now.Add(3 * time.Second),
},
{
UUID: uuid.MustParse("5caf5ab2-d460-430f-87fa-8ed2983ae8fb"),
Expiration: now.Add(4 * time.Second),
},
},
expected: []*CachedNonce{
{
UUID: uuid.MustParse("d6ef381f-6234-432d-b204-d8957fe60360"),
Expiration: now.Add(1 * time.Second),
},
{
UUID: uuid.MustParse("cdc3673d-7946-459a-b458-cbbde0eecd04"),
Expiration: now.Add(2 * time.Second),
},
{
UUID: uuid.MustParse("38c6a201-0b8b-46eb-ab69-c7b2716d408e"),
Expiration: now.Add(3 * time.Second),
},
{
UUID: uuid.MustParse("5caf5ab2-d460-430f-87fa-8ed2983ae8fb"),
Expiration: now.Add(4 * time.Second),
},
},
},
{
name: "first nonce is expired",
nonces: []*CachedNonce{
{
UUID: uuid.MustParse("d6ef381f-6234-432d-b204-d8957fe60360"),
Expiration: now.Add(-1 * time.Second),
},
{
UUID: uuid.MustParse("cdc3673d-7946-459a-b458-cbbde0eecd04"),
Expiration: now.Add(2 * time.Second),
},
{
UUID: uuid.MustParse("38c6a201-0b8b-46eb-ab69-c7b2716d408e"),
Expiration: now.Add(3 * time.Second),
},
{
UUID: uuid.MustParse("5caf5ab2-d460-430f-87fa-8ed2983ae8fb"),
Expiration: now.Add(4 * time.Second),
},
},
expected: []*CachedNonce{
{
UUID: uuid.MustParse("cdc3673d-7946-459a-b458-cbbde0eecd04"),
Expiration: now.Add(2 * time.Second),
},
{
UUID: uuid.MustParse("38c6a201-0b8b-46eb-ab69-c7b2716d408e"),
Expiration: now.Add(3 * time.Second),
},
{
UUID: uuid.MustParse("5caf5ab2-d460-430f-87fa-8ed2983ae8fb"),
Expiration: now.Add(4 * time.Second),
},
},
},
{
name: "all but last nonce expired",
nonces: []*CachedNonce{
{
UUID: uuid.MustParse("d6ef381f-6234-432d-b204-d8957fe60360"),
Expiration: now.Add(-1 * time.Second),
},
{
UUID: uuid.MustParse("cdc3673d-7946-459a-b458-cbbde0eecd04"),
Expiration: now.Add(-1 * time.Second),
},
{
UUID: uuid.MustParse("38c6a201-0b8b-46eb-ab69-c7b2716d408e"),
Expiration: now.Add(-1 * time.Second),
},
{
UUID: uuid.MustParse("5caf5ab2-d460-430f-87fa-8ed2983ae8fb"),
Expiration: now.Add(4 * time.Second),
},
},
expected: []*CachedNonce{
{
UUID: uuid.MustParse("5caf5ab2-d460-430f-87fa-8ed2983ae8fb"),
Expiration: now.Add(4 * time.Second),
},
},
},
{
name: "all nonces expired",
nonces: []*CachedNonce{
{
UUID: uuid.MustParse("d6ef381f-6234-432d-b204-d8957fe60360"),
Expiration: now.Add(-1 * time.Second),
},
{
UUID: uuid.MustParse("cdc3673d-7946-459a-b458-cbbde0eecd04"),
Expiration: now.Add(-1 * time.Second),
},
{
UUID: uuid.MustParse("38c6a201-0b8b-46eb-ab69-c7b2716d408e"),
Expiration: now.Add(-1 * time.Second),
},
{
UUID: uuid.MustParse("5caf5ab2-d460-430f-87fa-8ed2983ae8fb"),
Expiration: now.Add(-1 * time.Second),
},
},
expected: nil,
},
}

for _, tc := range testCases {
nc := NonceCache{
cache: tc.nonces,
}

pruned := nc.PruneNonces()

require.Equal(t, len(tc.nonces)-len(tc.expected), pruned, tc.name)

require.Equal(t, tc.expected, nc.cache, tc.name)
}
}

func TestNonceCacheDemandSlow(t *testing.T) {
Expand Down

0 comments on commit 407fab7

Please sign in to comment.