update bucket sync status when refreshing the cache (#5551)

* update bucket sync status when refreshing the cache Signed-off-by: Alan Protasio <[email protected]> * fix datarace Signed-off-by: Alan Protasio <[email protected]> --------- Signed-off-by: Alan Protasio <[email protected]>
cortexproject · Sep 7, 2023 · 53bc491 · 53bc491
1 parent 94192ff
commit 53bc491
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 4 deletions.
diff --git a/pkg/storage/tsdb/bucketindex/loader.go b/pkg/storage/tsdb/bucketindex/loader.go
@@ -96,12 +96,13 @@ func (l *Loader) GetIndex(ctx context.Context, userID string) (*Index, Status, e
 	if entry := l.indexes[userID]; entry != nil {
 		idx := entry.index
 		err := entry.err
+		ss := entry.syncStatus
 		l.indexesMx.RUnlock()
 
 		// We don't check if the index is stale because it's the responsibility
 		// of the background job to keep it updated.
 		entry.requestedAt.Store(time.Now().Unix())
-		return idx, entry.syncStatus, err
+		return idx, ss, err
 	}
 	l.indexesMx.RUnlock()
 
@@ -203,6 +204,16 @@ func (l *Loader) updateCachedIndex(ctx context.Context, userID string) {
 
 	l.loadAttempts.Inc()
 	startTime := time.Now()
+	ss, err := ReadSyncStatus(ctx, l.bkt, userID, l.logger)
+	if err != nil {
+		level.Warn(l.logger).Log("msg", "unable to read bucket index status", "user", userID, "err", err)
+	}
+
+	// Update Sync Status
+	l.indexesMx.Lock()
+	l.indexes[userID].syncStatus = ss
+	l.indexesMx.Unlock()
+
 	idx, err := ReadIndex(readCtx, l.bkt, userID, l.cfgProvider, l.logger)
 	if err != nil && !errors.Is(err, ErrIndexNotFound) && !errors.Is(err, bucket.ErrCustomerManagedKeyAccessDenied) {
 		l.loadFailures.Inc()
@@ -212,7 +223,7 @@ func (l *Loader) updateCachedIndex(ctx context.Context, userID string) {
 
 	l.loadDuration.Observe(time.Since(startTime).Seconds())
 
-	// We cache it either it was successfully refreshed or wasn't found. An use case for caching the ErrIndexNotFound
+	// We cache it either it was successfully refreshed,  wasn't found or when is a CMK error. An use case for caching the ErrIndexNotFound
 	// is when a tenant has rules configured but hasn't started remote writing yet. Rules will be evaluated and
 	// bucket index loaded by the ruler.
 	l.indexesMx.Lock()

diff --git a/pkg/storage/tsdb/bucketindex/loader_test.go b/pkg/storage/tsdb/bucketindex/loader_test.go
@@ -604,15 +604,16 @@ func TestLoader_ShouldUpdateIndexInBackgroundOnPreviousKeyAccessDenied(t *testin
 
 	_, ss, err := loader.GetIndex(ctx, user)
 	require.True(t, errors.Is(err, bucket.ErrCustomerManagedKeyAccessDenied))
+	// SyncStatus does not exists
 	require.Equal(t, Unknown, ss.Status)
 
 	// Verify is the index sync status is being returned
 	ss.Status = CustomerManagedKeyError
 	ss.NonQueryableReason = CustomerManagedKeyError
 	WriteSyncStatus(ctx, bkt, user, ss, log.NewNopLogger())
 
-	// Check not cached
-	loader.deleteCachedIndex(user)
+	// Update the index with the new sync status
+	require.NoError(t, loader.checkCachedIndexes(ctx))
 	_, ss, err = loader.GetIndex(ctx, user)
 	require.True(t, errors.Is(err, bucket.ErrCustomerManagedKeyAccessDenied))
 	require.Equal(t, CustomerManagedKeyError, ss.Status)