store-gateway: record request ambient time for LabelNames and LabelVa…

…lues (#7622) * store-gateway: record request ambient time for LabelNames and LabelValues The ambient time (`queryStats.streamingSeriesAmbientTime`) is used to observe the `"other"` stage duration in `BucketStore.recordStreamingSeriesStats` after running LabelNames, LabelValues and Series requests. But `queryStats.streamingSeriesAmbientTime` is only populated for `Series` requests. This results in skewed `"other"` timings. Signed-off-by: Dimitar Dimitrov <[email protected]> * Update CHANGELOG.md entry Signed-off-by: Dimitar Dimitrov <[email protected]> * Fix linter Signed-off-by: Dimitar Dimitrov <[email protected]> --------- Signed-off-by: Dimitar Dimitrov <[email protected]>
grafana · Mar 15, 2024 · 3ebb958 · 3ebb958
1 parent 79c6155
commit 3ebb958
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 5 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,7 @@
 * [BUGFIX] Querier, store-gateway: Protect against panics raised during snappy encoding. #7520
 * [BUGFIX] Ingester: Prevent timely compaction of empty blocks. #7624
 * [BUGFIX] querier: Don't cache context.Canceled errors for bucket index. #7620
+* [BUGFIX] Store-gateway: account for `"other"` time in LabelValues and LabelNames requests. #7622
 
 ### Mixin
 

diff --git a/pkg/storegateway/bucket.go b/pkg/storegateway/bucket.go
@@ -577,11 +577,7 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storegatewaypb.Stor
 		reqBlockMatchers []*labels.Matcher
 	)
 	defer s.recordSeriesCallResult(stats)
-	defer func(requestStart time.Time) {
-		stats.update(func(stats *queryStats) {
-			stats.streamingSeriesAmbientTime += time.Since(requestStart)
-		})
-	}(time.Now())
+	defer s.recordRequestAmbientTime(stats, time.Now())
 
 	if req.Hints != nil {
 		reqHints := &hintspb.SeriesRequestHints{}
@@ -713,6 +709,12 @@ func (s *BucketStore) Series(req *storepb.SeriesRequest, srv storegatewaypb.Stor
 	return nil
 }
 
+func (s *BucketStore) recordRequestAmbientTime(stats *safeQueryStats, requestStart time.Time) {
+	stats.update(func(stats *queryStats) {
+		stats.streamingSeriesAmbientTime += time.Since(requestStart)
+	})
+}
+
 func (s *BucketStore) limitConcurrentQueries(ctx context.Context, stats *safeQueryStats) (done func(), err error) {
 	span, spanCtx := opentracing.StartSpanFromContext(ctx, "store_query_gate_ismyturn")
 	defer span.Finish()
@@ -1333,6 +1335,7 @@ func (s *BucketStore) LabelNames(ctx context.Context, req *storepb.LabelNamesReq
 	)
 
 	defer s.recordLabelNamesCallResult(stats)
+	defer s.recordRequestAmbientTime(stats, time.Now())
 
 	var reqBlockMatchers []*labels.Matcher
 	if req.Hints != nil {
@@ -1525,6 +1528,7 @@ func (s *BucketStore) LabelValues(ctx context.Context, req *storepb.LabelValuesR
 
 	stats := newSafeQueryStats()
 	defer s.recordLabelValuesCallResult(stats)
+	defer s.recordRequestAmbientTime(stats, time.Now())
 
 	resHints := &hintspb.LabelValuesResponseHints{}