Cleanup dangling request queue metrics (#6433)

* cleanup dangling request queue metrics Signed-off-by: Ben Ye <[email protected]> * fix lint Signed-off-by: Ben Ye <[email protected]> --------- Signed-off-by: Ben Ye <[email protected]>
cortexproject · Dec 17, 2024 · 3269f9d · 3269f9d
1 parent b72a536
commit 3269f9d
Show file tree

Hide file tree

Showing 5 changed files with 20 additions and 4 deletions.
diff --git a/pkg/frontend/v1/frontend.go b/pkg/frontend/v1/frontend.go
@@ -183,6 +183,7 @@ func (f *Frontend) cleanupInactiveUserMetrics(user string) {
 	f.discardedRequests.DeletePartialMatch(prometheus.Labels{
 		"user": user,
 	})
+	f.requestQueue.CleanupInactiveUserMetrics(user)
 }
 
 // RoundTripGRPC round trips a proto (instead of a HTTP request).

diff --git a/pkg/frontend/v1/frontend_test.go b/pkg/frontend/v1/frontend_test.go
@@ -212,11 +212,14 @@ func TestFrontendMetricsCleanup(t *testing.T) {
 				# HELP cortex_query_frontend_queue_length Number of queries in the queue.
 				# TYPE cortex_query_frontend_queue_length gauge
 				cortex_query_frontend_queue_length{priority="0",type="fifo",user="1"} 0
-			`), "cortex_query_frontend_queue_length"))
+				# HELP cortex_request_queue_requests_total Total number of query requests going to the request queue.
+				# TYPE cortex_request_queue_requests_total counter
+				cortex_request_queue_requests_total{priority="0",user="1"} 1
+			`), "cortex_query_frontend_queue_length", "cortex_request_queue_requests_total"))
 
 			fr.cleanupInactiveUserMetrics("1")
 
-			require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(""), "cortex_query_frontend_queue_length"))
+			require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(""), "cortex_query_frontend_queue_length", "cortex_request_queue_requests_total"))
 		}
 
 		testFrontend(t, defaultFrontendConfig(), handler, test, matchMaxConcurrency, nil, reg)

diff --git a/pkg/scheduler/queue/queue.go b/pkg/scheduler/queue/queue.go
@@ -247,3 +247,7 @@ func (q *RequestQueue) QuerierDisconnecting() {
 func (q *RequestQueue) GetConnectedQuerierWorkersMetric() float64 {
 	return float64(q.connectedQuerierWorkers.Load())
 }
+
+func (q *RequestQueue) CleanupInactiveUserMetrics(user string) {
+	q.totalRequests.DeletePartialMatch(prometheus.Labels{"user": user})
+}
diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go
@@ -532,6 +532,7 @@ func (s *Scheduler) cleanupMetricsForInactiveUser(user string) {
 	s.discardedRequests.DeletePartialMatch(prometheus.Labels{
 		"user": user,
 	})
+	s.requestQueue.CleanupInactiveUserMetrics(user)
 }
 
 func (s *Scheduler) getConnectedFrontendClientsMetric() float64 {

diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go
@@ -430,15 +430,22 @@ func TestSchedulerMetrics(t *testing.T) {
 		# TYPE cortex_query_scheduler_queue_length gauge
 		cortex_query_scheduler_queue_length{priority="0",type="fifo",user="another"} 1
 		cortex_query_scheduler_queue_length{priority="0",type="fifo",user="test"} 1
-	`), "cortex_query_scheduler_queue_length"))
+		# HELP cortex_request_queue_requests_total Total number of query requests going to the request queue.
+		# TYPE cortex_request_queue_requests_total counter
+		cortex_request_queue_requests_total{priority="0",user="another"} 1
+		cortex_request_queue_requests_total{priority="0",user="test"} 1
+	`), "cortex_query_scheduler_queue_length", "cortex_request_queue_requests_total"))
 
 	scheduler.cleanupMetricsForInactiveUser("test")
 
 	require.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(`
 		# HELP cortex_query_scheduler_queue_length Number of queries in the queue.
 		# TYPE cortex_query_scheduler_queue_length gauge
 		cortex_query_scheduler_queue_length{priority="0",type="fifo",user="another"} 1
-	`), "cortex_query_scheduler_queue_length"))
+		# HELP cortex_request_queue_requests_total Total number of query requests going to the request queue.
+		# TYPE cortex_request_queue_requests_total counter
+		cortex_request_queue_requests_total{priority="0",user="another"} 1
+	`), "cortex_query_scheduler_queue_length", "cortex_request_queue_requests_total"))
 }
 
 func initFrontendLoop(t *testing.T, client schedulerpb.SchedulerForFrontendClient, frontendAddr string) schedulerpb.SchedulerForFrontend_FrontendLoopClient {