Cleanup dangling request queue metrics

pkg/frontend/v1/frontend.go

@@ -183,6 +183,7 @@ func (f *Frontend) cleanupInactiveUserMetrics(user string) {
 	f.discardedRequests.DeletePartialMatch(prometheus.Labels{
 		"user": user,
 	})
+	f.requestQueue.CleanupInactiveUserMetrics(user)
 }
 
 // RoundTripGRPC round trips a proto (instead of a HTTP request).

pkg/frontend/v1/frontend_test.go

@@ -212,11 +212,14 @@ func TestFrontendMetricsCleanup(t *testing.T) {
 				# HELP cortex_query_frontend_queue_length Number of queries in the queue.
 				# TYPE cortex_query_frontend_queue_length gauge
 				cortex_query_frontend_queue_length{priority="0",type="fifo",user="1"} 0
-			`), "cortex_query_frontend_queue_length"))
+        	    # HELP cortex_request_queue_requests_total Total number of query requests going to the request queue.
+        	    # TYPE cortex_request_queue_requests_total counter
+        	    cortex_request_queue_requests_total{priority="0",user="1"} 1
+			`), "cortex_query_frontend_queue_length", "cortex_request_queue_requests_total"))
 
 			fr.cleanupInactiveUserMetrics("1")
 
-			require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(""), "cortex_query_frontend_queue_length"))
+			require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(""), "cortex_query_frontend_queue_length", "cortex_request_queue_requests_total"))
 		}
 
 		testFrontend(t, defaultFrontendConfig(), handler, test, matchMaxConcurrency, nil, reg)

pkg/scheduler/queue/queue.go

@@ -247,3 +247,7 @@ func (q *RequestQueue) QuerierDisconnecting() {
 func (q *RequestQueue) GetConnectedQuerierWorkersMetric() float64 {
 	return float64(q.connectedQuerierWorkers.Load())
 }
+
+func (q *RequestQueue) CleanupInactiveUserMetrics(user string) {
+	q.totalRequests.DeletePartialMatch(prometheus.Labels{"user": user})
+}

pkg/scheduler/scheduler.go

@@ -532,6 +532,7 @@ func (s *Scheduler) cleanupMetricsForInactiveUser(user string) {
 	s.discardedRequests.DeletePartialMatch(prometheus.Labels{
 		"user": user,
 	})
+	s.requestQueue.CleanupInactiveUserMetrics(user)
 }
 
 func (s *Scheduler) getConnectedFrontendClientsMetric() float64 {

pkg/scheduler/scheduler_test.go

@@ -430,15 +430,22 @@ func TestSchedulerMetrics(t *testing.T) {
 		# TYPE cortex_query_scheduler_queue_length gauge
 		cortex_query_scheduler_queue_length{priority="0",type="fifo",user="another"} 1
 		cortex_query_scheduler_queue_length{priority="0",type="fifo",user="test"} 1
-	`), "cortex_query_scheduler_queue_length"))
+        # HELP cortex_request_queue_requests_total Total number of query requests going to the request queue.
+        # TYPE cortex_request_queue_requests_total counter
+        cortex_request_queue_requests_total{priority="0",user="another"} 1
+        cortex_request_queue_requests_total{priority="0",user="test"} 1
+	`), "cortex_query_scheduler_queue_length", "cortex_request_queue_requests_total"))
 
 	scheduler.cleanupMetricsForInactiveUser("test")
 
 	require.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(`
 		# HELP cortex_query_scheduler_queue_length Number of queries in the queue.
 		# TYPE cortex_query_scheduler_queue_length gauge
 		cortex_query_scheduler_queue_length{priority="0",type="fifo",user="another"} 1
-	`), "cortex_query_scheduler_queue_length"))
+        # HELP cortex_request_queue_requests_total Total number of query requests going to the request queue.
+        # TYPE cortex_request_queue_requests_total counter
+        cortex_request_queue_requests_total{priority="0",user="another"} 1
+	`), "cortex_query_scheduler_queue_length", "cortex_request_queue_requests_total"))
 }
 
 func initFrontendLoop(t *testing.T, client schedulerpb.SchedulerForFrontendClient, frontendAddr string) schedulerpb.SchedulerForFrontend_FrontendLoopClient {