simulating skipping requests when store-gateways are overloaded

grafana · May 27, 2024 · 14a1429 · 14a1429
1 parent 64bfaaf
commit 14a1429
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/development/mimir-microservices-mode/config/mimir.yaml b/development/mimir-microservices-mode/config/mimir.yaml
@@ -138,6 +138,7 @@ frontend:
   cache_results: true
   additional_query_queue_dimensions_enabled: true
   shard_active_series_queries: true
+  max_outstanding_per_tenant: 10000
 
   # Uncomment when using "dns" service discovery mode for query-scheduler.
   # scheduler_address: "query-scheduler:9011"
@@ -163,6 +164,7 @@ query_scheduler:
   # Change to "dns" to switch to query-scheduler DNS-based service discovery.
   service_discovery_mode: "ring"
   additional_query_queue_dimensions_enabled: true
+  max_outstanding_requests_per_tenant: 10000
 
 limits:
   # Limit max query time range to 31d

diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go
@@ -454,7 +454,7 @@ func (s *Scheduler) QuerierLoop(querier schedulerpb.SchedulerForQuerier_QuerierL
 
 		if exceedsThreshold {
 			level.Info(s.requestQueue.Log).Log(
-				"msg", "experimental: querier worker connections in use by query component exceed utilization threshold. dropping request",
+				"msg", "experimental: querier worker connections in use by query component exceed utilization threshold. request dropped",
 				"query_component_name", queryComponentName,
 				"overloaded_query_component", queryComponent,
 			)