From 14a1429e7af4b43a3301075befaa66a118d98648 Mon Sep 17 00:00:00 2001 From: francoposa Date: Mon, 27 May 2024 15:26:26 -0700 Subject: [PATCH] simulating skipping requests when store-gateways are overloaded --- development/mimir-microservices-mode/config/mimir.yaml | 2 ++ pkg/scheduler/scheduler.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/development/mimir-microservices-mode/config/mimir.yaml b/development/mimir-microservices-mode/config/mimir.yaml index fc4136000d4..a55e9ae8ed8 100644 --- a/development/mimir-microservices-mode/config/mimir.yaml +++ b/development/mimir-microservices-mode/config/mimir.yaml @@ -138,6 +138,7 @@ frontend: cache_results: true additional_query_queue_dimensions_enabled: true shard_active_series_queries: true + max_outstanding_per_tenant: 10000 # Uncomment when using "dns" service discovery mode for query-scheduler. # scheduler_address: "query-scheduler:9011" @@ -163,6 +164,7 @@ query_scheduler: # Change to "dns" to switch to query-scheduler DNS-based service discovery. service_discovery_mode: "ring" additional_query_queue_dimensions_enabled: true + max_outstanding_requests_per_tenant: 10000 limits: # Limit max query time range to 31d diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index 5debdb7651e..410560655ee 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -454,7 +454,7 @@ func (s *Scheduler) QuerierLoop(querier schedulerpb.SchedulerForQuerier_QuerierL if exceedsThreshold { level.Info(s.requestQueue.Log).Log( - "msg", "experimental: querier worker connections in use by query component exceed utilization threshold. dropping request", + "msg", "experimental: querier worker connections in use by query component exceed utilization threshold. request dropped", "query_component_name", queryComponentName, "overloaded_query_component", queryComponent, )