diff --git a/docs/components/query-frontend.md b/docs/components/query-frontend.md index e374f19ec16..16976bfbd6a 100644 --- a/docs/components/query-frontend.md +++ b/docs/components/query-frontend.md @@ -77,6 +77,12 @@ config: max_get_multi_batch_size: 0 dns_provider_update_interval: 0s auto_discovery: false + set_async_circuit_breaker_enabled: false + set_async_circuit_breaker_half_open_max_requests: 0 + set_async_circuit_breaker_open_duration: 0s + set_async_circuit_breaker_min_requests: 0 + set_async_circuit_breaker_consecutive_failures: 0 + set_async_circuit_breaker_failure_percent: 0 expiration: 0s ``` @@ -132,6 +138,12 @@ config: master_name: "" max_async_buffer_size: 10000 max_async_concurrency: 20 + set_async_circuit_breaker_enabled: false + set_async_circuit_breaker_half_open_max_requests: 10 + set_async_circuit_breaker_open_duration: 5s + set_async_circuit_breaker_min_requests: 50 + set_async_circuit_breaker_consecutive_failures: 5 + set_async_circuit_breaker_failure_percent: 0.05 expiration: 24h0m0s ``` diff --git a/docs/components/store.md b/docs/components/store.md index 8ecc53d68fd..aa37dd483a9 100644 --- a/docs/components/store.md +++ b/docs/components/store.md @@ -325,6 +325,12 @@ config: max_get_multi_batch_size: 0 dns_provider_update_interval: 0s auto_discovery: false + set_async_circuit_breaker_enabled: false + set_async_circuit_breaker_half_open_max_requests: 0 + set_async_circuit_breaker_open_duration: 0s + set_async_circuit_breaker_min_requests: 0 + set_async_circuit_breaker_consecutive_failures: 0 + set_async_circuit_breaker_failure_percent: 0 enabled_items: [] ttl: 0s ``` @@ -340,6 +346,21 @@ While the remaining settings are **optional**: - `max_async_concurrency`: maximum number of concurrent asynchronous operations can occur. - `max_async_buffer_size`: maximum number of enqueued asynchronous operations allowed. - `max_get_multi_concurrency`: maximum number of concurrent connections when fetching keys. If set to `0`, the concurrency is unlimited. +- `set_async_circuit_breaker_enabled`: `true` to enable circuite breaker for asynchronous operations. + The circuit breaker consists of three states: closed, half-open, and open. + It begins in the closed state. When the total requests exceed `set_async_circuit_breaker_min_requests`, + and either consecutive failures occur or the failure percentage is excessively high according + to the configured values, the circuit breaker transitions to the open state. + This results in the rejection of all asynchronous operations. After `set_async_circuit_breaker_open_duration`, + the circuit breaker transitions to the half-open state, where it allows `set_async_circuit_breaker_half_open_max_requests` + asynchronous operations to be processed in order to test if the conditions have improved. If they have not, + the state transitions back to open; if they have, it transitions to the closed state. Following each 10 seconds + interval in the closed state, the circuit breaker resets its metrics and repeats this cycle. +- `set_async_circuit_breaker_half_open_max_requests`: maximum number of requests allowed to pass through when the circuit breaker is half-open. If set to 0, the circuit breaker allows only 1 request. +- `set_async_circuit_breaker_open_duration`: the period of the open state after which the state of the circuit breaker becomes half-open. If set to 0, the circuit breaker resets it to 60 seconds. +- `set_async_circuit_breaker_min_requests`: minimal requests to trigger the circuit breaker, 0 signifies no requirements. +- `set_async_circuit_breaker_consecutive_failures`: consecutive failures based on `set_async_circuit_breaker_min_requests` to determine if the circuit breaker should open. +- `set_async_circuit_breaker_failure_percent`: the failure percentage, which is based on `set_async_circuit_breaker_min_requests`, to determine if the circuit breaker should open. - `max_get_multi_batch_size`: maximum number of keys a single underlying operation should fetch. If more keys are specified, internally keys are splitted into multiple batches and fetched concurrently, honoring `max_get_multi_concurrency`. If set to `0`, the batch size is unlimited. - `max_item_size`: maximum size of an item to be stored in memcached. This option should be set to the same value of memcached `-I` flag (defaults to 1MB) in order to avoid wasting network round trips to store items larger than the max item size allowed in memcached. If set to `0`, the item size is unlimited. - `dns_provider_update_interval`: the DNS discovery update interval. @@ -376,6 +397,12 @@ config: master_name: "" max_async_buffer_size: 10000 max_async_concurrency: 20 + set_async_circuit_breaker_enabled: false + set_async_circuit_breaker_half_open_max_requests: 10 + set_async_circuit_breaker_open_duration: 5s + set_async_circuit_breaker_min_requests: 50 + set_async_circuit_breaker_consecutive_failures: 5 + set_async_circuit_breaker_failure_percent: 0.05 enabled_items: [] ttl: 0s ``` diff --git a/pkg/cacheutil/memcached_client.go b/pkg/cacheutil/memcached_client.go index a7d08274b3a..57547099323 100644 --- a/pkg/cacheutil/memcached_client.go +++ b/pkg/cacheutil/memcached_client.go @@ -153,16 +153,6 @@ type MemcachedClientConfig struct { AutoDiscovery bool `yaml:"auto_discovery"` // SetAsyncCircuitBreakerEnabled enables circuite breaker for SetAsync operations. - // - // The circuit breaker consists of three states: closed, half-open, and open. - // It begins in the closed state. When the total requests exceed SetAsyncCircuitBreakerMinRequests, - // and either consecutive failures occur or the failure percentage is excessively high according - // to the configured values, the circuit breaker transitions to the open state. - // This results in the rejection of all SetAsync requests. After SetAsyncCircuitBreakerOpenDuration, - // the circuit breaker transitions to the half-open state, where it allows SetAsyncCircuitBreakerHalfOpenMaxRequests - // SetAsync requests to be processed in order to test if the conditions have improved. If they have not, - // the state transitions back to open; if they have, it transitions to the closed state. Following each 10 seconds - // interval in the closed state, the circuit breaker resets its metrics and repeats this cycle. SetAsyncCircuitBreakerEnabled bool `yaml:"set_async_circuit_breaker_enabled"` // SetAsyncCircuitBreakerHalfOpenMaxRequests is the maximum number of requests allowed to pass through // when the circuit breaker is half-open. diff --git a/pkg/cacheutil/redis_client.go b/pkg/cacheutil/redis_client.go index b2aef1197db..6a0a9aedc67 100644 --- a/pkg/cacheutil/redis_client.go +++ b/pkg/cacheutil/redis_client.go @@ -40,6 +40,8 @@ var ( TLSConfig: TLSConfig{}, MaxAsyncConcurrency: 20, MaxAsyncBufferSize: 10000, + + SetAsyncCircuitBreakerEnabled: false, SetAsyncCircuitBreakerHalfOpenMaxRequests: 10, SetAsyncCircuitBreakerOpenDuration: 5 * time.Second, SetAsyncCircuitBreakerMinRequests: 50, @@ -126,16 +128,6 @@ type RedisClientConfig struct { MaxAsyncConcurrency int `yaml:"max_async_concurrency"` // SetAsyncCircuitBreakerEnabled enables circuite breaker for SetAsync operations. - // - // The circuit breaker consists of three states: closed, half-open, and open. - // It begins in the closed state. When the total requests exceed SetAsyncCircuitBreakerMinRequests, - // and either consecutive failures occur or the failure percentage is excessively high according - // to the configured values, the circuit breaker transitions to the open state. - // This results in the rejection of all SetAsync requests. After SetAsyncCircuitBreakerOpenDuration, - // the circuit breaker transitions to the half-open state, where it allows SetAsyncCircuitBreakerHalfOpenMaxRequests - // SetAsync requests to be processed in order to test if the conditions have improved. If they have not, - // the state transitions back to open; if they have, it transitions to the closed state. Following each 10 seconds - // interval in the closed state, the circuit breaker resets its metrics and repeats this cycle. SetAsyncCircuitBreakerEnabled bool `yaml:"set_async_circuit_breaker_enabled"` // SetAsyncCircuitBreakerHalfOpenMaxRequests is the maximum number of requests allowed to pass through // when the circuit breaker is half-open.