diff --git a/CHANGELOG.md b/CHANGELOG.md index 66c679a888c..bfe8e4cd13a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,7 @@ * [ENHANCEMENT] Query-scheduler: Remove the experimental `query-scheduler.prioritize-query-components` flag. Request queues always prioritize query component dequeuing above tenant fairness. #9703 * [ENHANCEMENT] Ingester: Emit traces for block syncing, to join up block-upload traces. #9656 * [ENHANCEMENT] Querier: Enable the optional querying of additional storage queryables. #9712 +* [ENHANCEMENT] Ingester: Disable the push circuit breaker when ingester is in read-only mode. #9760 * [BUGFIX] Fix issue where functions such as `rate()` over native histograms could return incorrect values if a float stale marker was present in the selected range. #9508 * [BUGFIX] Fix issue where negation of native histograms (eg. `-some_native_histogram_series`) did nothing. #9508 * [BUGFIX] Fix issue where `metric might not be a counter, name does not end in _total/_sum/_count/_bucket` annotation would be emitted even if `rate` or `increase` did not have enough samples to compute a result. #9508 diff --git a/pkg/ingester/circuitbreaker.go b/pkg/ingester/circuitbreaker.go index 4893d4d53f4..e89c4935dbf 100644 --- a/pkg/ingester/circuitbreaker.go +++ b/pkg/ingester/circuitbreaker.go @@ -269,11 +269,6 @@ func newIngesterCircuitBreaker(pushCfg CircuitBreakerConfig, readCfg CircuitBrea } } -func (cb *ingesterCircuitBreaker) activate() { - cb.push.activate() - cb.read.activate() -} - // tryAcquirePushPermit tries to acquire a permit to use the push circuit breaker and returns whether a permit was acquired. // If it was possible, tryAcquirePushPermit returns a function that should be called to release the acquired permit. // If it was not possible, the causing error is returned. diff --git a/pkg/ingester/downscale.go b/pkg/ingester/downscale.go index c81c4125e08..508d82fe3b6 100644 --- a/pkg/ingester/downscale.go +++ b/pkg/ingester/downscale.go @@ -48,6 +48,8 @@ func (i *Ingester) PrepareInstanceRingDownscaleHandler(w http.ResponseWriter, r http.Error(w, err.Error(), http.StatusInternalServerError) return } + // Deactivate the push circuit breaker in read-only mode. Calling this repeatedly is fine. + i.circuitBreaker.push.deactivate() case http.MethodDelete: // Clear the read-only status. @@ -57,6 +59,8 @@ func (i *Ingester) PrepareInstanceRingDownscaleHandler(w http.ResponseWriter, r http.Error(w, err.Error(), http.StatusInternalServerError) return } + // Activate the push circuit breaker when exiting read-only mode. Calling this repeatedly is fine. + i.circuitBreaker.push.activate() } ro, rots := i.lifecycler.GetReadOnlyState() diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 13b66bd64ca..5d6a28c0a98 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -633,7 +633,11 @@ func (i *Ingester) starting(ctx context.Context) (err error) { return errors.Wrap(err, "failed to start ingester subservices after ingester ring lifecycler") } - i.circuitBreaker.activate() + i.circuitBreaker.read.activate() + if ro, _ := i.lifecycler.GetReadOnlyState(); !ro { + // If the ingester is not read-only, activate the push circuit breaker. + i.circuitBreaker.push.activate() + } return nil }