Disable ingester push circuit breaker in read-only mode (#9760)

* Disable ingester push circuit breaker in read-only mode

* Update CHANGELOG

* Remove unused function

(cherry picked from commit 2532be9)

CHANGELOG.md

@@ -48,6 +48,7 @@
 * [ENHANCEMENT] Query-scheduler: Remove the experimental `query-scheduler.prioritize-query-components` flag. Request queues always prioritize query component dequeuing above tenant fairness. #9703
 * [ENHANCEMENT] Ingester: Emit traces for block syncing, to join up block-upload traces. #9656
 * [ENHANCEMENT] Querier: Enable the optional querying of additional storage queryables. #9712
+* [ENHANCEMENT] Ingester: Disable the push circuit breaker when ingester is in read-only mode. #9760
 * [BUGFIX] Fix issue where functions such as `rate()` over native histograms could return incorrect values if a float stale marker was present in the selected range. #9508
 * [BUGFIX] Fix issue where negation of native histograms (eg. `-some_native_histogram_series`) did nothing. #9508
 * [BUGFIX] Fix issue where `metric might not be a counter, name does not end in _total/_sum/_count/_bucket` annotation would be emitted even if `rate` or `increase` did not have enough samples to compute a result. #9508

pkg/ingester/circuitbreaker.go

@@ -269,11 +269,6 @@ func newIngesterCircuitBreaker(pushCfg CircuitBreakerConfig, readCfg CircuitBrea
 	}
 }
 
-func (cb *ingesterCircuitBreaker) activate() {
-	cb.push.activate()
-	cb.read.activate()
-}
-
 // tryAcquirePushPermit tries to acquire a permit to use the push circuit breaker and returns whether a permit was acquired.
 // If it was possible, tryAcquirePushPermit returns a function that should be called to release the acquired permit.
 // If it was not possible, the causing error is returned.

pkg/ingester/downscale.go

@@ -48,6 +48,8 @@ func (i *Ingester) PrepareInstanceRingDownscaleHandler(w http.ResponseWriter, r
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
+		// Deactivate the push circuit breaker in read-only mode. Calling this repeatedly is fine.
+		i.circuitBreaker.push.deactivate()
 
 	case http.MethodDelete:
 		// Clear the read-only status.
@@ -57,6 +59,8 @@ func (i *Ingester) PrepareInstanceRingDownscaleHandler(w http.ResponseWriter, r
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
+		// Activate the push circuit breaker when exiting read-only mode. Calling this repeatedly is fine.
+		i.circuitBreaker.push.activate()
 	}
 
 	ro, rots := i.lifecycler.GetReadOnlyState()

pkg/ingester/ingester.go

@@ -633,7 +633,11 @@ func (i *Ingester) starting(ctx context.Context) (err error) {
 		return errors.Wrap(err, "failed to start ingester subservices after ingester ring lifecycler")
 	}
 
-	i.circuitBreaker.activate()
+	i.circuitBreaker.read.activate()
+	if ro, _ := i.lifecycler.GetReadOnlyState(); !ro {
+		// If the ingester is not read-only, activate the push circuit breaker.
+		i.circuitBreaker.push.activate()
+	}
 	return nil
 }