add MimirHPANeedsToBeScaledUp alert

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Add `MimirHPANeedsToBeScaledUp` alert, to detect when Mimir's HPAs have reached maximum capacity.
+
 ### Changed
 
 - alertmanager alerts: add link to dashboard

helm/prometheus-rules/templates/platform/atlas/alerting-rules/mimir.rules.yml

@@ -137,4 +137,30 @@ spec:
         severity: page
         team: atlas
         topic: observability
+    - alert: MimirHPANeedsToBeScaledUp
+      annotations:
+        description: '{{`Mimir ${ labels.horizontalpodautoscaler } HPA has reached maximum replicas and consume too much resources, it needs to be scaled up.`}}'
+        opsrecipe: mimir-ingester/
+      expr: |-
+        (
+          kube_horizontalpodautoscaler_status_desired_replicas{namespace="mimir"} >=
+          on(cluster_id, customer, installation, namespace, horizontalpodautoscaler)
+          kube_horizontalpodautoscaler_spec_max_replicas{namespace="mimir"}
+        )
+        and on(cluster_id, customer, installation, namespace, horizontalpodautoscaler)
+        (
+          kube_horizontalpodautoscaler_status_target_metric{namespace="mimir"} >
+          on(cluster_id, customer, installation, namespace, horizontalpodautoscaler, metric_name, metric_target_type)
+          kube_horizontalpodautoscaler_spec_target_metric{namespace="mimir"}
+        )
+      for: 30m
+      labels:
+        area: platform
+        cancel_if_cluster_status_creating: "true"
+        cancel_if_cluster_status_deleting: "true"
+        cancel_if_cluster_status_updating: "true"
+        cancel_if_outside_working_hours: "true"
+        severity: page
+        team: atlas
+        topic: observability
 {{- end }}