add MimirContinuousTestFailingOnWrites and MimirContinuousTestFailing…

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Add `MimirContinuousTestFailingOnWrites` and `MimirContinuousTestFailingOnReads` alerts.
+
 ### Removed
 
 - Remove the `mimir.enabled` property to replace it with the MC flavor as all CAPI MCs now run Mimir.

helm/prometheus-rules/templates/platform/atlas/alerting-rules/mimir.rules.yml

@@ -169,4 +169,40 @@ spec:
         severity: page
         team: atlas
         topic: observability
+  - name: mimir.continuous-test
+    rules:
+    - alert: MimirContinuousTestFailingOnWrites
+      annotations:
+        dashboard: mimir-continous-test/mimir-continous-test
+        description: 'Mimir continous-test detected errors in the write path.'
+        opsrecipe: mimir/
+      # Query is based on the following upstream mixin alerting rule : https://github.com/grafana/mimir/blob/main/operations/mimir-mixin-compiled/alerts.yaml#L1097
+      expr: sum by(cluster_id, installation, namespace, pipeline, provider, test) (rate(mimir_continuous_test_writes_failed_total[5m])) > 0
+      for: 1h
+      labels:
+        area: platform
+        cancel_if_cluster_status_creating: "true"
+        cancel_if_cluster_status_deleting: "true"
+        cancel_if_cluster_status_updating: "true"
+        cancel_if_outside_working_hours: "true"
+        severity: page
+        team: atlas
+        topic: observability
+    - alert: MimirContinuousTestFailingOnReads
+      annotations:
+        dashboard: mimir-continous-test/mimir-continous-test
+        description: 'Mimir continous-test detected errors in the write path.'
+        opsrecipe: mimir/
+      # Query is based on the following upstream mixin alerting rule : https://github.com/grafana/mimir/blob/main/operations/mimir-mixin-compiled/alerts.yaml#L1097
+      expr: sum by(cluster_id, installation, namespace, pipeline, provider, test) (rate(mimir_continuous_test_queries_failed_total[5m])) > 0
+      for: 1h
+      labels:
+        area: platform
+        cancel_if_cluster_status_creating: "true"
+        cancel_if_cluster_status_deleting: "true"
+        cancel_if_cluster_status_updating: "true"
+        cancel_if_outside_working_hours: "true"
+        severity: page
+        team: atlas
+        topic: observability
 {{- end }}

test/tests/providers/capi/capa/platform/atlas/alerting-rules/mimir.rules.test.yml

@@ -390,3 +390,65 @@ tests:
         eval_time: 205m 
       - alertname: MimirCompactorFailedCompaction
         eval_time: 350m
+
+  # Test for MimirContinuousTestFailingOnWrites alert
+  - interval: 1m
+    input_series:
+      # Test: none, rate > 0, rate = 0
+      - series: 'mimir_continuous_test_writes_failed_total{cluster_id="golem", installation="golem", namespace="mimir", pipeline="testing", provider="capa"}'
+        values: "_x20 1+1x80 0+0x70"
+    alert_rule_test:
+      - alertname: MimirContinuousTestFailingOnWrites
+        eval_time: 40m
+      - alertname: MimirContinuousTestFailingOnWrites
+        eval_time: 95m
+        exp_alerts:
+          - exp_labels:
+              area: platform
+              cancel_if_outside_working_hours: "true"
+              cancel_if_cluster_status_creating: "true"
+              cancel_if_cluster_status_deleting: "true"
+              cancel_if_cluster_status_updating: "true"
+              cluster_id: golem
+              installation: golem
+              namespace: mimir
+              severity: page
+              team: atlas
+              topic: observability
+            exp_annotations:
+              dashboard: mimir-continous-test/mimir-continous-test
+              description: "Mimir continous-test detected errors in the write path."
+              opsrecipe: "mimir/"
+      - alertname: MimirContinuousTestFailingOnWrites
+        eval_time: 160m
+
+  # Test for MimirContinuousTestFailingOnReads alert
+  - interval: 1m
+    input_series:
+      # Test: none, rate > 0, rate = 0
+      - series: 'mimir_continuous_test_queries_failed_total{cluster_id="golem", installation="golem", namespace="mimir", pipeline="testing", provider="capa"}'
+        values: "_x20 1+1x80 0+0x70"
+    alert_rule_test:
+      - alertname: MimirContinuousTestFailingOnReads
+        eval_time: 40m
+      - alertname: MimirContinuousTestFailingOnReads
+        eval_time: 95m
+        exp_alerts:
+          - exp_labels:
+              area: platform
+              cancel_if_outside_working_hours: "true"
+              cancel_if_cluster_status_creating: "true"
+              cancel_if_cluster_status_deleting: "true"
+              cancel_if_cluster_status_updating: "true"
+              cluster_id: golem
+              installation: golem
+              namespace: mimir
+              severity: page
+              team: atlas
+              topic: observability
+            exp_annotations:
+              dashboard: mimir-continous-test/mimir-continous-test
+              description: "Mimir continous-test detected errors in the write path."
+              opsrecipe: "mimir/"
+      - alertname: MimirContinuousTestFailingOnReads
+        eval_time: 160m