Remove test-exporter based alert and runbook. (#7774)

* Remove obsolete alert and runbook. Signed-off-by: Peter Štibraný <[email protected]> * CHANGELOG, rebuild alerts. Signed-off-by: Peter Štibraný <[email protected]> --------- Signed-off-by: Peter Štibraný <[email protected]>
grafana · Apr 2, 2024 · 4dc4303 · 4dc4303
1 parent adb6380
commit 4dc4303
Show file tree

Hide file tree

Showing 6 changed files with 1 addition and 60 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -27,6 +27,7 @@
 
 ### Mixin
 
+* [CHANGE] Alerts: Removed obsolete `MimirQueriesIncorrect` alert that used test-exporter metrics. Test-exporter support was however removed in Mimir 2.0 release. #7774
 * [FEATURE] Dashboards: added 'Remote ruler reads networking' dashboard. #7751
 * [ENHANCEMENT] Alerts: allow configuring alerts range interval via `_config.base_alerts_range_interval_minutes`. #7591
 * [ENHANCEMENT] Dashboards: Add panels for monitoring distributor and ingester when using ingest-storage. These panels are disabled by default, but can be enabled using `show_ingest_storage_panels: true` config option. Similarly existing panels used when distributors and ingesters use gRPC for forwarding requests can be disabled by setting `show_grpc_ingestion_panels: false`. #7670 #7699

diff --git a/docs/sources/mimir/manage/mimir-runbooks/_index.md b/docs/sources/mimir/manage/mimir-runbooks/_index.md
@@ -677,10 +677,6 @@ How to **investigate**:
 - Ensure the compactor is successfully running
 - Look for any error in the compactor logs
 
-### MimirQueriesIncorrect
-
-_TODO: this runbook has not been written yet._
-
 ### MimirInconsistentRuntimeConfig
 
 This alert fires if multiple replicas of the same Mimir service are using a different runtime config for a longer period of time.

diff --git a/...amonitoring-values-generated/mimir-distributed/templates/metamonitoring/mixin-alerts.yaml b/...amonitoring-values-generated/mimir-distributed/templates/metamonitoring/mixin-alerts.yaml
@@ -48,18 +48,6 @@ spec:
       for: 15m
       labels:
         severity: warning
-    - alert: MimirQueriesIncorrect
-      annotations:
-        message: |
-          The Mimir cluster {{ $labels.cluster }}/{{ $labels.namespace }} is experiencing {{ printf "%.2f" $value }}% incorrect query results.
-        runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirqueriesincorrect
-      expr: |
-        100 * sum by (cluster, namespace) (rate(test_exporter_test_case_result_total{result="fail"}[5m]))
-          /
-        sum by (cluster, namespace) (rate(test_exporter_test_case_result_total[5m])) > 1
-      for: 15m
-      labels:
-        severity: warning
     - alert: MimirInconsistentRuntimeConfig
       annotations:
         message: |

diff --git a/operations/mimir-mixin-compiled-baremetal/alerts.yaml b/operations/mimir-mixin-compiled-baremetal/alerts.yaml
@@ -36,18 +36,6 @@ groups:
     for: 15m
     labels:
       severity: warning
-  - alert: MimirQueriesIncorrect
-    annotations:
-      message: |
-        The Mimir cluster {{ $labels.cluster }}/{{ $labels.namespace }} is experiencing {{ printf "%.2f" $value }}% incorrect query results.
-      runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirqueriesincorrect
-    expr: |
-      100 * sum by (cluster, namespace) (rate(test_exporter_test_case_result_total{result="fail"}[5m]))
-        /
-      sum by (cluster, namespace) (rate(test_exporter_test_case_result_total[5m])) > 1
-    for: 15m
-    labels:
-      severity: warning
   - alert: MimirInconsistentRuntimeConfig
     annotations:
       message: |

diff --git a/operations/mimir-mixin-compiled/alerts.yaml b/operations/mimir-mixin-compiled/alerts.yaml
@@ -36,18 +36,6 @@ groups:
     for: 15m
     labels:
       severity: warning
-  - alert: MimirQueriesIncorrect
-    annotations:
-      message: |
-        The Mimir cluster {{ $labels.cluster }}/{{ $labels.namespace }} is experiencing {{ printf "%.2f" $value }}% incorrect query results.
-      runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirqueriesincorrect
-    expr: |
-      100 * sum by (cluster, namespace) (rate(test_exporter_test_case_result_total{result="fail"}[5m]))
-        /
-      sum by (cluster, namespace) (rate(test_exporter_test_case_result_total[5m])) > 1
-    for: 15m
-    labels:
-      severity: warning
   - alert: MimirInconsistentRuntimeConfig
     annotations:
       message: |

diff --git a/operations/mimir-mixin/alerts/alerts.libsonnet b/operations/mimir-mixin/alerts/alerts.libsonnet
@@ -79,26 +79,6 @@ local utils = import 'mixin-utils/utils.libsonnet';
             ||| % $._config,
           },
         },
-        {
-          alert: $.alertName('QueriesIncorrect'),
-          expr: |||
-            100 * sum by (%(group_by)s) (rate(test_exporter_test_case_result_total{result="fail"}[%(range_interval)s]))
-              /
-            sum by (%(group_by)s) (rate(test_exporter_test_case_result_total[%(range_interval)s])) > 1
-          ||| % {
-            group_by: $._config.alert_aggregation_labels,
-            range_interval: $.alertRangeInterval(5),
-          },
-          'for': '15m',
-          labels: {
-            severity: 'warning',
-          },
-          annotations: {
-            message: |||
-              The %(product)s cluster %(alert_aggregation_variables)s is experiencing {{ printf "%%.2f" $value }}%% incorrect query results.
-            ||| % $._config,
-          },
-        },
         {
           alert: $.alertName('InconsistentRuntimeConfig'),
           expr: |||