Remove reference to per-instance labels in GossipMembersMismatch warn…

…ing (#6146) * Remove reference to per-instance labels in GossipMembersMismatch warning. * Add changelog entry.
grafana · Sep 28, 2023 · 934201d · 934201d
1 parent b7c052c
commit 934201d
Show file tree

Hide file tree

Showing 5 changed files with 9 additions and 7 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -29,6 +29,8 @@
 
 ### Mixin
 
+* [BUGFIX] Alerts: fixed issue where `GossipMembersMismatch` warning message referred to per-instance labels that were not produced by the alert query. #6146
+
 ### Jsonnet
 
 * [ENHANCEMENT] Double the amount of rule groups for each user tier. #5897

diff --git a/...amonitoring-values-generated/mimir-distributed/templates/metamonitoring/mixin-alerts.yaml b/...amonitoring-values-generated/mimir-distributed/templates/metamonitoring/mixin-alerts.yaml
@@ -489,8 +489,8 @@ spec:
     rules:
     - alert: MimirGossipMembersMismatch
       annotations:
-        message: Mimir instance {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
-          }} sees incorrect number of gossip members.
+        message: One or more Mimir instances in {{ $labels.cluster }}/{{ $labels.namespace
+          }} see incorrect number of gossip members.
         runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirgossipmembersmismatch
       expr: |
         avg by (cluster, namespace) (memberlist_client_cluster_members_count) != sum by (cluster, namespace) (up{job=~".+/(admin-api|alertmanager|compactor.*|distributor|ingester.*|querier.*|ruler|ruler-querier.*|store-gateway.*|cortex|mimir|mimir-write.*|mimir-read.*|mimir-backend.*)"})

diff --git a/operations/mimir-mixin-compiled-baremetal/alerts.yaml b/operations/mimir-mixin-compiled-baremetal/alerts.yaml
@@ -467,8 +467,8 @@ groups:
   rules:
   - alert: MimirGossipMembersMismatch
     annotations:
-      message: Mimir instance {{ $labels.instance }} in {{ $labels.cluster }}/{{ $labels.namespace
-        }} sees incorrect number of gossip members.
+      message: One or more Mimir instances in {{ $labels.cluster }}/{{ $labels.namespace
+        }} see incorrect number of gossip members.
       runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirgossipmembersmismatch
     expr: |
       avg by (cluster, namespace) (memberlist_client_cluster_members_count) != sum by (cluster, namespace) (up{job=~".+/(admin-api|alertmanager|compactor.*|distributor|ingester.*|querier.*|ruler|ruler-querier.*|store-gateway.*|cortex|mimir|mimir-write.*|mimir-read.*|mimir-backend.*)"})

diff --git a/operations/mimir-mixin-compiled/alerts.yaml b/operations/mimir-mixin-compiled/alerts.yaml
@@ -477,8 +477,8 @@ groups:
   rules:
   - alert: MimirGossipMembersMismatch
     annotations:
-      message: Mimir instance {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace
-        }} sees incorrect number of gossip members.
+      message: One or more Mimir instances in {{ $labels.cluster }}/{{ $labels.namespace
+        }} see incorrect number of gossip members.
       runbook_url: https://grafana.com/docs/mimir/latest/operators-guide/mimir-runbooks/#mimirgossipmembersmismatch
     expr: |
       avg by (cluster, namespace) (memberlist_client_cluster_members_count) != sum by (cluster, namespace) (up{job=~".+/(admin-api|alertmanager|compactor.*|distributor|ingester.*|querier.*|ruler|ruler-querier.*|store-gateway.*|cortex|mimir|mimir-write.*|mimir-read.*|mimir-backend.*)"})

diff --git a/operations/mimir-mixin/alerts/alerts.libsonnet b/operations/mimir-mixin/alerts/alerts.libsonnet
@@ -708,7 +708,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
             severity: 'warning',
           },
           annotations: {
-            message: '%(product)s instance %(alert_instance_variable)s in %(alert_aggregation_variables)s sees incorrect number of gossip members.' % $._config,
+            message: 'One or more %(product)s instances in %(alert_aggregation_variables)s see incorrect number of gossip members.' % $._config,
           },
         },
       ],