Enhance MySQLDown alert

vexxhost · Dec 11, 2024 · 209954c · 209954c
1 parent cfea8c4
commit 209954c
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 0 deletions.
diff --git a/roles/kube_prometheus_stack/files/jsonnet/mixins.libsonnet b/roles/kube_prometheus_stack/files/jsonnet/mixins.libsonnet
@@ -16,6 +16,10 @@ local disabledAlerts = [
   // * Dropped `CephPGImbalance`
   // the balancer module takes care of this
   'CephPGImbalance',
+
+  // * Dropped `MySQLDown` due to noisy alerts even
+  //   the replication still more than minimum
+  'MySQLDown',
 ];
 
 // NOTE(mnaser): This is the default mapping for severities:
@@ -141,6 +145,29 @@ local mixins = {
                 severity: 'warning',
               },
             },
+            {
+              alert: 'MysqlClusterDown',
+              'for': '1m',
+              expr: 'count(mysql_up==0) != count(mysql_up)',
+              labels: {
+                severity: 'info',
+              },
+              annotations: {
+                summary: '{{ $value }} percona-xtradb replication down',
+              },
+            },
+            {
+              alert: 'MysqlClusterDown',
+              'for': '1m',
+              expr: 'count(mysql_up==1) < 3',
+              labels: {
+                severity: 'warning',
+              },
+              annotations: {
+                summary: 'Only {{ $value }} percona-xtradb cluster are online',
+                description: "percona-xtradb cluster less than 3 replication, please check with kubectl get pods -n openstack -l app.kubernetes.io/component=pxc",
+              },              
+            },
           ],
         },
       ],

diff --git a/roles/kube_prometheus_stack/files/jsonnet/tests.yml b/roles/kube_prometheus_stack/files/jsonnet/tests.yml
@@ -105,3 +105,18 @@ tests:
       - eval_time: 5m
         alertname: NodeTimeSkewDetected
         exp_alerts: []
+
+  - interval: 1m
+    input_series:
+      - series: 'mysql_up'
+        values: '1'
+    input_series:
+      - series: 'mysql_up'
+        values: '1'
+    input_series:
+      - series: 'mysql_up'
+        values: '0'                
+    alert_rule_test:
+      - eval_time: 5m
+        alertname: MysqlClusterDown
+        exp_alerts: []