diff --git a/roles/kube_prometheus_stack/files/jsonnet/mixins.libsonnet b/roles/kube_prometheus_stack/files/jsonnet/mixins.libsonnet index 2b876b754..4997edb89 100644 --- a/roles/kube_prometheus_stack/files/jsonnet/mixins.libsonnet +++ b/roles/kube_prometheus_stack/files/jsonnet/mixins.libsonnet @@ -16,6 +16,10 @@ local disabledAlerts = [ // * Dropped `CephPGImbalance` // the balancer module takes care of this 'CephPGImbalance', + + // * Dropped `MySQLDown` due to noisy alerts even + // the replication still more than minimum + 'MySQLDown', ]; // NOTE(mnaser): This is the default mapping for severities: @@ -141,6 +145,29 @@ local mixins = { severity: 'warning', }, }, + { + alert: 'MysqlClusterDown', + 'for': '1m', + expr: 'count(mysql_up==0) != count(mysql_up)', + labels: { + severity: 'info', + }, + annotations: { + summary: '{{ $value }} percona-xtradb replication down', + }, + }, + { + alert: 'MysqlClusterDown', + 'for': '1m', + expr: 'count(mysql_up==1) < 3', + labels: { + severity: 'warning', + }, + annotations: { + summary: 'Only {{ $value }} percona-xtradb cluster are online', + description: "percona-xtradb cluster less than 3 replication, please check with kubectl get pods -n openstack -l app.kubernetes.io/component=pxc", + }, + }, ], }, ], diff --git a/roles/kube_prometheus_stack/files/jsonnet/tests.yml b/roles/kube_prometheus_stack/files/jsonnet/tests.yml index 4775bb2b8..216dc24b5 100644 --- a/roles/kube_prometheus_stack/files/jsonnet/tests.yml +++ b/roles/kube_prometheus_stack/files/jsonnet/tests.yml @@ -105,3 +105,18 @@ tests: - eval_time: 5m alertname: NodeTimeSkewDetected exp_alerts: [] + + - interval: 1m + input_series: + - series: 'mysql_up' + values: '1' + input_series: + - series: 'mysql_up' + values: '1' + input_series: + - series: 'mysql_up' + values: '0' + alert_rule_test: + - eval_time: 5m + alertname: MysqlClusterDown + exp_alerts: [] \ No newline at end of file