From ce76182f0e89eba71d4a1cee087eb6367e38c0c9 Mon Sep 17 00:00:00 2001 From: Pierre Riteau Date: Fri, 13 Oct 2023 17:18:37 +0200 Subject: [PATCH] Adapt RabbitMQ connection alert threshold The threshold of 1000 connections to a RabbitMQ server can be easily reached even on moderately-sized deployments. On several deployments we saw around 25 connections per hypervisor and between 250 and 750 connections per controller. The threshold is based on twice these numbers to be conservative. --- etc/kayobe/kolla/config/prometheus/rabbitmq.rules | 2 +- .../notes/rabbitmq-connection-alert-85cc7b29ddf8e3c3.yaml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/rabbitmq-connection-alert-85cc7b29ddf8e3c3.yaml diff --git a/etc/kayobe/kolla/config/prometheus/rabbitmq.rules b/etc/kayobe/kolla/config/prometheus/rabbitmq.rules index 8144cb4cf..d72230421 100644 --- a/etc/kayobe/kolla/config/prometheus/rabbitmq.rules +++ b/etc/kayobe/kolla/config/prometheus/rabbitmq.rules @@ -56,7 +56,7 @@ groups: annotations: description: RabbitMQ too much unack on {{ $labels.instance }} - alert: RabbitMQTooMuchConnections - expr: rabbitmq_connections > 1000 + expr: rabbitmq_connections > {% endraw %}{{ (1500 * groups['controllers'] | length + 50 * groups['compute'] | length) }}{% raw %} for: 2m labels: severity: warning diff --git a/releasenotes/notes/rabbitmq-connection-alert-85cc7b29ddf8e3c3.yaml b/releasenotes/notes/rabbitmq-connection-alert-85cc7b29ddf8e3c3.yaml new file mode 100644 index 000000000..dffc8b9ef --- /dev/null +++ b/releasenotes/notes/rabbitmq-connection-alert-85cc7b29ddf8e3c3.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Adapt threshold of RabbitMQ connection alert based on the size of the + deployment to avoid spurious alerts.