Add CiliumAPITooSlow (#1209)

Co-authored-by: Quentin Bisson <[email protected]>
giantswarm · Jun 6, 2024 · b45bd99 · b45bd99
1 parent 5ad1ff9
commit b45bd99
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 0 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Added a new alerting rule to `falco.rules.yml` to fire an alert for XZ-backdoor.
+- Add `CiliumAPITooSlow`.
 
 ### Fixed
 

diff --git a/helm/prometheus-rules/templates/platform/cabbage/alerting-rules/cilium.rules.yml b/helm/prometheus-rules/templates/platform/cabbage/alerting-rules/cilium.rules.yml
@@ -33,6 +33,18 @@ spec:
         severity: page
         team: cabbage
         topic: cilium
+    - alert: CiliumAPITooSlow
+      annotations:
+        description: '{{`Cilium API processing time is >50s pod="{{ $labels.pod }}" node="{{ $labels.node }}" method="{{ $labels.method}}" path="{{ $labels.path }}"`}}'
+        opsrecipe: cilium-performance-issues/#slow-cilium-api
+      expr: avg(rate(cilium_agent_api_process_time_seconds_sum{}[5m])/rate(cilium_agent_api_process_time_seconds_count{}[5m]) > 50) by (cluster_id, node, pod, method, path, installation, pipeline, provider)
+      for: 20m
+      labels:
+        area: platform
+        cancel_if_outside_working_hours: "true"
+        severity: page
+        team: cabbage
+        topic: cilium
     - alert: CiliumNetworkPolicyFailed
       annotations:
         description: '{{`Too many Cilium Network Policy errors.`}}'