Fix prometheus agent alerts (#913)

* Fix Prometheus agent failing alerts Signed-off-by: QuentinBisson <[email protected]> * Add outside_working_hours_inhibition back to agent shard alert Signed-off-by: QuentinBisson <[email protected]> --------- Signed-off-by: QuentinBisson <[email protected]>
giantswarm · Sep 19, 2023 · 98f70c8 · 98f70c8
1 parent 737a5e9
commit 98f70c8
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 4 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,7 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - Add missing prometheus-agent inhibition to `KubeStateMetricsDown` alert
-- Change time duration before `ManagementClusterDeploymentMissingAWS` pages because it is dependant on the `PrometheusAgentFailing` alert. 
+- Change time duration before `ManagementClusterDeploymentMissingAWS` pages because it is dependant on the `PrometheusAgentFailing` alert.
+
+### Fixed
+
+- Remove `cancel_if_outside_working_hours` from PrometheusAgent alerts.
 
 ## [2.132.0] - 2023-09-15
 

diff --git a/helm/prometheus-rules/templates/alerting-rules/prometheus-agent.rules.yml b/helm/prometheus-rules/templates/alerting-rules/prometheus-agent.rules.yml
@@ -38,7 +38,6 @@ spec:
         cancel_if_cluster_is_not_running_prometheus_agent: "true"
         cancel_if_cluster_status_creating: "true"
         cancel_if_cluster_status_deleting: "true"
-        cancel_if_outside_working_hours: "true"
     ## Page Atlas if prometheus agent is missing shards to send samples to MC prometheus.
     - alert: PrometheusAgentShardsMissing
       annotations:

diff --git a/test/tests/providers/global/prometheus-agent.rules.test.yml b/test/tests/providers/global/prometheus-agent.rules.test.yml
@@ -22,7 +22,6 @@ tests:
               cancel_if_cluster_is_not_running_prometheus_agent: "true"
               cancel_if_cluster_status_creating: "true"
               cancel_if_cluster_status_deleting: "true"
-              cancel_if_outside_working_hours: "true"
             exp_annotations:
               dashboard: "promRW001/prometheus-remote-write"
               description: "Prometheus agent remote write is failing."
@@ -44,7 +43,6 @@ tests:
               cancel_if_cluster_is_not_running_prometheus_agent: "true"
               cancel_if_cluster_status_creating: "true"
               cancel_if_cluster_status_deleting: "true"
-              cancel_if_outside_working_hours: "true"
             exp_annotations:
               dashboard: "promRW001/prometheus-remote-write"
               description: "Prometheus agent remote write is failing."