change the alert for the openshift-image-mirroring failures to only f…

…ire when there have been no successful runs for 12 hours, and add the keep_firing_for to 2h so that alert manager won't notify us too frequently when this is the case (#60057)
openshift · Dec 19, 2024 · fc0461b · fc0461b
1 parent 8c69fb3
commit fc0461b
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 2 deletions.
diff --git a/clusters/app.ci/openshift-user-workload-monitoring/mixins/_prometheus/dptp_alerts.libsonnet b/clusters/app.ci/openshift-user-workload-monitoring/mixins/_prometheus/dptp_alerts.libsonnet
@@ -241,9 +241,15 @@
           {
             alert: 'openshift-mirroring-failures',
             expr: |||
-              increase(prowjob_state_transitions{job_name="periodic-image-mirroring-openshift",state="failure"}[5m]) > 0
+              sum by (job_name) (
+                rate(
+                  prowjob_state_transitions{job="prow-controller-manager",job_name!~"rehearse.*",state="success"}[12h]
+                )
+              )
+              * on (job_name) group_left max by (job_name) (prow_job_labels{job_agent="kubernetes",label_ci_openshift_io_role="image-mirroring",label_ci_openshift_io_area="openshift"}) == 0
             |||,
             'for': '1m',
+            'keep_firing_for': '2h',
             labels: {
               severity: 'critical',
             },

diff --git a/...ci/openshift-user-workload-monitoring/mixins/prometheus_out/ci-alerts_prometheusrule.yaml b/...ci/openshift-user-workload-monitoring/mixins/prometheus_out/ci-alerts_prometheusrule.yaml
@@ -292,8 +292,14 @@ spec:
       annotations:
         message: OpenShift image mirroring jobs have failed. View failed jobs at the <https://prow.ci.openshift.org/?job=periodic-image-mirroring-openshift|overview>.
       expr: |
-        increase(prowjob_state_transitions{job_name="periodic-image-mirroring-openshift",state="failure"}[5m]) > 0
+        sum by (job_name) (
+          rate(
+            prowjob_state_transitions{job="prow-controller-manager",job_name!~"rehearse.*",state="success"}[12h]
+          )
+        )
+        * on (job_name) group_left max by (job_name) (prow_job_labels{job_agent="kubernetes",label_ci_openshift_io_role="image-mirroring",label_ci_openshift_io_area="openshift"}) == 0
       for: 1m
+      keep_firing_for: 2h
       labels:
         severity: critical
   - name: ghproxy