Fix issue creation for grafana alerts (#30424)

* fix issue creation for grafana alerts * fix issue creation for grafana alerts * fix issue creation for grafana alerts * fix dashboard category issues * addressed the comments * code cleanup * restrict number of runs to consider
apache · Feb 29, 2024 · a20705c · a20705c
1 parent aebb34a
commit a20705c
Show file tree

Hide file tree

Showing 6 changed files with 170 additions and 157 deletions.
diff --git a/.github/workflows/flaky_test_detection.yml b/.github/workflows/flaky_test_detection.yml
@@ -21,16 +21,19 @@ permissions:
   issues: write
 
 on:
-  workflow_dispatch: 
+  schedule:
+    - cron: '30 */3 * * *'
+  workflow_dispatch:
     inputs:
       READ_ONLY:
         description: 'Run in read-only mode'
         required: false
         default: 'true'
-    
+
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   READ_ONLY: ${{ inputs.READ_ONLY }}
+
 jobs:
   flaky-test-detection:
     runs-on: [self-hosted, ubuntu-20.04, main]

diff --git a/.test-infra/metrics/grafana/dashboards/GA-Post-Commits_status_dashboard.json b/.test-infra/metrics/grafana/dashboards/GA-Post-Commits_status_dashboard.json
diff --git a/.test-infra/metrics/grafana/provisioning/alerts/flaky_test.yaml b/.test-infra/metrics/grafana/provisioning/alerts/flaky_test.yaml
@@ -21,7 +21,7 @@ groups:
       rules:
         - uid: flaky_test
           title: flaky_test
-          condition: E
+          condition: B
           data:
             - refId: A
               relativeTimeRange:
@@ -35,113 +35,49 @@ groups:
                 intervalMs: 1000
                 maxDataPoints: 43200
                 rawQuery: true
-                rawSql: "SELECT COUNT(run_id), CAST (github_workflows.workflow_id AS TEXT), name AS workflow_name , filename AS workflow_filename , dashboard_category , github_workflows.url AS workflow_url, CAST(threshold AS TEXT) AS workflow_treshold \nFROM github_workflow_runs INNER JOIN github_workflows\nON github_workflow_runs.workflow_id = github_workflows.workflow_id \nWHERE status = 'failure' \nGROUP BY github_workflows.workflow_id, workflow_name, workflow_filename"
+                rawSql: |-
+                  SELECT COUNT(workflow_id), CAST(workflow_id AS TEXT), name AS workflow_name, filename AS workflow_filename, url AS workflow_url, CAST(threshold AS TEXT) AS workflow_threshold
+                  FROM github_workflows
+                  WHERE is_flaky = true
+                  GROUP BY workflow_id
                 refId: A
+                sql:
+                  columns:
+                    - parameters: [ ]
+                      type: function
+                  groupBy:
+                    - property:
+                        type: string
+                      type: groupBy
+                  limit: 50
             - refId: B
-              relativeTimeRange:
-                from: 600
-                to: 0
-              datasourceUid: beampsql
-              model:
-                datasource:
-                    type: postgres
-                    uid: beampsql
-                editorMode: code
-                format: table
-                hide: false
-                intervalMs: 1000
-                maxDataPoints: 43200
-                rawQuery: true
-                rawSql: SELECT MAX(threshold), CAST(workflow_id AS TEXT) FROM github_workflows GROUP BY workflow_id
-                refId: B
-            - refId: C
-              relativeTimeRange:
-                from: 600
-                to: 0
-              datasourceUid: __expr__
-              model:
-                conditions:
-                    - evaluator:
-                        params:
-                            - 0
-                            - 0
-                        type: gt
-                      operator:
-                        type: and
-                      query:
-                        params: []
-                      reducer:
-                        params: []
-                        type: avg
-                      type: query
-                datasource:
-                    name: Expression
-                    type: __expr__
-                    uid: __expr__
-                expression: 30 * $B
-                hide: false
-                intervalMs: 1000
-                maxDataPoints: 43200
-                refId: C
-                type: math
-            - refId: D
               relativeTimeRange:
                 from: 600
                 to: 0
               datasourceUid: __expr__
               model:
                 conditions:
-                    - evaluator:
-                        params:
-                            - 0
-                            - 0
-                        type: gt
-                      operator:
-                        type: and
-                      query:
-                        params: []
-                      reducer:
-                        params: []
-                        type: avg
-                      type: query
+                  - evaluator:
+                      params:
+                        - 0
+                        - 0
+                      type: gt
+                    operator:
+                      type: and
+                    query:
+                      params: [ ]
+                    reducer:
+                      params: [ ]
+                      type: avg
+                    type: query
                 datasource:
-                    name: Expression
-                    type: __expr__
-                    uid: __expr__
-                expression: $C - $A
-                hide: false
+                  name: Expression
+                  type: __expr__
+                  uid: __expr__
+                expression: A
                 intervalMs: 1000
                 maxDataPoints: 43200
-                refId: D
-                type: math
-            - refId: E
-              relativeTimeRange:
-                from: 600
-                to: 0
-              datasourceUid: __expr__
-              model:
-                conditions:
-                    - evaluator:
-                        params:
-                            - 0
-                            - 0
-                        type: lt
-                      operator:
-                        type: and
-                      query:
-                        params: []
-                      reducer:
-                        params: []
-                        type: avg
-                      type: query
-                datasource:
-                    name: Expression
-                    type: __expr__
-                    uid: __expr__
-                expression: D
-                intervalMs: 1000
-                maxDataPoints: 43200
-                refId: E
+                refId: B
                 type: threshold
           noDataState: NoData
           execErrState: Error