From 09fc703d49b54fcceaa8995671e47e3b9e737dc9 Mon Sep 17 00:00:00 2001 From: gabemontero Date: Wed, 4 Oct 2023 11:47:18 -0400 Subject: [PATCH] add row/panel for scheduling alert - this change adds a panel for alerts, and then adds the query we use for our SLO scheduling alert, setting the threshold to match our initial SLO overhead percentage, 3% - also updated the developer instructions to explain how to export without including the datasource uid's set during local testing --- .../argocd/pipeline-service-o11y/README.md | 5 +- .../pipeline-service-dashboard.json | 7134 +++++++++-------- 2 files changed, 3624 insertions(+), 3515 deletions(-) diff --git a/developer/openshift/gitops/argocd/pipeline-service-o11y/README.md b/developer/openshift/gitops/argocd/pipeline-service-o11y/README.md index d48c6ecd7..469285777 100644 --- a/developer/openshift/gitops/argocd/pipeline-service-o11y/README.md +++ b/developer/openshift/gitops/argocd/pipeline-service-o11y/README.md @@ -15,11 +15,10 @@ To iterate on your dashboard (deploying with `--use-current-branch`): 1. Navigate to the Pipeline Service dashboard in Grafana. 2. Add the panels and/or rows for your metrics as desired. -3. Click the "Share" icon to save the dashboard to JSON. +3. Click the "Share" icon to save the dashboard to JSON. You use the "Export" tab. Be sure and check "Export for sharing externally"; otherwise, the `datasource` references will have the `uid` entries specific to your cluster, which are not portable to staging or prod long term. NOTE: pipeline-service PR checks should prevent you from checking in dashboard JSON with those refs. 4. Copy the JSON into the pipeline-service-dashboard.json file, located in `operator/gitops/argocd/grafana/dashboards`. -5. Remove any `datasource` references with `uid` entries. Those `uid` entries will not work on staging or prod. And if you are editing the JSON file form staging or prod, those `uid` entries will become stale. -6. Commit the updated JSON, push to your branch, and verify the dashboard is +5. Commit the updated JSON, push to your branch, and verify the dashboard is updated once ArgoCD syncs your repository. ## Components diff --git a/operator/gitops/argocd/grafana/dashboards/pipeline-service-dashboard.json b/operator/gitops/argocd/grafana/dashboards/pipeline-service-dashboard.json index 5ad4b0b6c..eb4383ada 100644 --- a/operator/gitops/argocd/grafana/dashboards/pipeline-service-dashboard.json +++ b/operator/gitops/argocd/grafana/dashboards/pipeline-service-dashboard.json @@ -1,4 +1,59 @@ { + "__inputs": [ + { + "name": "DS_PROMETHEUS-APPSTUDIO-DS", + "label": "prometheus-appstudio-ds", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.1.6" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph (old)", + "version": "" + }, + { + "type": "panel", + "id": "heatmap", + "name": "Heatmap", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], "annotations": { "list": [ { @@ -20,129 +75,27 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 12, + "id": null, "links": [], "liveNow": false, "panels": [ { - "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "id": 4, - "panels": [], - "targets": [ - { - "refId": "A" - } - ], - "title": "Overview", + "id": 93, + "title": "Alerts", "type": "row" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "Completed PipelineRuns per hour", - "fieldConfig": { - "defaults": { - "unit": "short" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 1 + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS-APPSTUDIO-DS}" }, - "hiddenSeries": false, - "id": 2, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:84", - "alias": "{status=\"failed\"}", - "color": "#F2495C" - }, - { - "$$hashKey": "object:92", - "alias": "{status=\"success\"}", - "color": "#96D98D" - } - ], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum by (status) (rate(tekton_pipelines_controller_pipelinerun_count[1h]))* 3600", - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "PipelineRuns Per Hour", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "description": "PipelineRun success per hour", + "description": "The ratio of time needed for the Tekton controller to start processing a PipelineRun after its creation vs. the time needed to execute successful PipelineRuns", "fieldConfig": { "defaults": { "color": { @@ -160,7 +113,7 @@ }, { "color": "red", - "value": 80 + "value": 3 } ] }, @@ -171,10 +124,10 @@ "gridPos": { "h": 9, "w": 4, - "x": 12, + "x": 0, "y": 1 }, - "id": 8, + "id": 95, "options": { "colorMode": "value", "graphMode": "area", @@ -193,8 +146,13 @@ "pluginVersion": "9.1.6", "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS-APPSTUDIO-DS}" + }, + "editorMode": "code", "exemplar": true, - "expr": "sum (rate(tekton_pipelines_controller_pipelinerun_count{status=\"success\"}[1h])) / sum (rate(tekton_pipelines_controller_pipelinerun_count[1h]))", + "expr": " (sum(pipelinerun_duration_scheduled_seconds_sum) - sum(pipelinerun_duration_scheduled_seconds_sum offset 30m)) / (sum(pipelinerun_duration_scheduled_seconds_count) - sum(pipelinerun_duration_scheduled_seconds_count offset 30m)) / (sum(tekton_pipelines_controller_pipelinerun_duration_seconds_sum{status='success'}) - sum(tekton_pipelines_controller_pipelinerun_duration_seconds_sum{status='success'} offset 30m)) / (sum(tekton_pipelines_controller_pipelinerun_duration_seconds_count{status='success'}) - sum(tekton_pipelines_controller_pipelinerun_duration_seconds_count{status='success'} offset 30m))", "format": "table", "hide": false, "instant": false, @@ -203,3586 +161,3738 @@ "refId": "pipelinerun-success" } ], - "title": "PipelineRun Success", + "title": "Scheduling", "transformations": [], "type": "stat" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "Completed TaskRuns Per Hour", - "fieldConfig": { - "defaults": { - "unit": "short" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, + "collapsed": true, "gridPos": { - "h": 8, - "w": 12, + "h": 1, + "w": 24, "x": 0, "y": 10 }, - "hiddenSeries": false, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:198", - "alias": "{status=\"failed\"}", - "color": "#E02F44" - }, - { - "$$hashKey": "object:237", - "alias": "{status=\"success\"}", - "color": "#96D98D" - } - ], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum by (status) (rate(tekton_pipelines_controller_taskrun_count[1h]))* 3600", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "TaskRuns Per Hour", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true + "id": 4, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Completed PipelineRuns per hour", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 11 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:84", + "alias": "{status=\"failed\"}", + "color": "#F2495C" + }, + { + "$$hashKey": "object:92", + "alias": "{status=\"success\"}", + "color": "#96D98D" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (status) (rate(tekton_pipelines_controller_pipelinerun_count[1h]))* 3600", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "PipelineRuns Per Hour", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "description": "TaskRun success per hour", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "description": "PipelineRun success per hour", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - { - "color": "red", - "value": 80 - } - ] + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 4, - "x": 12, - "y": 10 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" + "gridPos": { + "h": 9, + "w": 4, + "x": 12, + "y": 11 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "exemplar": true, + "expr": "sum (rate(tekton_pipelines_controller_pipelinerun_count{status=\"success\"}[1h])) / sum (rate(tekton_pipelines_controller_pipelinerun_count[1h]))", + "format": "table", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "pipelinerun-success" + } ], - "fields": "", - "values": false + "title": "PipelineRun Success", + "transformations": [], + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Completed TaskRuns Per Hour", + "fieldConfig": { + "defaults": { + "unit": "short" + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:198", + "alias": "{status=\"failed\"}", + "color": "#E02F44" + }, + { + "$$hashKey": "object:237", + "alias": "{status=\"success\"}", + "color": "#96D98D" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (status) (rate(tekton_pipelines_controller_taskrun_count[1h]))* 3600", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "TaskRuns Per Hour", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "9.1.6", - "targets": [ { - "exemplar": true, - "expr": "sum (rate(tekton_pipelines_controller_taskrun_count{status=\"success\"}[1h])) / sum (rate(tekton_pipelines_controller_taskrun_count[1h]))", - "format": "table", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "", - "refId": "pipelinerun-success" + "description": "TaskRun success per hour", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 12, + "y": 20 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "exemplar": true, + "expr": "sum (rate(tekton_pipelines_controller_taskrun_count{status=\"success\"}[1h])) / sum (rate(tekton_pipelines_controller_taskrun_count[1h]))", + "format": "table", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "pipelinerun-success" + } + ], + "title": "TaskRun Success", + "transformations": [], + "type": "stat" } ], - "title": "TaskRun Success", - "transformations": [], - "type": "stat" + "title": "PipelineRun and TaskRun Rates", + "type": "row" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 18 + "y": 11 }, "id": 11, - "panels": [], - "repeat": "datasource", - "targets": [ + "panels": [ + { + "description": "Total average CPU usage of the Pipelines.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "cpu", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 13, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.5.17", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"tekton-pipelines-controller-.*\", namespace=\"openshift-pipelines\", container=\"\"}[5m]))", + "format": "table", + "interval": "", + "legendFormat": "", + "refId": "Pipelines Controller" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, { - "refId": "A" + "description": "Working set memory usage of Pipelines Controller Containers ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "memory", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 15, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.5.17", + "targets": [ + { + "exemplar": true, + "expr": "sum(container_memory_working_set_bytes{pod=~\"tekton-pipelines-controller-.*\", namespace=\"openshift-pipelines\", container=\"\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Memory Utilization", + "type": "timeseries" } ], + "repeat": "datasource", "title": "Pipelines-Controller Resource Utlization", "type": "row" }, { - "description": "Total average CPU usage of the Pipelines.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "cpu", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 29, + "panels": [ + { + "description": "Number of Pipelineruns executing currently", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - { - "color": "red", - "value": 80 + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] } - ] + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 13 + }, + "id": 31, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" }, - "unit": "short" + "pluginVersion": "9.1.6", + "targets": [ + { + "exemplar": true, + "expr": "tekton_pipelines_controller_running_pipelineruns_count{prometheus=\"openshift-user-workload-monitoring/user-workload\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Pipelineruns Running", + "type": "stat" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 19 - }, - "id": 13, - "options": { - "graph": {}, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + { + "description": "Number of Taskruns executing currently", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 13 + }, + "id": 33, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "exemplar": true, + "expr": "tekton_pipelines_controller_running_taskruns_count{prometheus=\"openshift-user-workload-monitoring/user-workload\"}", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Taskruns Running", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Frequency of Pipelines Controller restarts (This is experimental and may be removed in future)", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 13 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(kube_pod_container_status_restarts_total{namespace=\"openshift-pipelines\", pod=~\"tekton-pipelines-controller-.*\"}[7d])", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Pipelines Controller Container Restarts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "7.5.17", - "targets": [ { - "exemplar": true, - "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"tekton-pipelines-controller-.*\", namespace=\"openshift-pipelines\", container=\"\"}[5m]))", - "format": "table", - "interval": "", - "legendFormat": "", - "refId": "Pipelines Controller" - } - ], - "title": "CPU Usage", - "type": "timeseries" - }, - { - "description": "Working set memory usage of Pipelines Controller Containers ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "memory", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "description": "The workqueue for the Tekton Pipelines Reconcilers", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 37, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null + "pluginVersion": "7.5.17", + "targets": [ + { + "exemplar": true, + "expr": "avg(tekton_pipelines_controller_workqueue_depth)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Workqueue Depth", + "type": "timeseries" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Number of taskruns executing currently whose underlying Pods or Containers are suspended by k8s because of defined ResourceQuotas", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "hiddenSeries": false, + "id": 76, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "tekton_pipelines_controller_running_taskruns_throttled_by_quota_count", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Taskruns Throttled by Quota", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "description": "Scheduling latency for the Taskruns pods", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "color": "red", - "value": 80 - } - ] + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 19 - }, - "id": 15, - "options": { - "graph": {}, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 41, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "7.5.17", + "targets": [ + { + "exemplar": true, + "expr": "avg(tekton_pipelines_controller_taskruns_pod_latency)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Taskruns Pod Latency", + "type": "timeseries" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Number of taskruns executing currently whose underlying Pods or Containers are suspended by k8s because of Node level constraints", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 30 + }, + "hiddenSeries": false, + "id": 77, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "tekton_pipelines_controller_running_taskruns_throttled_by_node_count", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Taskruns Throttled by Node", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "7.5.17", - "targets": [ { - "exemplar": true, - "expr": "sum(container_memory_working_set_bytes{pod=~\"tekton-pipelines-controller-.*\", namespace=\"openshift-pipelines\", container=\"\"})", - "interval": "", - "legendFormat": "", - "refId": "A" + "description": "Total number of Kubernetes API requests by Tekton Controller", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 39, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "7.5.17", + "targets": [ + { + "exemplar": true, + "expr": "rate(tekton_pipelines_controller_client_results{prometheus=\"openshift-user-workload-monitoring/user-workload\"}[1h])*3600", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Pipelines Client Results", + "type": "timeseries" } ], - "title": "Memory Utilization", - "type": "timeseries" + "repeat": "datasource", + "title": "Pipelines Controller Monitoring", + "type": "row" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 27 + "y": 13 }, - "id": 29, - "panels": [], - "repeat": "datasource", - "targets": [ + "id": 43, + "panels": [ { - "refId": "A" - } - ], - "title": "Pipelines Controller Monitoring", - "type": "row" - }, - { - "description": "Number of Pipelineruns executing currently", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "description": "Average duration of successful Pipelineruns in seconds over one hour time window grouped by namespace.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - { - "color": "red", - "value": 80 + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 6, - "x": 0, - "y": 28 - }, - "id": 31, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 45, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "exemplar": true, + "expr": "sum by (namespace) ((avg_over_time(tekton_pipelines_controller_pipelinerun_duration_seconds_sum{status=\"success\"}[1h]) / avg_over_time(tekton_pipelines_controller_pipelinerun_duration_seconds_count{status=\"success\"}[1h])))", + "interval": "", + "legendFormat": "", + "refId": "A" + } ], - "fields": "", - "values": false + "title": "Average duration of successful pipelineruns", + "type": "bargauge" }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "9.1.6", - "targets": [ { - "exemplar": true, - "expr": "tekton_pipelines_controller_running_pipelineruns_count{prometheus=\"openshift-user-workload-monitoring/user-workload\"}", - "interval": "", - "legendFormat": "", - "refId": "A" + "description": "Average scheduling duration per PipelineRun in seconds for each namespace over the last 1 hour", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 47, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "exemplar": true, + "expr": "sum by (namespace) ((avg_over_time(pipelinerun_duration_scheduled_seconds_sum[1h]) / avg_over_time(pipelinerun_duration_scheduled_seconds_count[1h])))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Average scheduling duration", + "type": "bargauge" } ], - "title": "Pipelineruns Running", - "type": "stat" + "repeat": "datasource", + "title": "Pipelines Troubleshooting", + "type": "row" }, { - "description": "Number of Taskruns executing currently", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, + "collapsed": true, "gridPos": { - "h": 9, - "w": 6, - "x": 6, - "y": 28 + "h": 1, + "w": 24, + "x": 0, + "y": 14 }, - "id": 33, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" + "id": 17, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Total average CPU usage of the Pipelines-as-code watcher and controller pods over the past hour.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{pod=~\"pipelines-as-code-(watcher|controller)-.*\", namespace=\"openshift-pipelines\", container=\"\"}[5m]))*100", + "interval": "", + "legendFormat": "", + "refId": "A" + } ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "9.1.6", - "targets": [ - { - "exemplar": true, - "expr": "tekton_pipelines_controller_running_taskruns_count{prometheus=\"openshift-user-workload-monitoring/user-workload\"}", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Taskruns Running", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "Frequency of Pipelines Controller restarts (This is experimental and may be removed in future)", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 28 - }, - "hiddenSeries": false, - "id": 35, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "rate(kube_pod_container_status_restarts_total{namespace=\"openshift-pipelines\", pod=~\"tekton-pipelines-controller-.*\"}[7d])", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Pipelines Controller Container Restarts", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "description": "The workqueue for the Tekton Pipelines Reconcilers", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 37 - }, - "id": 37, - "options": { - "graph": {}, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "7.5.17", - "targets": [ - { - "exemplar": true, - "expr": "avg(tekton_pipelines_controller_workqueue_depth)", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Workqueue Depth", - "type": "timeseries" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "Number of taskruns executing currently whose underlying Pods or Containers are suspended by k8s because of defined ResourceQuotas", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 37 - }, - "hiddenSeries": false, - "id": 76, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "tekton_pipelines_controller_running_taskruns_throttled_by_quota_count", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Taskruns Throttled by Quota", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "description": "Scheduling latency for the Taskruns pods", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "thresholds": [], + "timeRegions": [], + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:171", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:172", + "format": "short", + "logBase": 1, + "show": true } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 45 - }, - "id": 41, - "options": { - "graph": {}, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "7.5.17", - "targets": [ - { - "exemplar": true, - "expr": "avg(tekton_pipelines_controller_taskruns_pod_latency)", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Taskruns Pod Latency", - "type": "timeseries" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "Number of taskruns executing currently whose underlying Pods or Containers are suspended by k8s because of Node level constraints", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 45 - }, - "hiddenSeries": false, - "id": 77, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "tekton_pipelines_controller_running_taskruns_throttled_by_node_count", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Taskruns Throttled by Node", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true + ], + "yaxis": { + "align": false + } }, { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "description": "Total number of Kubernetes API requests by Tekton Controller", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Working set memory usage of Pipelines-as-code watcher and controller pods over the past hour.", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (pod) (container_memory_working_set_bytes{pod=~\"pipelines-as-code-(watcher|controller)-.*\", namespace=\"openshift-pipelines\", container=\"\"})", + "interval": "", + "legendFormat": "", + "refId": "A" } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 53 - }, - "id": 39, - "options": { - "graph": {}, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "7.5.17", - "targets": [ - { - "exemplar": true, - "expr": "rate(tekton_pipelines_controller_client_results{prometheus=\"openshift-user-workload-monitoring/user-workload\"}[1h])*3600", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Pipelines Client Results", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 61 - }, - "id": 43, - "panels": [], - "repeat": "datasource", - "targets": [ - { - "refId": "A" - } - ], - "title": "Pipelines Troubleshooting", - "type": "row" - }, - { - "description": "Average duration of successful Pipelineruns in seconds over one hour time window grouped by namespace.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 62 - }, - "id": 45, - "options": { - "displayMode": "gradient", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "text": {} - }, - "pluginVersion": "9.1.6", - "targets": [ - { - "exemplar": true, - "expr": "sum by (namespace) ((avg_over_time(tekton_pipelines_controller_pipelinerun_duration_seconds_sum{status=\"success\"}[1h]) / avg_over_time(tekton_pipelines_controller_pipelinerun_duration_seconds_count{status=\"success\"}[1h])))", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Average duration of successful pipelineruns", - "type": "bargauge" - }, - { - "description": "Average scheduling duration per PipelineRun in seconds for each namespace over the last 1 hour", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 62 - }, - "id": 47, - "options": { - "displayMode": "gradient", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "text": {} - }, - "pluginVersion": "9.1.6", - "targets": [ - { - "exemplar": true, - "expr": "sum by (namespace) ((avg_over_time(pipelinerun_duration_scheduled_seconds_sum[1h]) / avg_over_time(pipelinerun_duration_scheduled_seconds_count[1h])))", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "title": "Average scheduling duration", - "type": "bargauge" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 70 - }, - "id": 17, - "panels": [], - "repeat": "datasource", - "targets": [ - { - "refId": "A" - } - ], - "title": "Pipelines-as-code Monitoring", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "Total average CPU usage of the Pipelines-as-code watcher and controller pods over the past hour.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 71 - }, - "hiddenSeries": false, - "id": 21, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{pod=~\"pipelines-as-code-(watcher|controller)-.*\", namespace=\"openshift-pipelines\", container=\"\"}[5m]))*100", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:171", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:172", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "Working set memory usage of Pipelines-as-code watcher and controller pods over the past hour.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 71 - }, - "hiddenSeries": false, - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum by (pod) (container_memory_working_set_bytes{pod=~\"pipelines-as-code-(watcher|controller)-.*\", namespace=\"openshift-pipelines\", container=\"\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory Utilization", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:80", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:81", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "Number of Pipelineruns watched on pull_request and push events", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 79 - }, - "hiddenSeries": false, - "id": 27, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum by (event_type) (rate(pac_watcher_pipelines_as_code_pipelinerun_count{prometheus=\"openshift-user-workload-monitoring/user-workload\"}[1h]))*3600", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "PipelineRuns Watched Per Hour", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:440", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:441", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "The workqueue depth for the Pipelines-as-code Reconcilers", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 79 - }, - "hiddenSeries": false, - "id": 25, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "avg(pac_watcher_workqueue_depth)", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Watcher Workqueue Depth", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:351", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:352", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "Total number of kubernetes api request for pipelines-as-code watcher", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 87 - }, - "hiddenSeries": false, - "id": 23, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "rate(pac_watcher_client_results{prometheus=\"openshift-user-workload-monitoring/user-workload\"}[1h])*3600", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Watcher Client Results", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:262", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:263", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 95 - }, - "id": 49, - "panels": [], - "targets": [ - { - "refId": "A" - } - ], - "title": "Metrics Exporter Monitoring", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 96 - }, - "hiddenSeries": false, - "id": 51, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "workqueue_depth{namespace='openshift-pipelines',pod=~'pipeline-metrics-exporter.+'}", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Workqueue Depth", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 104 - }, - "hiddenSeries": false, - "id": 53, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace='openshift-pipelines',pod=~'pipeline-metrics-exporter-.*', container=\"\"}[5m]))", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "CPU Utilization", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 112 - }, - "hiddenSeries": false, - "id": 55, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "9.1.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "sum(container_memory_working_set_bytes{namespace='openshift-pipelines',pod=~'pipeline-metrics-exporter-.*', container=\"\"})", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Memory Utilization", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 120 - }, - "id": 60, - "panels": [], - "repeat": "datasource", - "targets": [ - { - "refId": "A" - } - ], - "title": "Tekton Results", - "type": "row" - }, - { - "description": "Status of API and Watcher", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-RdYlGr" - }, - "mappings": [ - { - "options": { - "0": { - "text": "Down" - }, - "1": { - "text": "OK" - } - }, - "type": "value" - } - ], - "max": 1, - "min": 0, - "noValue": "No Data", - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 50 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 121 - }, - "id": 61, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "value_and_name" - }, - "pluginVersion": "9.1.6", - "targets": [ - { - "exemplar": true, - "expr": "sum(up{job=\"tekton-results-api\"})", - "format": "time_series", - "hide": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "API", - "refId": "api up" - }, - { - "exemplar": true, - "expr": "sum(up{job=\"tekton-results-watcher\"})", - "format": "time_series", - "hide": false, - "interval": "", - "legendFormat": "Watcher", - "refId": "watcher up" - } - ], - "title": "Deployment Status", - "transformations": [], - "type": "stat" - }, - { - "description": "Request success rate", - "fieldConfig": { - "defaults": { - "color": { - "mode": "continuous-RdYlGr" - }, - "mappings": [], - "max": 1, - "min": 0, - "noValue": "No Data", - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 3, - "x": 6, - "y": 121 - }, - "id": 62, - "interval": "15s", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "9.1.6", - "targets": [ - { - "exemplar": true, - "expr": "1 - ((sum(rate(grpc_server_handled_total{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\", grpc_code=~\"Internal|Unavailable|Unknown|Unimplemented\"}[$__rate_interval]))) / sum(rate(grpc_server_started_total{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\"}[$__rate_interval])))\n", - "interval": "", - "intervalFactor": 4, - "legendFormat": "Success Rate", - "refId": "request success rate" - } - ], - "title": "API Success Rate", - "type": "stat" - }, - { - "description": "Watcher work queue depth", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "thresholds" - }, - "mappings": [], - "noValue": "No Data", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 3, - "x": 9, - "y": 121 - }, - "id": 63, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "9.1.6", - "targets": [ - { - "exemplar": true, - "expr": "sum(watcher_workqueue_depth{job=\"tekton-results-watcher\"})", - "interval": "", - "intervalFactor": 1, - "legendFormat": "Queue Depth", - "refId": "work queue depth" - } - ], - "title": "Watcher Work Queue Depth", - "type": "stat" - }, - { - "description": "Requests/Second", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "fixed" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 12, - "y": 121 - }, - "id": 64, - "interval": "15s", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "9.1.6", - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(grpc_server_started_total{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\"}[$__rate_interval]))", - "interval": "", - "intervalFactor": 4, - "legendFormat": "Request Rate", - "refId": "request per second" - } - ], - "title": "API Request Rate", - "type": "stat" - }, - { - "description": "Request latency 99%-tile", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "fixed" - }, - "mappings": [], - "noValue": "No Data", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 16, - "y": 121 - }, - "id": 65, - "interval": "15s", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "9.1.6", - "targets": [ - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\"}[$__rate_interval])) by (le) )", - "interval": "", - "intervalFactor": 4, - "legendFormat": "Latency", - "refId": "request latency" - } - ], - "title": "API Request Latency", - "type": "stat" - }, - { - "description": "Reconcile latency 99%-tile - \nLatency of reconcile operations.", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "green", - "mode": "thresholds" - }, - "mappings": [], - "noValue": "No Data", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 20, - "y": 121 - }, - "id": 66, - "interval": "15s", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "9.1.6", - "targets": [ - { - "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(watcher_reconcile_latency_bucket{job=\"tekton-results-watcher\"}[$__rate_interval])) by (le) )", - "interval": "", - "intervalFactor": 4, - "legendFormat": "Latency", - "refId": "reconcile latency" - } - ], - "title": "Watcher Reconcile Latency", - "type": "stat" - }, - { - "description": "Pod CPU utilisation ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "CPU", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 10, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 13, - "w": 12, - "x": 0, - "y": 127 - }, - "id": 67, - "interval": "15s", - "options": { - "graph": {}, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "7.5.17", - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"tekton-results-api-.*\", namespace=\"tekton-results\", container=\"\"}[$__rate_interval]))", - "instant": false, - "interval": "", - "intervalFactor": 4, - "legendFormat": "API Server", - "refId": "api" - }, - { - "exemplar": true, - "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"tekton-results-watcher-.*\", namespace=\"tekton-results\", container=\"\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "intervalFactor": 4, - "legendFormat": "Watcher", - "refId": "watcher" - } - ], - "title": "Container CPU Utilisation", - "type": "timeseries" - }, - { - "description": "Pod memory utilisation", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Size", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 10, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 13, - "w": 12, - "x": 12, - "y": 127 - }, - "id": 68, - "interval": "15s", - "options": { - "graph": {}, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "thresholds": [], + "timeRegions": [], + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:80", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:81", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "7.5.17", - "targets": [ { - "exemplar": true, - "expr": "sum(container_memory_working_set_bytes{pod=~\"tekton-results-api-.*\",namespace=\"tekton-results\", container=\"\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 4, - "legendFormat": "API Server", - "refId": "api" + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Number of Pipelineruns watched on pull_request and push events", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 27, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (event_type) (rate(pac_watcher_pipelines_as_code_pipelinerun_count{prometheus=\"openshift-user-workload-monitoring/user-workload\"}[1h]))*3600", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "PipelineRuns Watched Per Hour", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:440", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:441", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, { - "exemplar": true, - "expr": "sum(container_memory_working_set_bytes{pod=~\"tekton-results-watcher-.*\",namespace=\"tekton-results\", container=\"\"})", - "hide": false, - "interval": "", - "intervalFactor": 4, - "legendFormat": "Watcher", - "refId": "watcher" + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "The workqueue depth for the Pipelines-as-code Reconcilers", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "avg(pac_watcher_workqueue_depth)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Watcher Workqueue Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:351", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:352", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "Total number of kubernetes api request for pipelines-as-code watcher", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "rate(pac_watcher_client_results{prometheus=\"openshift-user-workload-monitoring/user-workload\"}[1h])*3600", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Watcher Client Results", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:262", + "format": "short", + "logBase": 1, + "show": true + }, + { + "$$hashKey": "object:263", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } } ], - "title": "Container Memory Utilisation", - "type": "timeseries" + "repeat": "datasource", + "title": "Pipelines-as-code Monitoring", + "type": "row" }, { - "description": "RPC execution rate", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Response Rate", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 10, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, + "collapsed": true, "gridPos": { - "h": 13, - "w": 12, + "h": 1, + "w": 24, "x": 0, - "y": 140 + "y": 15 }, - "id": 69, - "interval": "15s", - "options": { - "graph": {}, - "legend": { - "calcs": [ - "mean" + "id": 49, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 51, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "workqueue_depth{namespace='openshift-pipelines',pod=~'pipeline-metrics-exporter.+'}", + "interval": "", + "legendFormat": "", + "refId": "A" + } ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "thresholds": [], + "timeRegions": [], + "title": "Workqueue Depth", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "7.5.17", - "targets": [ { - "exemplar": true, - "expr": "sum(rate(grpc_server_handled_total{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\"}[$__rate_interval])) by (grpc_method)", - "interval": "", - "intervalFactor": 4, - "legendFormat": "{{grpc_method}}", - "refId": "response rate" + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 53, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace='openshift-pipelines',pod=~'pipeline-metrics-exporter-.*', container=\"\"}[5m]))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 55, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.1.6", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(container_memory_working_set_bytes{namespace='openshift-pipelines',pod=~'pipeline-metrics-exporter-.*', container=\"\"})", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Memory Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } } ], - "title": "API Response Rate", - "type": "timeseries" + "title": "Metrics Exporter Monitoring", + "type": "row" }, { - "cards": { - "cardPadding": 2, - "cardRound": 2 - }, - "color": { - "cardColor": "#73BF69", - "colorScale": "sqrt", - "colorScheme": "interpolateSpectral", - "exponent": 0.5, - "mode": "spectrum" + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 }, - "dataFormat": "tsbuckets", - "description": "Latency Distribution", - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "id": 60, + "panels": [ + { + "description": "Status of API and Watcher", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "mappings": [ + { + "options": { + "0": { + "text": "Down" + }, + "1": { + "text": "OK" + } + }, + "type": "value" + } + ], + "max": 1, + "min": 0, + "noValue": "No Data", + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 50 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 17 + }, + "id": 61, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value_and_name" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "exemplar": true, + "expr": "sum(up{job=\"tekton-results-api\"})", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "API", + "refId": "api up" }, - "scaleDistribution": { - "type": "linear" + { + "exemplar": true, + "expr": "sum(up{job=\"tekton-results-watcher\"})", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "Watcher", + "refId": "watcher up" } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 13, - "w": 12, - "x": 12, - "y": 140 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 70, - "interval": "15s", - "legend": { - "show": true - }, - "options": { - "calculate": false, - "calculation": {}, - "cellGap": 2, - "cellRadius": 2, - "cellValues": {}, - "color": { - "exponent": 0.5, - "fill": "#73BF69", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "Spectral", - "steps": 128 + ], + "title": "Deployment Status", + "transformations": [], + "type": "stat" }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" + { + "description": "Request success rate", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "mappings": [], + "max": 1, + "min": 0, + "noValue": "No Data", + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 6, + "y": 17 + }, + "id": 62, + "interval": "15s", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "exemplar": true, + "expr": "1 - ((sum(rate(grpc_server_handled_total{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\", grpc_code=~\"Internal|Unavailable|Unknown|Unimplemented\"}[$__rate_interval]))) / sum(rate(grpc_server_started_total{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\"}[$__rate_interval])))\n", + "interval": "", + "intervalFactor": 4, + "legendFormat": "Success Rate", + "refId": "request success rate" + } + ], + "title": "API Success Rate", + "type": "stat" }, - "filterValues": { - "le": 1e-9 + { + "description": "Watcher work queue depth", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "thresholds" + }, + "mappings": [], + "noValue": "No Data", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 9, + "y": 17 + }, + "id": 63, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "exemplar": true, + "expr": "sum(watcher_workqueue_depth{job=\"tekton-results-watcher\"})", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Queue Depth", + "refId": "work queue depth" + } + ], + "title": "Watcher Work Queue Depth", + "type": "stat" }, - "legend": { - "show": true + { + "description": "Requests/Second", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 17 + }, + "id": 64, + "interval": "15s", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(grpc_server_started_total{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\"}[$__rate_interval]))", + "interval": "", + "intervalFactor": 4, + "legendFormat": "Request Rate", + "refId": "request per second" + } + ], + "title": "API Request Rate", + "type": "stat" }, - "rowsFrame": { - "layout": "auto" + { + "description": "Request latency 99%-tile", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "noValue": "No Data", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 16, + "y": 17 + }, + "id": 65, + "interval": "15s", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\"}[$__rate_interval])) by (le) )", + "interval": "", + "intervalFactor": 4, + "legendFormat": "Latency", + "refId": "request latency" + } + ], + "title": "API Request Latency", + "type": "stat" }, - "showValue": "never", - "tooltip": { - "show": true, - "yHistogram": true + { + "description": "Reconcile latency 99%-tile - \nLatency of reconcile operations.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "thresholds" + }, + "mappings": [], + "noValue": "No Data", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 20, + "y": 17 + }, + "id": 66, + "interval": "15s", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(watcher_reconcile_latency_bucket{job=\"tekton-results-watcher\"}[$__rate_interval])) by (le) )", + "interval": "", + "intervalFactor": 4, + "legendFormat": "Latency", + "refId": "reconcile latency" + } + ], + "title": "Watcher Reconcile Latency", + "type": "stat" }, - "yAxis": { - "axisPlacement": "left", - "decimals": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "9.1.6", - "reverseYBuckets": false, - "targets": [ { - "exemplar": true, - "expr": "sum(increase(grpc_server_handling_seconds_bucket{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\"}[$__rate_interval])) by (le)", - "interval": "", - "intervalFactor": 4, - "legendFormat": "{{le}}", - "refId": "latency distribution" - } - ], - "title": "API Latency Distribution", - "tooltip": { - "show": true, - "showHistogram": true - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" - }, - { - "description": "Server error distribution across gRPC methods. \nStatus Codes:\nInternal, Unavailable, Unknown, Unimplemented", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Request Rate", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 10, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "description": "Pod CPU utilisation ", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "CPU", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 67, + "interval": "15s", + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] + "pluginVersion": "7.5.17", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"tekton-results-api-.*\", namespace=\"tekton-results\", container=\"\"}[$__rate_interval]))", + "instant": false, + "interval": "", + "intervalFactor": 4, + "legendFormat": "API Server", + "refId": "api" + }, + { + "exemplar": true, + "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\"tekton-results-watcher-.*\", namespace=\"tekton-results\", container=\"\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "intervalFactor": 4, + "legendFormat": "Watcher", + "refId": "watcher" + } + ], + "title": "Container CPU Utilisation", + "type": "timeseries" + }, + { + "description": "Pod memory utilisation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Size", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] }, - "unit": "reqps" + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 68, + "interval": "15s", + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.5.17", + "targets": [ + { + "exemplar": true, + "expr": "sum(container_memory_working_set_bytes{pod=~\"tekton-results-api-.*\",namespace=\"tekton-results\", container=\"\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 4, + "legendFormat": "API Server", + "refId": "api" + }, + { + "exemplar": true, + "expr": "sum(container_memory_working_set_bytes{pod=~\"tekton-results-watcher-.*\",namespace=\"tekton-results\", container=\"\"})", + "hide": false, + "interval": "", + "intervalFactor": 4, + "legendFormat": "Watcher", + "refId": "watcher" + } + ], + "title": "Container Memory Utilisation", + "type": "timeseries" }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "error codes" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "points" + { + "description": "RPC execution rate", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "id": "unit", - "value": "percentunit" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Response Rate", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - { - "id": "custom.axisLabel", - "value": "Error Distribution" - } - ] - } - ] - }, - "gridPos": { - "h": 13, - "w": 12, - "x": 0, - "y": 153 - }, - "id": 71, - "interval": "15s", - "options": { - "graph": {}, - "legend": { - "calcs": [ - "mean" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 69, + "interval": "15s", + "options": { + "graph": {}, + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "7.5.17", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(grpc_server_handled_total{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\"}[$__rate_interval])) by (grpc_method)", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{grpc_method}}", + "refId": "response rate" + } ], - "displayMode": "list", - "placement": "right", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "7.5.17", - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(grpc_server_handled_total{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\", grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_method)", - "interval": "", - "intervalFactor": 4, - "legendFormat": "RPC: {{grpc_method}}", - "refId": "error methods" + "title": "API Response Rate", + "type": "timeseries" }, { - "exemplar": true, - "expr": "sum(rate(grpc_server_handled_total{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\", grpc_code=~\"Internal|Unavailable|Unknown|Unimplemented\"}[$__rate_interval])) by (grpc_code) / ignoring(grpc_code) group_left sum(rate(grpc_server_handled_total{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\", grpc_code!=\"OK\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "intervalFactor": 4, - "legendFormat": "Error: {{grpc_code}}", - "refId": "error codes" - } - ], - "title": "API Server Errors", - "type": "timeseries" - }, - { - "description": "Other error distribution across gRPC methods. \nStatus Codes Except:\nInternal, Unavailable, Unknown, Unimplemented", - "fieldConfig": { - "defaults": { + "cards": { + "cardPadding": 2, + "cardRound": 2 + }, "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Request Rate", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 10, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "cardColor": "#73BF69", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "description": "Latency Distribution", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 36 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 70, + "interval": "15s", + "legend": { + "show": true + }, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellRadius": 2, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#73BF69", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] + "pluginVersion": "9.1.6", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(increase(grpc_server_handling_seconds_bucket{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\"}[$__rate_interval])) by (le)", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{le}}", + "refId": "latency distribution" + } + ], + "title": "API Latency Distribution", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true }, - "unit": "reqps" + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "error codes" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "points" + { + "description": "Server error distribution across gRPC methods. \nStatus Codes:\nInternal, Unavailable, Unknown, Unimplemented", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "id": "unit", - "value": "percentunit" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Request Rate", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] }, + "unit": "reqps" + }, + "overrides": [ { - "id": "custom.axisLabel", - "value": "Error Distribution" + "matcher": { + "id": "byFrameRefID", + "options": "error codes" + }, + "properties": [ + { + "id": "custom.drawStyle", + "value": "points" + }, + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.axisLabel", + "value": "Error Distribution" + } + ] } ] - } - ] - }, - "gridPos": { - "h": 13, - "w": 12, - "x": 12, - "y": 153 - }, - "id": 72, - "interval": "15s", - "options": { - "graph": {}, - "legend": { - "calcs": [ - "mean" + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 71, + "interval": "15s", + "options": { + "graph": {}, + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "7.5.17", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(grpc_server_handled_total{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\", grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_method)", + "interval": "", + "intervalFactor": 4, + "legendFormat": "RPC: {{grpc_method}}", + "refId": "error methods" + }, + { + "exemplar": true, + "expr": "sum(rate(grpc_server_handled_total{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\", grpc_code=~\"Internal|Unavailable|Unknown|Unimplemented\"}[$__rate_interval])) by (grpc_code) / ignoring(grpc_code) group_left sum(rate(grpc_server_handled_total{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\", grpc_code!=\"OK\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "intervalFactor": 4, + "legendFormat": "Error: {{grpc_code}}", + "refId": "error codes" + } ], - "displayMode": "list", - "placement": "right", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "7.5.17", - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(grpc_server_handled_total{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\", grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_method)", - "interval": "", - "intervalFactor": 4, - "legendFormat": "RPC: {{grpc_method}}", - "refId": "error methods" + "title": "API Server Errors", + "type": "timeseries" }, { - "exemplar": true, - "expr": "sum(rate(grpc_server_handled_total{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\", grpc_code!~\"Internal|Unavailable|Unknown|Unimplemented\"}[$__rate_interval])) by (grpc_code) / ignoring(grpc_code) group_left sum(rate(grpc_server_handled_total{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\", grpc_code!=\"OK\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "intervalFactor": 4, - "legendFormat": "Error: {{grpc_code}}", - "refId": "error codes" - } - ], - "title": "API Other Errors", - "type": "timeseries" - }, - { - "description": "Number of reconcile operations", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Reconcile Rate", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 10, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ + "description": "Other error distribution across gRPC methods. \nStatus Codes Except:\nInternal, Unavailable, Unknown, Unimplemented", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Request Rate", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [ { - "color": "green", - "value": null + "matcher": { + "id": "byFrameRefID", + "options": "error codes" + }, + "properties": [ + { + "id": "custom.drawStyle", + "value": "points" + }, + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.axisLabel", + "value": "Error Distribution" + } + ] } ] }, - "unit": "reqps" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Failed" + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 72, + "interval": "15s", + "options": { + "graph": {}, + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "right", + "showLegend": false }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] + "tooltip": { + "mode": "single", + "sort": "none" + } }, - { - "matcher": { - "id": "byName", - "options": "Success" + "pluginVersion": "7.5.17", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(grpc_server_handled_total{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\", grpc_code!=\"OK\"}[$__rate_interval])) by (grpc_method)", + "interval": "", + "intervalFactor": 4, + "legendFormat": "RPC: {{grpc_method}}", + "refId": "error methods" }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" + { + "exemplar": true, + "expr": "sum(rate(grpc_server_handled_total{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\", grpc_code!~\"Internal|Unavailable|Unknown|Unimplemented\"}[$__rate_interval])) by (grpc_code) / ignoring(grpc_code) group_left sum(rate(grpc_server_handled_total{job=\"tekton-results-api\", grpc_service=~\"tekton.results.v1alpha2.*\", grpc_code!=\"OK\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "intervalFactor": 4, + "legendFormat": "Error: {{grpc_code}}", + "refId": "error codes" + } + ], + "title": "API Other Errors", + "type": "timeseries" + }, + { + "description": "Number of reconcile operations", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Reconcile Rate", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Failed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] } ] - } - ] - }, - "gridPos": { - "h": 13, - "w": 12, - "x": 0, - "y": 166 - }, - "id": 73, - "interval": "15s", - "options": { - "graph": {}, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "7.5.17", - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(watcher_reconcile_count{job=\"tekton-results-watcher\"}[$__rate_interval])) by (success)", - "format": "time_series", - "interval": "", - "intervalFactor": 4, - "legendFormat": "{{success}}", - "refId": "work queue depth" - } - ], - "title": "Watcher Reconcile Rate", - "transformations": [ - { - "id": "renameByRegex", - "options": { - "regex": "(true)", - "renamePattern": "Success" - } - }, - { - "id": "renameByRegex", + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 62 + }, + "id": 73, + "interval": "15s", "options": { - "regex": "(false)", - "renamePattern": "Failed" - } - } - ], - "type": "timeseries" - }, - { - "description": "Work queue latency at queue depth", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Latency", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 10, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "pluginVersion": "7.5.17", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(watcher_reconcile_count{job=\"tekton-results-watcher\"}[$__rate_interval])) by (success)", + "format": "time_series", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{success}}", + "refId": "work queue depth" + } + ], + "title": "Watcher Reconcile Rate", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "(true)", + "renamePattern": "Success" } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byFrameRefID", - "options": "work queue depth" }, - "properties": [ - { - "id": "unit", - "value": "none" - }, - { - "id": "custom.drawStyle", - "value": "bars" - }, - { - "id": "custom.lineWidth", - "value": 1 - }, - { - "id": "custom.fillOpacity", - "value": 10 + { + "id": "renameByRegex", + "options": { + "regex": "(false)", + "renamePattern": "Failed" + } + } + ], + "type": "timeseries" + }, + { + "description": "Work queue latency at queue depth", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "id": "custom.gradientMode", - "value": "none" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Latency", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - { - "id": "custom.axisLabel", - "value": "Items" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] }, + "unit": "s" + }, + "overrides": [ { - "id": "color", - "value": { - "mode": "fixed" - } + "matcher": { + "id": "byFrameRefID", + "options": "work queue depth" + }, + "properties": [ + { + "id": "unit", + "value": "none" + }, + { + "id": "custom.drawStyle", + "value": "bars" + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 10 + }, + { + "id": "custom.gradientMode", + "value": "none" + }, + { + "id": "custom.axisLabel", + "value": "Items" + }, + { + "id": "color", + "value": { + "mode": "fixed" + } + } + ] } ] - } - ] - }, - "gridPos": { - "h": 13, - "w": 12, - "x": 12, - "y": 166 - }, - "id": 74, - "interval": "15s", - "options": { - "graph": {}, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "7.5.17", - "targets": [ - { - "exemplar": true, - "expr": "sum(rate(watcher_workqueue_queue_latency_seconds_count{job=\"tekton-results-watcher\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "intervalFactor": 4, - "legendFormat": "Queue Latency", - "refId": "work queue latency" - }, - { - "exemplar": true, - "expr": "sum(rate(watcher_reconcile_latency_count{job=\"tekton-results-watcher\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "intervalFactor": 4, - "legendFormat": "Reconcile Latency", - "refId": "reconcile latency" - }, - { - "exemplar": true, - "expr": "sum(watcher_work_queue_depth{job=\"tekton-results-watcher\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 4, - "legendFormat": "Queue Depth", - "refId": "work queue depth" + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 62 + }, + "id": 74, + "interval": "15s", + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "7.5.17", + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(watcher_workqueue_queue_latency_seconds_count{job=\"tekton-results-watcher\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "intervalFactor": 4, + "legendFormat": "Queue Latency", + "refId": "work queue latency" + }, + { + "exemplar": true, + "expr": "sum(rate(watcher_reconcile_latency_count{job=\"tekton-results-watcher\"}[$__rate_interval]))", + "hide": false, + "interval": "", + "intervalFactor": 4, + "legendFormat": "Reconcile Latency", + "refId": "reconcile latency" + }, + { + "exemplar": true, + "expr": "sum(watcher_work_queue_depth{job=\"tekton-results-watcher\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 4, + "legendFormat": "Queue Depth", + "refId": "work queue depth" + } + ], + "title": "Watcher Latency", + "type": "timeseries" } ], - "title": "Watcher Latency", - "type": "timeseries" + "repeat": "datasource", + "title": "Tekton Results", + "type": "row" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 179 + "y": 17 }, "id": 79, - "panels": [], - "repeat": "datasource", - "repeatDirection": "h", - "targets": [ + "panels": [ { - "refId": "A" - } - ], - "title": "Tekton Chains", - "type": "row" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "color": "red", - "value": 80 + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 180 - }, - "id": 81, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "editorMode": "builder", - "expr": "rate(watcher_go_gc_cpu_fraction{app=\"tekton-chains-controller\"}[5m])", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Tekton Chains CPU Fraction Use", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 81, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 180 - }, - "id": 83, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "targets": [ + { + "editorMode": "builder", + "expr": "rate(watcher_go_gc_cpu_fraction{app=\"tekton-chains-controller\"}[5m])", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Tekton Chains CPU Fraction Use", + "type": "timeseries" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { - "editorMode": "builder", - "expr": "rate(watcher_workqueue_longest_running_processor_seconds_count{app=\"tekton-chains-controller\"}[5m])", - "legendFormat": "__auto", - "range": true, - "refId": "A" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 83, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "editorMode": "builder", + "expr": "rate(watcher_workqueue_longest_running_processor_seconds_count{app=\"tekton-chains-controller\"}[5m])", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Tekton Chains Workqueue Rate", + "type": "timeseries" } ], - "title": "Tekton Chains Workqueue Rate", - "type": "timeseries" + "repeat": "datasource", + "repeatDirection": "h", + "title": "Tekton Chains", + "type": "row" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 188 + "y": 18 }, "id": 85, - "panels": [], - "title": "Sprayproxy", - "type": "row" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "color": "red", - "value": 80 + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 189 - }, - "id": 87, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "editorMode": "code", - "expr": "rate(sprayproxy_http_inbound_requests_total{namespace=\"sprayproxy\"}[5m]) * 300", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Incoming Requests", - "type": "timeseries" - }, - { - "description": "Forwarded attempts to backend server(s)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 87, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 189 - }, - "id": 89, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "targets": [ + { + "editorMode": "code", + "expr": "rate(sprayproxy_http_inbound_requests_total{namespace=\"sprayproxy\"}[5m]) * 300", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Incoming Requests", + "type": "timeseries" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { - "editorMode": "code", - "expr": "rate(sprayproxy_http_forwarded_requests_total[5m]) * 300", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Forwarded Requests", - "type": "timeseries" - }, - { - "description": "Forwarded request duration in seconds", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "description": "Forwarded attempts to backend server(s)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "color": "red", - "value": 80 + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 197 - }, - "id": 91, - "options": { - "displayMode": "gradient", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 89, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "editorMode": "code", + "expr": "rate(sprayproxy_http_forwarded_requests_total[5m]) * 300", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } ], - "fields": "", - "values": false + "title": "Forwarded Requests", + "type": "timeseries" }, - "showUnfilled": true - }, - "pluginVersion": "9.1.6", - "targets": [ { - "editorMode": "code", - "exemplar": false, - "expr": "rate(sprayproxy_http_response_time_duration_seconds_bucket[5m]) * 300", - "format": "heatmap", - "instant": false, - "interval": "", - "legendFormat": "{{le}}", - "range": true, - "refId": "A" + "description": "Forwarded request duration in seconds", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 91, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "editorMode": "code", + "exemplar": false, + "expr": "rate(sprayproxy_http_response_time_duration_seconds_bucket[5m]) * 300", + "format": "heatmap", + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Response Time Duration", + "type": "bargauge" } ], - "title": "Response Time Duration", - "type": "bargauge" + "title": "Sprayproxy", + "type": "row" } ], "refresh": "10s", @@ -3793,7 +3903,7 @@ "list": [ { "current": { - "selected": true, + "selected": false, "text": "prometheus-appstudio-ds", "value": "prometheus-appstudio-ds" },