diff --git a/config/grafana/dashboards/integration-service-dashboard.json b/config/grafana/dashboards/integration-service-dashboard.json index e34ad4da1..781d75dc9 100644 --- a/config/grafana/dashboards/integration-service-dashboard.json +++ b/config/grafana/dashboards/integration-service-dashboard.json @@ -1271,6 +1271,100 @@ "timeShift": null } + { + "id": 28, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "type": "graph", + "title": "Latency - Release Created", + "thresholds": [], + "pluginVersion": "9.1.6", + "description": "Track the time between all integration tests passed/snapshot finished testing and when release is created", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "aliasColors": {}, + "dashLength": 10, + "fill": 1, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "pointradius": 2, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.90, sum(rate(release_latency_seconds_bucket[5m])) by (le))", + "instant": false, + "interval": "", + "legendFormat": "{{ le }}", + "refId": "A", + "editorMode": "code" + } + ], + "timeRegions": [], + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + }, + "bars": false, + "dashes": false, + "fillGradient": 0, + "hiddenSeries": false, + "percentage": false, + "points": false, + "stack": false, + "steppedLine": false, + "timeFrom": null, + "timeShift": null, + "interval": "5m" + } + ], "schemaVersion": 27, "style": "dark", diff --git a/metrics/integration.go b/metrics/integration.go index 9bd83e1bf..e1368b307 100644 --- a/metrics/integration.go +++ b/metrics/integration.go @@ -61,6 +61,14 @@ var ( }, []string{"type", "reason"}, ) + + ReleaseLatencySeconds = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Name: "release_latency_seconds", + Help: "Latency between integration tests completion and release creation", + Buckets: []float64{7, 15, 30, 60, 150, 300, 450, 600, 750, 900, 1050}, + }, + ) ) func RegisterCompletedSnapshot(conditiontype, reason string, startTime metav1.Time, completionTime *metav1.Time) { @@ -100,6 +108,10 @@ func RegisterNewIntegrationPipelineRun(snapshotCreatedTime metav1.Time, pipeline RegisterPipelineRunStarted(snapshotCreatedTime, pipelineRunStartTime) } +func RegisterReleaseLatency(integrationTestsFinishTime metav1.Time, releaseCreationTime metav1.Time) { + ReleaseLatencySeconds.Observe(releaseCreationTime.Sub(integrationTestsFinishTime.Time).Seconds()) +} + func init() { metrics.Registry.MustRegister( SnapshotCreatedToPipelineRunStartedSeconds, @@ -109,5 +121,6 @@ func init() { SnapshotDurationSeconds, SnapshotInvalidTotal, SnapshotTotal, + ReleaseLatencySeconds, ) }