From 1202f63bb8bcffafbee46e773de2b13bd015a69b Mon Sep 17 00:00:00 2001 From: Jennefer Cullinan Date: Wed, 13 Sep 2023 08:58:53 +0100 Subject: [PATCH] feat: new grafana panel for release latency seconds new metric release latency seconds new grafana panel of same name add functionality collect timestamps for release latency testing not written until code advised correct added timestamp start point in gitops snapshot.go added timestamp end point in snapshot_adapter.go timestamps called and compared in metrics integ.go Signed-off-by: Jennefer Cullinan --- .../integration-service-dashboard.json | 113 ++++++++++++++++++ controllers/snapshot/snapshot_adapter.go | 7 ++ gitops/snapshot.go | 8 +- metrics/integration.go | 13 ++ 4 files changed, 140 insertions(+), 1 deletion(-) diff --git a/config/grafana/dashboards/integration-service-dashboard.json b/config/grafana/dashboards/integration-service-dashboard.json index 3002ee8e7..6df14f4ab 100644 --- a/config/grafana/dashboards/integration-service-dashboard.json +++ b/config/grafana/dashboards/integration-service-dashboard.json @@ -1269,6 +1269,119 @@ "steppedLine": false, "timeFrom": null, "timeShift": null + }, + + { + "id": 28, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 50 + }, + "type": "heatmap", + "title": "Latency - Release Created", + "pluginVersion": "9.1.6", + "maxDataPoints": 25, + "description": "Track the time between all integration tests passed/snapshot finished testing and when release is created", + "legend": { + "show": false + }, + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + + "dataFormat": "tsbuckets", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.90, sum(rate(release_latency_seconds_bucket[5m])) by (le))", + "format": "heatmap", + "interval": "", + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "tooltip": { + "show": true, + "showHistogram": false + }, + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "options": { + "calculate": false, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "min": null, + "max": null, + "unit": "short", + "decimals": null + }, + "rowsFrame": { + "layout": "auto" + }, + "color": { + "mode": "scheme", + "fill": "#b4ff00", + "scale": "exponential", + "exponent": 0.5, + "scheme": "Oranges", + "steps": 128, + "reverse": false + }, + "cellGap": 2, + "filterValues": { + "le": 1e-9 + }, + "tooltip": { + "show": true, + "yHistogram": false + }, + "legend": { + "show": false + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "calculation": {}, + "cellValues": {}, + "showValue": "never" + }, + "reverseYBuckets": false, + "timeFrom": null, + "timeShift": null, + "xBucketNumber": null, + "xBucketSize": null, + "yBucketNumber": null, + "yBucketSize": null } ], diff --git a/controllers/snapshot/snapshot_adapter.go b/controllers/snapshot/snapshot_adapter.go index 535718a3a..62b7ef9e7 100644 --- a/controllers/snapshot/snapshot_adapter.go +++ b/controllers/snapshot/snapshot_adapter.go @@ -21,6 +21,7 @@ import ( "fmt" "reflect" "strings" + "time" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" @@ -41,6 +42,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +var endCreateRelease time.Time + // Adapter holds the objects needed to reconcile a Release. type Adapter struct { snapshot *applicationapiv1alpha1.Snapshot @@ -427,6 +430,10 @@ func (a *Adapter) createMissingReleasesForReleasePlans(application *applicationa if err != nil { return err } + + // Set the end timestamp here after successfully creating a release + endCreateRelease = time.Now() + a.logger.LogAuditEvent("Created a new Release", newRelease, h.LogActionAdd, "releasePlan.Name", releasePlan.Name) diff --git a/gitops/snapshot.go b/gitops/snapshot.go index 043b01c66..1d48c2dbd 100644 --- a/gitops/snapshot.go +++ b/gitops/snapshot.go @@ -142,6 +142,7 @@ const ( var ( // SnapshotComponentLabel contains the name of the updated Snapshot component - it should match the pipeline label. SnapshotComponentLabel = tekton.ComponentNameLabel + startCreateRelease time.Time ) // MarkSnapshotAsPassed updates the AppStudio Test succeeded condition for the Snapshot to passed. @@ -327,8 +328,13 @@ func HaveAppStudioTestsFinished(snapshot *applicationapiv1alpha1.Snapshot) bool // HaveAppStudioTestsSucceeded checks if the AppStudio tests have finished by checking if the AppStudio Test Succeeded condition is set. func HaveAppStudioTestsSucceeded(snapshot *applicationapiv1alpha1.Snapshot) bool { if meta.FindStatusCondition(snapshot.Status.Conditions, AppStudioTestSucceededCondition) == nil { - return meta.IsStatusConditionTrue(snapshot.Status.Conditions, LegacyTestSucceededCondition) + if meta.IsStatusConditionTrue(snapshot.Status.Conditions, LegacyTestSucceededCondition) { + startCreateRelease = time.Now() + return true + } + return false } + startCreateRelease = time.Now() return meta.IsStatusConditionTrue(snapshot.Status.Conditions, AppStudioTestSucceededCondition) } diff --git a/metrics/integration.go b/metrics/integration.go index 9bd83e1bf..a7d3a1359 100644 --- a/metrics/integration.go +++ b/metrics/integration.go @@ -61,6 +61,14 @@ var ( }, []string{"type", "reason"}, ) + + ReleaseLatencySeconds = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Name: "release_latency_seconds", + Help: "Latency between integration tests completion and release creation", + Buckets: []float64{0.05, 0.1, 0.5, 1, 2, 3, 4, 5, 10, 15, 30}, + }, + ) ) func RegisterCompletedSnapshot(conditiontype, reason string, startTime metav1.Time, completionTime *metav1.Time) { @@ -100,6 +108,10 @@ func RegisterNewIntegrationPipelineRun(snapshotCreatedTime metav1.Time, pipeline RegisterPipelineRunStarted(snapshotCreatedTime, pipelineRunStartTime) } +func RegisterReleaseLatency(startCreateReleaseTime metav1.Time, endCreateRelease metav1.Time) { + ReleaseLatencySeconds.Observe(endCreateRelease.Sub(startCreateReleaseTime.Time).Seconds()) +} + func init() { metrics.Registry.MustRegister( SnapshotCreatedToPipelineRunStartedSeconds, @@ -109,5 +121,6 @@ func init() { SnapshotDurationSeconds, SnapshotInvalidTotal, SnapshotTotal, + ReleaseLatencySeconds, ) }