feat: new grafana panel for release latency seconds

new metric release latency seconds new grafana panel of same name add functionality collect timestamps for release latency testing not written until code advised correct added timestamp start point in gitops snapshot.go added timestamp end point in snapshot_adapter.go timestamps called and compared in metrics integ.go Signed-off-by: Jennefer Cullinan <[email protected]>
konflux-ci · Sep 18, 2023 · 1202f63 · 1202f63
1 parent d768469
commit 1202f63
Show file tree

Hide file tree

Showing 4 changed files with 140 additions and 1 deletion.
diff --git a/config/grafana/dashboards/integration-service-dashboard.json b/config/grafana/dashboards/integration-service-dashboard.json
@@ -1269,6 +1269,119 @@
       "steppedLine": false,
       "timeFrom": null,
       "timeShift": null
+    },
+
+    {
+      "id": 28,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 50
+      },
+      "type": "heatmap",
+      "title": "Latency -  Release Created",
+      "pluginVersion": "9.1.6",
+      "maxDataPoints": 25,
+      "description": "Track the time between all integration tests passed/snapshot finished testing and when release is created",
+      "legend": {
+        "show": false
+      },
+      "cards": {
+        "cardPadding": null,
+        "cardRound": null
+      },
+      "color": {
+        "cardColor": "#b4ff00",
+        "colorScale": "sqrt",
+        "colorScheme": "interpolateOranges",
+        "exponent": 0.5,
+        "mode": "spectrum"
+      },
+
+      "dataFormat": "tsbuckets",
+          "fieldConfig": {
+            "defaults": {},
+            "overrides": []
+          },
+
+      "heatmap": {},
+      "hideZeroBuckets": true,
+      "highlightCards": true,
+      "targets": [
+      { 
+          "exemplar": true,
+          "expr": "histogram_quantile(0.90, sum(rate(release_latency_seconds_bucket[5m])) by (le))",
+          "format": "heatmap",
+          "interval": "",
+          "legendFormat": "{{le}}",
+          "refId": "A"
+     }
+      ],
+      "tooltip": {
+        "show": true,
+        "showHistogram": false
+      },
+      "xAxis": {
+        "show": true
+      },
+      "yAxis": {
+        "decimals": null,
+        "format": "short",
+        "logBase": 1,
+        "max": null,
+        "min": null,
+        "show": true,
+        "splitFactor": null
+      },
+      "yBucketBound": "auto",
+      "options": {
+        "calculate": false,
+        "yAxis": {
+          "axisPlacement": "left",
+          "reverse": false,
+          "min": null,
+          "max": null,
+          "unit": "short",
+          "decimals": null
+        },
+        "rowsFrame": {
+          "layout": "auto"
+        },
+        "color": {
+          "mode": "scheme",
+          "fill": "#b4ff00",
+          "scale": "exponential",
+          "exponent": 0.5,
+          "scheme": "Oranges",
+          "steps": 128,
+          "reverse": false
+        },
+        "cellGap": 2,
+        "filterValues": {
+          "le": 1e-9
+        },
+        "tooltip": {
+          "show": true,
+          "yHistogram": false
+        },
+        "legend": {
+          "show": false
+        },
+        "exemplars": {
+          "color": "rgba(255,0,255,0.7)"
+        },
+        "calculation": {},
+        "cellValues": {},
+        "showValue": "never"
+      },
+      "reverseYBuckets": false,
+      "timeFrom": null,
+      "timeShift": null,
+      "xBucketNumber": null,
+      "xBucketSize": null,
+      "yBucketNumber": null,
+      "yBucketSize": null
     }
 
   ],

diff --git a/controllers/snapshot/snapshot_adapter.go b/controllers/snapshot/snapshot_adapter.go
@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"reflect"
 	"strings"
+	"time"
 
 	"k8s.io/apimachinery/pkg/types"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -41,6 +42,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+var endCreateRelease time.Time
+
 // Adapter holds the objects needed to reconcile a Release.
 type Adapter struct {
 	snapshot    *applicationapiv1alpha1.Snapshot
@@ -427,6 +430,10 @@ func (a *Adapter) createMissingReleasesForReleasePlans(application *applicationa
 			if err != nil {
 				return err
 			}
+
+			// Set the end timestamp here after successfully creating a release
+			endCreateRelease = time.Now()
+
 			a.logger.LogAuditEvent("Created a new Release", newRelease, h.LogActionAdd,
 				"releasePlan.Name", releasePlan.Name)
 

diff --git a/gitops/snapshot.go b/gitops/snapshot.go
@@ -142,6 +142,7 @@ const (
 var (
 	// SnapshotComponentLabel contains the name of the updated Snapshot component - it should match the pipeline label.
 	SnapshotComponentLabel = tekton.ComponentNameLabel
+	startCreateRelease     time.Time
 )
 
 // MarkSnapshotAsPassed updates the AppStudio Test succeeded condition for the Snapshot to passed.
@@ -327,8 +328,13 @@ func HaveAppStudioTestsFinished(snapshot *applicationapiv1alpha1.Snapshot) bool
 // HaveAppStudioTestsSucceeded checks if the AppStudio tests have finished by checking if the AppStudio Test Succeeded condition is set.
 func HaveAppStudioTestsSucceeded(snapshot *applicationapiv1alpha1.Snapshot) bool {
 	if meta.FindStatusCondition(snapshot.Status.Conditions, AppStudioTestSucceededCondition) == nil {
-		return meta.IsStatusConditionTrue(snapshot.Status.Conditions, LegacyTestSucceededCondition)
+		if meta.IsStatusConditionTrue(snapshot.Status.Conditions, LegacyTestSucceededCondition) {
+			startCreateRelease = time.Now()
+			return true
+		}
+		return false
 	}
+	startCreateRelease = time.Now()
 	return meta.IsStatusConditionTrue(snapshot.Status.Conditions, AppStudioTestSucceededCondition)
 }
 

diff --git a/metrics/integration.go b/metrics/integration.go
@@ -61,6 +61,14 @@ var (
 		},
 		[]string{"type", "reason"},
 	)
+
+	ReleaseLatencySeconds = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Name:    "release_latency_seconds",
+			Help:    "Latency between integration tests completion and release creation",
+			Buckets: []float64{0.05, 0.1, 0.5, 1, 2, 3, 4, 5, 10, 15, 30},
+		},
+	)
 )
 
 func RegisterCompletedSnapshot(conditiontype, reason string, startTime metav1.Time, completionTime *metav1.Time) {
@@ -100,6 +108,10 @@ func RegisterNewIntegrationPipelineRun(snapshotCreatedTime metav1.Time, pipeline
 	RegisterPipelineRunStarted(snapshotCreatedTime, pipelineRunStartTime)
 }
 
+func RegisterReleaseLatency(startCreateReleaseTime metav1.Time, endCreateRelease metav1.Time) {
+	ReleaseLatencySeconds.Observe(endCreateRelease.Sub(startCreateReleaseTime.Time).Seconds())
+}
+
 func init() {
 	metrics.Registry.MustRegister(
 		SnapshotCreatedToPipelineRunStartedSeconds,
@@ -109,5 +121,6 @@ func init() {
 		SnapshotDurationSeconds,
 		SnapshotInvalidTotal,
 		SnapshotTotal,
+		ReleaseLatencySeconds,
 	)
 }