Merge pull request #310 from jencull/stoneintg-532

feat: new grafana panel for release latency seconds

config/grafana/dashboards/integration-service-dashboard.json

@@ -1310,6 +1310,76 @@
       "yBucketBound": "auto",
       "yBucketNumber": null,
       "yBucketSize": null
+    },
+    {
+      "cards": {
+        "cardPadding": null,
+        "cardRound": null
+      },
+      "color": {
+        "cardColor": "#b4ff00",
+        "colorScale": "sqrt",
+        "colorScheme": "interpolateOranges",
+        "exponent": 0.5,
+        "mode": "spectrum"
+      },
+      "dataFormat": "tsbuckets",
+      "description": "Measure release creation latency from end of testing to release created",
+      "fieldConfig": {
+        "defaults": {},
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 27
+      },
+      "heatmap": {},
+      "hideZeroBuckets": true,
+      "highlightCards": true,
+      "id": 28,
+      "legend": {
+        "show": false
+      },
+      "maxDataPoints": 25,
+      "pluginVersion": "7.5.17",
+      "reverseYBuckets": false,
+      "targets": [
+        {
+          "exemplar": true,
+          "expr": "histogram_quantile(0.90, sum(rate(release_latency_seconds_bucket[5m])) by (le)) ",
+          "format": "heatmap",
+          "interval": "",
+          "legendFormat": "{{le}}",
+          "refId": "A"
+        }
+      ],
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "Latency of Release Creation",
+      "tooltip": {
+        "show": true,
+        "showHistogram": false
+      },
+      "type": "heatmap",
+      "xAxis": {
+        "show": true
+      },
+      "xBucketNumber": null,
+      "xBucketSize": null,
+      "yAxis": {
+        "decimals": null,
+        "format": "short",
+        "logBase": 1,
+        "max": null,
+        "min": null,
+        "show": true,
+        "splitFactor": null
+      },
+      "yBucketBound": "auto",
+      "yBucketNumber": null,
+      "yBucketSize": null
     }
 
   ],

controllers/snapshot/snapshot_adapter.go

@@ -29,6 +29,7 @@ import (
 	"github.com/redhat-appstudio/integration-service/api/v1beta1"
 	"github.com/redhat-appstudio/integration-service/gitops"
 	h "github.com/redhat-appstudio/integration-service/helpers"
+	"github.com/redhat-appstudio/integration-service/metrics"
 	"github.com/redhat-appstudio/integration-service/release"
 	"github.com/redhat-appstudio/integration-service/tekton"
 
@@ -474,6 +475,8 @@ func (a *Adapter) createMissingReleasesForReleasePlans(application *applicationa
 		return err
 	}
 
+	firstRelease := true
+
 	for _, releasePlan := range *releasePlans {
 		releasePlan := releasePlan // G601
 		existingRelease := release.FindMatchingReleaseWithReleasePlan(releases, releasePlan)
@@ -507,6 +510,14 @@ func (a *Adapter) createMissingReleasesForReleasePlans(application *applicationa
 			}
 			a.logger.Info("Marked Release status automated", "release.Name", newRelease.Name)
 		}
+		// Register the first release time for metrics calculation
+		if firstRelease {
+			startTime, ok := gitops.GetAppStudioTestsFinishedTime(a.snapshot)
+			if ok {
+				metrics.RegisterReleaseLatency(startTime)
+				firstRelease = false
+			}
+		}
 	}
 	return nil
 }

gitops/snapshot.go

@@ -360,9 +360,31 @@ func HaveAppStudioTestsSucceeded(snapshot *applicationapiv1alpha1.Snapshot) bool
 	if meta.FindStatusCondition(snapshot.Status.Conditions, AppStudioTestSucceededCondition) == nil {
 		return meta.IsStatusConditionTrue(snapshot.Status.Conditions, LegacyTestSucceededCondition)
 	}
+
 	return meta.IsStatusConditionTrue(snapshot.Status.Conditions, AppStudioTestSucceededCondition)
 }
 
+// GetTestSucceededCondition checks status of tests on the snapshot
+func GetTestSucceededCondition(snapshot *applicationapiv1alpha1.Snapshot) (condition *metav1.Condition, ok bool) {
+
+	condition = meta.FindStatusCondition(snapshot.Status.Conditions, AppStudioTestSucceededCondition)
+	if condition == nil {
+		condition = meta.FindStatusCondition(snapshot.Status.Conditions, LegacyTestSucceededCondition)
+	}
+
+	ok = (condition != nil && condition.Status != metav1.ConditionUnknown)
+	return
+}
+
+// GetAppStudioTestsFinishedTime finds the timestamp of tests succeeded condition
+func GetAppStudioTestsFinishedTime(snapshot *applicationapiv1alpha1.Snapshot) (metav1.Time, bool) {
+	condition, ok := GetTestSucceededCondition(snapshot)
+	if ok {
+		return condition.LastTransitionTime, true
+	}
+	return metav1.Time{}, false
+}
+
 // CanSnapshotBePromoted checks if the Snapshot in question can be promoted for deployment and release.
 func CanSnapshotBePromoted(snapshot *applicationapiv1alpha1.Snapshot) (bool, []string) {
 	canBePromoted := true

gitops/snapshot_test.go

@@ -227,6 +227,62 @@ var _ = Describe("Gitops functions for managing Snapshots", Ordered, func() {
 		Expect(checkResult).To(BeFalse())
 	})
 
+	It("returns true if only AppStudioTestSucceededCondition is set", func() {
+		appStudioTestSucceededCondition := "AppStudioTestSucceeded" // Local variable
+		condition := metav1.Condition{
+			Type:   appStudioTestSucceededCondition,
+			Status: metav1.ConditionTrue,
+		}
+		meta.SetStatusCondition(&hasSnapshot.Status.Conditions, condition)
+		Expect(gitops.HaveAppStudioTestsSucceeded(hasSnapshot)).To(BeTrue())
+	})
+
+	It("returns true if only LegacyTestSucceededCondition is set", func() {
+		legacyTestSucceededCondition := "HACBSStudioTestSucceeded" // Local variable
+		condition := metav1.Condition{
+			Type:   legacyTestSucceededCondition,
+			Status: metav1.ConditionTrue,
+		}
+		meta.SetStatusCondition(&hasSnapshot.Status.Conditions, condition)
+		Expect(gitops.HaveAppStudioTestsSucceeded(hasSnapshot)).To(BeTrue())
+	})
+
+	It("returns the LastTransitionTime when AppStudioTestSucceededCondition is set", func() {
+		appStudioTestSucceededCondition := "AppStudioTestSucceeded" // Local variable
+		testTime := metav1.NewTime(time.Now())
+		condition := metav1.Condition{
+			Type:               appStudioTestSucceededCondition,
+			Status:             metav1.ConditionTrue,
+			LastTransitionTime: testTime,
+		}
+		meta.SetStatusCondition(&hasSnapshot.Status.Conditions, condition)
+
+		returnedTime, ok := gitops.GetAppStudioTestsFinishedTime(hasSnapshot)
+		Expect(ok).To(BeTrue())
+		Expect(returnedTime).To(Equal(testTime))
+	})
+
+	It("returns the LastTransitionTime when LegacyTestSucceededCondition is set", func() {
+		legacyTestSucceededCondition := "HACBSStudioTestSucceeded"
+		testTime := metav1.NewTime(time.Now())
+		condition := metav1.Condition{
+			Type:               legacyTestSucceededCondition,
+			Status:             metav1.ConditionTrue,
+			LastTransitionTime: testTime,
+		}
+		meta.SetStatusCondition(&hasSnapshot.Status.Conditions, condition)
+
+		returnedTime, ok := gitops.GetAppStudioTestsFinishedTime(hasSnapshot)
+		Expect(ok).To(BeTrue())
+		Expect(returnedTime).To(Equal(testTime))
+	})
+
+	It("returns zero time when neither condition is set", func() {
+		returnedTime, ok := gitops.GetAppStudioTestsFinishedTime(hasSnapshot)
+		Expect(ok).To(BeFalse())
+		Expect(returnedTime).To(Equal(metav1.Time{})) // Empty or zero time
+	})
+
 	It("ensures that a new Snapshots can be successfully created", func() {
 		snapshotComponents := []applicationapiv1alpha1.SnapshotComponent{}
 		createdSnapshot := gitops.NewSnapshot(hasApp, &snapshotComponents)

metrics/integration.go

@@ -17,6 +17,8 @@ limitations under the License.
 package metrics
 
 import (
+	"time"
+
 	"github.com/prometheus/client_golang/prometheus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
@@ -87,6 +89,14 @@ var (
 		},
 		[]string{"type", "reason"},
 	)
+
+	ReleaseLatencySeconds = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Name:    "release_latency_seconds",
+			Help:    "Latency between integration tests completion and release creation",
+			Buckets: []float64{0.05, 0.1, 0.5, 1, 2, 3, 4, 5, 10, 15, 30},
+		},
+	)
 )
 
 func RegisterCompletedSnapshot(conditiontype, reason string, startTime metav1.Time, completionTime *metav1.Time) {
@@ -131,6 +141,11 @@ func RegisterNewIntegrationPipelineRun(snapshotCreatedTime metav1.Time, pipeline
 	RegisterPipelineRunStarted(snapshotCreatedTime, pipelineRunStartTime)
 }
 
+func RegisterReleaseLatency(startTime metav1.Time) {
+	latency := time.Since(startTime.Time).Seconds()
+	ReleaseLatencySeconds.Observe(latency)
+}
+
 func init() {
 	metrics.Registry.MustRegister(
 		SnapshotCreatedToPipelineRunStartedSeconds,
@@ -141,5 +156,6 @@ func init() {
 		SnapshotDurationSeconds,
 		SnapshotInvalidTotal,
 		SnapshotTotal,
+		ReleaseLatencySeconds,
 	)
 }

metrics/integration_test.go

@@ -26,9 +26,8 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/testutil"
 	"github.com/redhat-appstudio/operator-toolkit/test"
-	"sigs.k8s.io/controller-runtime/pkg/metrics"
-
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
 var _ = Describe("Metrics Integration", Ordered, func() {
@@ -217,4 +216,50 @@ var _ = Describe("Metrics Integration", Ordered, func() {
 				))).To(Succeed())
 		})
 	})
+
+	Context("When RegisterReleaseLatency is called", func() {
+
+		metrics.Registry.Unregister(ReleaseLatencySeconds)
+
+		BeforeAll(func() {
+			// Mocking metrics to reset data with each test.
+			ReleaseLatencySeconds = prometheus.NewHistogram(
+				prometheus.HistogramOpts{
+					Name:    "release_latency_seconds",
+					Help:    "Latency between integration tests completion and release creation",
+					Buckets: []float64{0.05, 0.1, 0.5, 1, 2, 3, 4, 5, 10, 15, 30},
+				},
+			)
+			// Register this metric with the registry
+			metrics.Registry.MustRegister(ReleaseLatencySeconds)
+		})
+
+		AfterAll(func() {
+			metrics.Registry.Unregister(ReleaseLatencySeconds)
+		})
+
+		var startTime, completionTime metav1.Time
+
+		BeforeEach(func() {
+			// Set completion time to current time and start time to 60 seconds prior.
+			completionTime = metav1.Time{Time: time.Now()}
+			startTime = metav1.Time{Time: completionTime.Time.Add(-60 * time.Second)}
+		})
+
+		It("adds an observation to ReleaseLatencySeconds", func() {
+			RegisterReleaseLatency(startTime)
+
+			// Calculate observed latency
+			observedLatency := completionTime.Sub(startTime.Time).Seconds()
+
+			// Ensure the observed latency is within ±5ms of 60 seconds
+			Expect(observedLatency).To(BeNumerically(">=", 59.995))
+			Expect(observedLatency).To(BeNumerically("<=", 60.005))
+		})
+
+		It("measures latency for only one release", func() {
+			RegisterReleaseLatency(startTime)
+			Expect(testutil.CollectAndCount(ReleaseLatencySeconds)).To(Equal(1))
+		})
+	})
 })