diff --git a/config/grafana/dashboards/integration-service-dashboard.json b/config/grafana/dashboards/integration-service-dashboard.json index f7e045975..f3788312b 100644 --- a/config/grafana/dashboards/integration-service-dashboard.json +++ b/config/grafana/dashboards/integration-service-dashboard.json @@ -1310,6 +1310,76 @@ "yBucketBound": "auto", "yBucketNumber": null, "yBucketSize": null + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "description": "Measure release creation latency from end of testing to release created", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 28, + "legend": { + "show": false + }, + "maxDataPoints": 25, + "pluginVersion": "7.5.17", + "reverseYBuckets": false, + "targets": [ + { + "exemplar": true, + "expr": "histogram_quantile(0.90, sum(rate(release_latency_seconds_bucket[5m])) by (le)) ", + "format": "heatmap", + "interval": "", + "legendFormat": "{{le}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Latency of Release Creation", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null } ], diff --git a/controllers/snapshot/snapshot_adapter.go b/controllers/snapshot/snapshot_adapter.go index fa9638566..795bca6ee 100644 --- a/controllers/snapshot/snapshot_adapter.go +++ b/controllers/snapshot/snapshot_adapter.go @@ -29,6 +29,7 @@ import ( "github.com/redhat-appstudio/integration-service/api/v1beta1" "github.com/redhat-appstudio/integration-service/gitops" h "github.com/redhat-appstudio/integration-service/helpers" + "github.com/redhat-appstudio/integration-service/metrics" "github.com/redhat-appstudio/integration-service/release" "github.com/redhat-appstudio/integration-service/tekton" @@ -474,6 +475,8 @@ func (a *Adapter) createMissingReleasesForReleasePlans(application *applicationa return err } + firstRelease := true + for _, releasePlan := range *releasePlans { releasePlan := releasePlan // G601 existingRelease := release.FindMatchingReleaseWithReleasePlan(releases, releasePlan) @@ -507,6 +510,14 @@ func (a *Adapter) createMissingReleasesForReleasePlans(application *applicationa } a.logger.Info("Marked Release status automated", "release.Name", newRelease.Name) } + // Register the first release time for metrics calculation + if firstRelease { + startTime, ok := gitops.GetAppStudioTestsFinishedTime(a.snapshot) + if ok { + metrics.RegisterReleaseLatency(startTime) + firstRelease = false + } + } } return nil } diff --git a/gitops/snapshot.go b/gitops/snapshot.go index c9202ff40..f998a1883 100644 --- a/gitops/snapshot.go +++ b/gitops/snapshot.go @@ -360,9 +360,31 @@ func HaveAppStudioTestsSucceeded(snapshot *applicationapiv1alpha1.Snapshot) bool if meta.FindStatusCondition(snapshot.Status.Conditions, AppStudioTestSucceededCondition) == nil { return meta.IsStatusConditionTrue(snapshot.Status.Conditions, LegacyTestSucceededCondition) } + return meta.IsStatusConditionTrue(snapshot.Status.Conditions, AppStudioTestSucceededCondition) } +// GetTestSucceededCondition checks status of tests on the snapshot +func GetTestSucceededCondition(snapshot *applicationapiv1alpha1.Snapshot) (condition *metav1.Condition, ok bool) { + + condition = meta.FindStatusCondition(snapshot.Status.Conditions, AppStudioTestSucceededCondition) + if condition == nil { + condition = meta.FindStatusCondition(snapshot.Status.Conditions, LegacyTestSucceededCondition) + } + + ok = (condition != nil && condition.Status != metav1.ConditionUnknown) + return +} + +// GetAppStudioTestsFinishedTime finds the timestamp of tests succeeded condition +func GetAppStudioTestsFinishedTime(snapshot *applicationapiv1alpha1.Snapshot) (metav1.Time, bool) { + condition, ok := GetTestSucceededCondition(snapshot) + if ok { + return condition.LastTransitionTime, true + } + return metav1.Time{}, false +} + // CanSnapshotBePromoted checks if the Snapshot in question can be promoted for deployment and release. func CanSnapshotBePromoted(snapshot *applicationapiv1alpha1.Snapshot) (bool, []string) { canBePromoted := true diff --git a/gitops/snapshot_test.go b/gitops/snapshot_test.go index 265d08270..384b4ec34 100644 --- a/gitops/snapshot_test.go +++ b/gitops/snapshot_test.go @@ -227,6 +227,62 @@ var _ = Describe("Gitops functions for managing Snapshots", Ordered, func() { Expect(checkResult).To(BeFalse()) }) + It("returns true if only AppStudioTestSucceededCondition is set", func() { + appStudioTestSucceededCondition := "AppStudioTestSucceeded" // Local variable + condition := metav1.Condition{ + Type: appStudioTestSucceededCondition, + Status: metav1.ConditionTrue, + } + meta.SetStatusCondition(&hasSnapshot.Status.Conditions, condition) + Expect(gitops.HaveAppStudioTestsSucceeded(hasSnapshot)).To(BeTrue()) + }) + + It("returns true if only LegacyTestSucceededCondition is set", func() { + legacyTestSucceededCondition := "HACBSStudioTestSucceeded" // Local variable + condition := metav1.Condition{ + Type: legacyTestSucceededCondition, + Status: metav1.ConditionTrue, + } + meta.SetStatusCondition(&hasSnapshot.Status.Conditions, condition) + Expect(gitops.HaveAppStudioTestsSucceeded(hasSnapshot)).To(BeTrue()) + }) + + It("returns the LastTransitionTime when AppStudioTestSucceededCondition is set", func() { + appStudioTestSucceededCondition := "AppStudioTestSucceeded" // Local variable + testTime := metav1.NewTime(time.Now()) + condition := metav1.Condition{ + Type: appStudioTestSucceededCondition, + Status: metav1.ConditionTrue, + LastTransitionTime: testTime, + } + meta.SetStatusCondition(&hasSnapshot.Status.Conditions, condition) + + returnedTime, ok := gitops.GetAppStudioTestsFinishedTime(hasSnapshot) + Expect(ok).To(BeTrue()) + Expect(returnedTime).To(Equal(testTime)) + }) + + It("returns the LastTransitionTime when LegacyTestSucceededCondition is set", func() { + legacyTestSucceededCondition := "HACBSStudioTestSucceeded" + testTime := metav1.NewTime(time.Now()) + condition := metav1.Condition{ + Type: legacyTestSucceededCondition, + Status: metav1.ConditionTrue, + LastTransitionTime: testTime, + } + meta.SetStatusCondition(&hasSnapshot.Status.Conditions, condition) + + returnedTime, ok := gitops.GetAppStudioTestsFinishedTime(hasSnapshot) + Expect(ok).To(BeTrue()) + Expect(returnedTime).To(Equal(testTime)) + }) + + It("returns zero time when neither condition is set", func() { + returnedTime, ok := gitops.GetAppStudioTestsFinishedTime(hasSnapshot) + Expect(ok).To(BeFalse()) + Expect(returnedTime).To(Equal(metav1.Time{})) // Empty or zero time + }) + It("ensures that a new Snapshots can be successfully created", func() { snapshotComponents := []applicationapiv1alpha1.SnapshotComponent{} createdSnapshot := gitops.NewSnapshot(hasApp, &snapshotComponents) diff --git a/metrics/integration.go b/metrics/integration.go index bb48dd778..a6194c6ba 100644 --- a/metrics/integration.go +++ b/metrics/integration.go @@ -17,6 +17,8 @@ limitations under the License. package metrics import ( + "time" + "github.com/prometheus/client_golang/prometheus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/metrics" @@ -87,6 +89,14 @@ var ( }, []string{"type", "reason"}, ) + + ReleaseLatencySeconds = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Name: "release_latency_seconds", + Help: "Latency between integration tests completion and release creation", + Buckets: []float64{0.05, 0.1, 0.5, 1, 2, 3, 4, 5, 10, 15, 30}, + }, + ) ) func RegisterCompletedSnapshot(conditiontype, reason string, startTime metav1.Time, completionTime *metav1.Time) { @@ -131,6 +141,11 @@ func RegisterNewIntegrationPipelineRun(snapshotCreatedTime metav1.Time, pipeline RegisterPipelineRunStarted(snapshotCreatedTime, pipelineRunStartTime) } +func RegisterReleaseLatency(startTime metav1.Time) { + latency := time.Since(startTime.Time).Seconds() + ReleaseLatencySeconds.Observe(latency) +} + func init() { metrics.Registry.MustRegister( SnapshotCreatedToPipelineRunStartedSeconds, @@ -141,5 +156,6 @@ func init() { SnapshotDurationSeconds, SnapshotInvalidTotal, SnapshotTotal, + ReleaseLatencySeconds, ) } diff --git a/metrics/integration_test.go b/metrics/integration_test.go index c5fa5f584..2b8b35ef4 100644 --- a/metrics/integration_test.go +++ b/metrics/integration_test.go @@ -26,9 +26,8 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/redhat-appstudio/operator-toolkit/test" - "sigs.k8s.io/controller-runtime/pkg/metrics" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/metrics" ) var _ = Describe("Metrics Integration", Ordered, func() { @@ -217,4 +216,50 @@ var _ = Describe("Metrics Integration", Ordered, func() { ))).To(Succeed()) }) }) + + Context("When RegisterReleaseLatency is called", func() { + + metrics.Registry.Unregister(ReleaseLatencySeconds) + + BeforeAll(func() { + // Mocking metrics to reset data with each test. + ReleaseLatencySeconds = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Name: "release_latency_seconds", + Help: "Latency between integration tests completion and release creation", + Buckets: []float64{0.05, 0.1, 0.5, 1, 2, 3, 4, 5, 10, 15, 30}, + }, + ) + // Register this metric with the registry + metrics.Registry.MustRegister(ReleaseLatencySeconds) + }) + + AfterAll(func() { + metrics.Registry.Unregister(ReleaseLatencySeconds) + }) + + var startTime, completionTime metav1.Time + + BeforeEach(func() { + // Set completion time to current time and start time to 60 seconds prior. + completionTime = metav1.Time{Time: time.Now()} + startTime = metav1.Time{Time: completionTime.Time.Add(-60 * time.Second)} + }) + + It("adds an observation to ReleaseLatencySeconds", func() { + RegisterReleaseLatency(startTime) + + // Calculate observed latency + observedLatency := completionTime.Sub(startTime.Time).Seconds() + + // Ensure the observed latency is within ±5ms of 60 seconds + Expect(observedLatency).To(BeNumerically(">=", 59.995)) + Expect(observedLatency).To(BeNumerically("<=", 60.005)) + }) + + It("measures latency for only one release", func() { + RegisterReleaseLatency(startTime) + Expect(testutil.CollectAndCount(ReleaseLatencySeconds)).To(Equal(1)) + }) + }) })