From b9747055e15fabcfd2903f2e2da3944fcb305c88 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jo=C3=A3o=20Vila=C3=A7a?= <machadovilaca@gmail.com>
Date: Wed, 20 Nov 2024 14:59:13 +0000
Subject: [PATCH] Reset kubevirt_hco_system_health_status on update (#3170)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: João Vilaça <machadovilaca@gmail.com>
---
 .../hyperconverged_controller_test.go         | 10 +++----
 pkg/monitoring/metrics/metrics_suite_test.go  | 13 +++++++++
 pkg/monitoring/metrics/operator_metrics.go    |  6 +++-
 .../metrics/operator_metrics_test.go          | 28 +++++++++++++++++++
 4 files changed, 51 insertions(+), 6 deletions(-)
 create mode 100644 pkg/monitoring/metrics/metrics_suite_test.go
 create mode 100644 pkg/monitoring/metrics/operator_metrics_test.go

diff --git a/controllers/hyperconverged/hyperconverged_controller_test.go b/controllers/hyperconverged/hyperconverged_controller_test.go
index de36cee34..1f6516553 100644
--- a/controllers/hyperconverged/hyperconverged_controller_test.go
+++ b/controllers/hyperconverged/hyperconverged_controller_test.go
@@ -192,7 +192,7 @@ var _ = Describe("HyperconvergedController", func() {
 					Message: "Initializing HyperConverged cluster",
 				})))
 
-				verifySystemHealthStatusError(foundResource)
+				verifySystemHealthStatusError(foundResource, reconcileInit)
 
 				expectedFeatureGates := []string{
 					"CPUManager",
@@ -314,7 +314,7 @@ var _ = Describe("HyperconvergedController", func() {
 					Message: "Initializing HyperConverged cluster",
 				})))
 
-				verifySystemHealthStatusError(foundResource)
+				verifySystemHealthStatusError(foundResource, reconcileInit)
 
 				res, err = r.Reconcile(context.TODO(), request)
 				Expect(err).ToNot(HaveOccurred())
@@ -401,7 +401,7 @@ var _ = Describe("HyperconvergedController", func() {
 					Message: reconcileCompletedMessage,
 				})))
 
-				verifySystemHealthStatusError(foundResource)
+				verifySystemHealthStatusError(foundResource, "SSPConditions")
 
 				Expect(foundResource.Status.RelatedObjects).To(HaveLen(21))
 				expectedRef := corev1.ObjectReference{
@@ -3893,10 +3893,10 @@ func verifySystemHealthStatusHealthy(hco *hcov1beta1.HyperConverged) {
 	ExpectWithOffset(1, systemHealthStatusMetric).To(Equal(metrics.SystemHealthStatusHealthy))
 }
 
-func verifySystemHealthStatusError(hco *hcov1beta1.HyperConverged) {
+func verifySystemHealthStatusError(hco *hcov1beta1.HyperConverged, reason string) {
 	ExpectWithOffset(1, hco.Status.SystemHealthStatus).To(Equal(systemHealthStatusError))
 
-	systemHealthStatusMetric, err := metrics.GetHCOMetricSystemHealthStatus(reconcileInit)
+	systemHealthStatusMetric, err := metrics.GetHCOMetricSystemHealthStatus(reason)
 	ExpectWithOffset(1, err).ToNot(HaveOccurred())
 	ExpectWithOffset(1, systemHealthStatusMetric).To(Equal(metrics.SystemHealthStatusError))
 }
diff --git a/pkg/monitoring/metrics/metrics_suite_test.go b/pkg/monitoring/metrics/metrics_suite_test.go
new file mode 100644
index 000000000..32d190ba2
--- /dev/null
+++ b/pkg/monitoring/metrics/metrics_suite_test.go
@@ -0,0 +1,13 @@
+package metrics_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestMetrics(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Metrics Suite")
+}
diff --git a/pkg/monitoring/metrics/operator_metrics.go b/pkg/monitoring/metrics/operator_metrics.go
index cf6463bf7..ad3e68821 100644
--- a/pkg/monitoring/metrics/operator_metrics.go
+++ b/pkg/monitoring/metrics/operator_metrics.go
@@ -16,6 +16,7 @@ const (
 )
 
 const (
+	SystemHealthStatusUnknown float64 = 0
 	SystemHealthStatusHealthy float64 = iota
 	SystemHealthStatusWarning
 	SystemHealthStatusError
@@ -119,14 +120,17 @@ func IsHCOMetricHyperConvergedExists() (bool, error) {
 }
 
 func SetHCOSystemHealthy() {
+	systemHealthStatus.Reset()
 	systemHealthStatus.WithLabelValues("healthy").Set(SystemHealthStatusHealthy)
 }
 
 func SetHCOSystemWarning(reason string) {
+	systemHealthStatus.Reset()
 	systemHealthStatus.WithLabelValues(reason).Set(SystemHealthStatusWarning)
 }
 
 func SetHCOSystemError(reason string) {
+	systemHealthStatus.Reset()
 	systemHealthStatus.WithLabelValues(reason).Set(SystemHealthStatusError)
 }
 
@@ -137,7 +141,7 @@ func GetHCOMetricSystemHealthStatus(reason string) (float64, error) {
 	value := dto.Gauge.GetValue()
 
 	if err != nil {
-		return 0, err
+		return SystemHealthStatusUnknown, err
 	}
 	return value, nil
 }
diff --git a/pkg/monitoring/metrics/operator_metrics_test.go b/pkg/monitoring/metrics/operator_metrics_test.go
new file mode 100644
index 000000000..3b3df11fd
--- /dev/null
+++ b/pkg/monitoring/metrics/operator_metrics_test.go
@@ -0,0 +1,28 @@
+package metrics_test
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/kubevirt/hyperconverged-cluster-operator/pkg/monitoring/metrics"
+)
+
+var _ = Describe("Operator Metrics", func() {
+	Context("kubevirt_hco_system_health_status", func() {
+		It("should set system error reason correctly", func() {
+			metrics.SetHCOSystemError("Reason1")
+			v, err := metrics.GetHCOMetricSystemHealthStatus("Reason1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(v).To(Equal(metrics.SystemHealthStatusError))
+
+			metrics.SetHCOSystemError("Reason2")
+			v, err = metrics.GetHCOMetricSystemHealthStatus("Reason2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(v).To(Equal(metrics.SystemHealthStatusError))
+
+			v, err = metrics.GetHCOMetricSystemHealthStatus("Reason1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(v).To(Equal(metrics.SystemHealthStatusUnknown))
+		})
+	})
+})