Reset kubevirt_hco_system_health_status on update (#3170)

Signed-off-by: João Vilaça <[email protected]>

controllers/hyperconverged/hyperconverged_controller_test.go

@@ -192,7 +192,7 @@ var _ = Describe("HyperconvergedController", func() {
 					Message: "Initializing HyperConverged cluster",
 				})))
 
-				verifySystemHealthStatusError(foundResource)
+				verifySystemHealthStatusError(foundResource, reconcileInit)
 
 				expectedFeatureGates := []string{
 					"CPUManager",
@@ -314,7 +314,7 @@ var _ = Describe("HyperconvergedController", func() {
 					Message: "Initializing HyperConverged cluster",
 				})))
 
-				verifySystemHealthStatusError(foundResource)
+				verifySystemHealthStatusError(foundResource, reconcileInit)
 
 				res, err = r.Reconcile(context.TODO(), request)
 				Expect(err).ToNot(HaveOccurred())
@@ -401,7 +401,7 @@ var _ = Describe("HyperconvergedController", func() {
 					Message: reconcileCompletedMessage,
 				})))
 
-				verifySystemHealthStatusError(foundResource)
+				verifySystemHealthStatusError(foundResource, "SSPConditions")
 
 				Expect(foundResource.Status.RelatedObjects).To(HaveLen(21))
 				expectedRef := corev1.ObjectReference{
@@ -3893,10 +3893,10 @@ func verifySystemHealthStatusHealthy(hco *hcov1beta1.HyperConverged) {
 	ExpectWithOffset(1, systemHealthStatusMetric).To(Equal(metrics.SystemHealthStatusHealthy))
 }
 
-func verifySystemHealthStatusError(hco *hcov1beta1.HyperConverged) {
+func verifySystemHealthStatusError(hco *hcov1beta1.HyperConverged, reason string) {
 	ExpectWithOffset(1, hco.Status.SystemHealthStatus).To(Equal(systemHealthStatusError))
 
-	systemHealthStatusMetric, err := metrics.GetHCOMetricSystemHealthStatus(reconcileInit)
+	systemHealthStatusMetric, err := metrics.GetHCOMetricSystemHealthStatus(reason)
 	ExpectWithOffset(1, err).ToNot(HaveOccurred())
 	ExpectWithOffset(1, systemHealthStatusMetric).To(Equal(metrics.SystemHealthStatusError))
 }

pkg/monitoring/metrics/metrics_suite_test.go

@@ -0,0 +1,13 @@
+package metrics_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestMetrics(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Metrics Suite")
+}

pkg/monitoring/metrics/operator_metrics.go

@@ -16,6 +16,7 @@ const (
 )
 
 const (
+	SystemHealthStatusUnknown float64 = 0
 	SystemHealthStatusHealthy float64 = iota
 	SystemHealthStatusWarning
 	SystemHealthStatusError
@@ -119,14 +120,17 @@ func IsHCOMetricHyperConvergedExists() (bool, error) {
 }
 
 func SetHCOSystemHealthy() {
+	systemHealthStatus.Reset()
 	systemHealthStatus.WithLabelValues("healthy").Set(SystemHealthStatusHealthy)
 }
 
 func SetHCOSystemWarning(reason string) {
+	systemHealthStatus.Reset()
 	systemHealthStatus.WithLabelValues(reason).Set(SystemHealthStatusWarning)
 }
 
 func SetHCOSystemError(reason string) {
+	systemHealthStatus.Reset()
 	systemHealthStatus.WithLabelValues(reason).Set(SystemHealthStatusError)
 }
 
@@ -137,7 +141,7 @@ func GetHCOMetricSystemHealthStatus(reason string) (float64, error) {
 	value := dto.Gauge.GetValue()
 
 	if err != nil {
-		return 0, err
+		return SystemHealthStatusUnknown, err
 	}
 	return value, nil
 }

pkg/monitoring/metrics/operator_metrics_test.go

@@ -0,0 +1,28 @@
+package metrics_test
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/kubevirt/hyperconverged-cluster-operator/pkg/monitoring/metrics"
+)
+
+var _ = Describe("Operator Metrics", func() {
+	Context("kubevirt_hco_system_health_status", func() {
+		It("should set system error reason correctly", func() {
+			metrics.SetHCOSystemError("Reason1")
+			v, err := metrics.GetHCOMetricSystemHealthStatus("Reason1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(v).To(Equal(metrics.SystemHealthStatusError))
+
+			metrics.SetHCOSystemError("Reason2")
+			v, err = metrics.GetHCOMetricSystemHealthStatus("Reason2")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(v).To(Equal(metrics.SystemHealthStatusError))
+
+			v, err = metrics.GetHCOMetricSystemHealthStatus("Reason1")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(v).To(Equal(metrics.SystemHealthStatusUnknown))
+		})
+	})
+})