fix: deployment health fixes

pkg/health/health.go

@@ -106,6 +106,7 @@ const (
 	HealthStatusRestart          HealthStatusCode = "Restarting"
 	HealthStatusStarting         HealthStatusCode = "Starting"
 	HealthStatusFailed           HealthStatusCode = "Failed"
+	HealthStatusFailedCreate     HealthStatusCode = "Failed Create"
 	HealthStatusUnschedulable    HealthStatusCode = "Unschedulable"
 	HealthStatusUpgradeFailed    HealthStatusCode = "UpgradeFailed"
 	HealthStatusOOMKilled        HealthStatusCode = "OOMKilled"

pkg/health/health_deployment.go

@@ -53,13 +53,28 @@ func getReplicaHealth(s ReplicaStatus) *HealthStatus {
 	age := time.Since(s.Object.GetCreationTimestamp().Time).Truncate(time.Minute).Abs()
 
 	gs := GetGenericStatus(s.Object)
+	available := gs.FindCondition("Available")
+	isAvailable := s.Ready > 0
+	if available.Status != "" {
+		isAvailable = available.Status == "True"
+	}
 
 	progressing := gs.FindCondition("Progressing")
+
+	failure := gs.FindCondition("ReplicaFailure")
+	if failure.Status == "True" {
+		hs.Status = HealthStatusFailedCreate
+		hs.Health = HealthUnhealthy
+		hs.Message = failure.Message
+		hs.Ready = true
+		return hs
+	}
+
 	isStarting := age < startDeadline
 	isProgressDeadlineExceeded := !isStarting && (progressing.Reason == "ProgressDeadlineExceeded")
-	hs.Ready = progressing.Status == "True"
+	hs.Ready = progressing.Status == "True" && progressing.Reason != "ReplicaSetUpdated"
 
-	hs.Health = lo.Ternary(s.Ready >= s.Desired, HealthHealthy, lo.Ternary(s.Ready > 0, HealthWarning, HealthUnhealthy))
+	hs.Health = lo.Ternary(isAvailable, HealthHealthy, lo.Ternary(s.Ready > 0, HealthWarning, HealthUnhealthy))
 
 	if s.Desired == 0 && s.Replicas == 0 {
 		hs.Ready = true
@@ -75,26 +90,32 @@ func getReplicaHealth(s ReplicaStatus) *HealthStatus {
 			hs.Status = "Pending"
 			hs.Health = HealthUnknown
 		}
-	} else if s.Ready == 0 && isStarting && !isProgressDeadlineExceeded {
-		hs.Health = HealthUnknown
+	} else if s.Ready == 0 && isStarting {
 		hs.Status = HealthStatusStarting
-	} else if s.Ready == 0 && !isStarting {
-		hs.Health = HealthUnhealthy
-		hs.Status = HealthStatusCrashLoopBackoff
+	} else if s.Ready == 0 {
+		hs.Status = lo.Ternary(isAvailable, HealthStatusUpdating, HealthStatusCrashLoopBackoff)
+	}
+
+	if isProgressDeadlineExceeded {
+		hs.Status = HealthStatusRolloutFailed
+		hs.Health = hs.Health.Worst(HealthWarning)
 	} else if s.Desired == 0 && s.Replicas > 0 {
 		hs.Status = HealthStatusScalingDown
-		hs.Health = lo.Ternary(isProgressDeadlineExceeded, HealthWarning, HealthHealthy)
 	} else if s.Ready == s.Desired && s.Desired == s.Updated && s.Replicas == s.Desired {
 		hs.Status = HealthStatusRunning
-	} else if s.Desired != s.Updated {
-		hs.Status = HealthStatusUpdating
+	} else if !isStarting && s.Desired != s.Updated {
+		hs.Status = HealthStatusRollingOut
 	} else if s.Replicas > s.Desired {
 		hs.Status = HealthStatusScalingDown
 	} else if s.Replicas < s.Desired {
 		hs.Status = HealthStatusScalingUp
 	}
 
-	if isStarting && hs.Health == HealthUnhealthy {
+	if s.Replicas != s.Desired || s.Replicas != s.Updated {
+		hs.Ready = false
+	}
+
+	if isStarting && (hs.Health == HealthUnhealthy || hs.Health == HealthWarning) {
 		hs.Health = HealthUnknown
 	}
 

pkg/health/health_test.go

@@ -143,21 +143,6 @@ func assertAppHealthWithOverwriteMsg(
 	assert.Equal(t, expectedMsg, health.Message)
 }
 
-func assertAppHealthWithOverwrite(
-	t *testing.T,
-	yamlPath string,
-	overwrites map[string]string,
-	expectedStatus health.HealthStatusCode,
-	expectedHealth health.Health,
-	expectedReady bool,
-) {
-	health, _ := getHealthStatus(yamlPath, t, overwrites)
-	assert.NotNil(t, health)
-	assert.Equal(t, expectedHealth, health.Health)
-	assert.Equal(t, expectedReady, health.Ready)
-	assert.Equal(t, expectedStatus, health.Status)
-}
-
 func getHealthStatus(
 	yamlPath string,
 	t *testing.T,
@@ -315,88 +300,10 @@ func TestExternalSecrets(t *testing.T) {
 	assertAppHealthMsg(t, b+"healthy.yaml", "SecretSynced", health.HealthHealthy, true)
 }
 
-func TestDeploymentHealth(t *testing.T) {
-	assertAppHealthMsg(t, "./testdata/nginx.yaml", health.HealthStatusRunning, health.HealthHealthy, true, "1/1 ready")
-	assertAppHealthMsg(
-		t,
-		"./deployment-scaled-up.yaml",
-		health.HealthStatusRunning,
-		health.HealthHealthy,
-		true,
-		"3/3 ready",
-	)
-
-	assertAppHealthMsg(
-		t,
-		"./deployment-rollout-failed.yaml",
-		health.HealthStatusUpdating,
-		health.HealthWarning,
-		true,
-		"1/2 ready, 1 updating",
-	)
-	assertAppHealthMsg(
-		t,
-		"./testdata/deployment-progressing.yaml",
-		health.HealthStatusUpdating,
-		health.HealthWarning,
-		true,
-		"1/2 ready, 1 updating",
-	)
-	assertAppHealthMsg(
-		t,
-		"./testdata/deployment-suspended.yaml",
-		health.HealthStatusSuspended,
-		health.HealthHealthy,
-		false,
-		"1/1 ready, 1 updating, 1 terminating",
-	)
-	assertAppHealthMsg(
-		t,
-		"./testdata/deployment-degraded.yaml",
-		health.HealthStatusUpdating,
-		health.HealthWarning,
-		true,
-		"1/2 ready, 1 updating",
-	)
-
-	assertAppHealthMsg(
-		t,
-		"./testdata/deployment-starting.yaml",
-		health.HealthStatusStarting,
-		health.HealthUnknown,
-		true,
-		"0/2 ready, 1 updating",
-	)
-	assertAppHealthMsg(
-		t,
-		"./testdata/deployment-scaling-down.yaml",
-		health.HealthStatusScalingDown,
-		health.HealthHealthy,
-		true,
-		"1/1 ready, 1 updating, 1 terminating",
-	)
-	assertAppHealthMsg(
-		t,
-		"./testdata/deployment-failed.yaml",
-		"Failed Create",
-		health.HealthUnhealthy,
-		false,
-		"0/1 ready",
-	)
-}
-
 func TestStatefulSetHealth(t *testing.T) {
+	starting := "./testdata/Kubernetes/StatefulSet/statefulset-starting.yaml"
 	assertAppHealthMsg(
-		t,
-		"./testdata/statefulset.yaml",
-		health.HealthStatusRunning,
-		health.HealthHealthy,
-		true,
-		"1/1 ready",
-	)
-	assertAppHealthMsg(
-		t,
-		"./testdata/statefulset-starting.yaml",
+		t, starting,
 		health.HealthStatusStarting,
 		health.HealthUnknown,
 		true,
@@ -406,7 +313,8 @@ func TestStatefulSetHealth(t *testing.T) {
 	)
 	assertAppHealthMsg(
 		t,
-		"./testdata/statefulset-starting.yaml",
+		starting,
+
 		health.HealthStatusStarting,
 		health.HealthUnknown,
 		true,
@@ -416,7 +324,7 @@ func TestStatefulSetHealth(t *testing.T) {
 	)
 	assertAppHealthMsg(
 		t,
-		"./testdata/statefulset-starting.yaml",
+		starting,
 		health.HealthStatusCrashLoopBackoff,
 		health.HealthUnhealthy,
 		true,
@@ -426,7 +334,7 @@ func TestStatefulSetHealth(t *testing.T) {
 	)
 	assertAppHealthMsg(
 		t,
-		"./testdata/statefulset-starting.yaml",
+		starting,
 		health.HealthStatusCrashLoopBackoff,
 		health.HealthUnhealthy,
 		true,
@@ -439,7 +347,7 @@ func TestStatefulSetHealth(t *testing.T) {
 func TestStatefulSetOnDeleteHealth(t *testing.T) {
 	assertAppHealthMsg(
 		t,
-		"./testdata/statefulset-ondelete.yaml",
+		"./testdata/Kubernetes/StatefulSet/statefulset-ondelete.yaml",
 		"TerminatingStalled",
 		health.HealthWarning,
 		false,
@@ -536,16 +444,6 @@ func TestHPA(t *testing.T) {
 	)
 }
 
-func TestReplicaSet(t *testing.T) {
-	assertAppHealthWithOverwrite(t, "./testdata/replicaset-ittools.yml", map[string]string{
-		"2024-08-03T06:06:18Z": time.Now().Add(-time.Minute * 2).UTC().Format("2006-01-02T15:04:05Z"),
-	}, health.HealthStatusRunning, health.HealthHealthy, false)
-
-	assertAppHealthWithOverwrite(t, "./testdata/replicaset-unhealthy-pods.yaml", map[string]string{
-		"2024-10-21T11:20:19Z": time.Now().Add(-time.Minute * 2).UTC().Format("2006-01-02T15:04:05Z"),
-	}, health.HealthStatusStarting, health.HealthUnknown, false)
-}
-
 // func TestAPIService(t *testing.T) {
 // 	assertAppHealthMsg(t, "./testdata/apiservice-v1-true.yaml", HealthStatusHealthy, health.HealthHealthy, true)
 // 	assertAppHealthMsg(t, "./testdata/apiservice-v1-false.yaml", HealthStatusProgressing, health.HealthHealthy, true)

pkg/health/testdata/deployment-degraded.yaml → pkg/health/testdata/Kubernetes/Deployment/deployment-degraded.yaml

@@ -2,9 +2,10 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    deployment.kubernetes.io/revision: "4"
-    kubectl.kubernetes.io/last-applied-configuration: |
-      {"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app.kubernetes.io/instance":"guestbook-default"},"name":"guestbook-ui","namespace":"default"},"spec":{"replicas":1,"selector":{"matchLabels":{"app":"guestbook-ui"}},"template":{"metadata":{"labels":{"app":"guestbook-ui","app.kubernetes.io/instance":"guestbook-default"}},"spec":{"containers":[{"image":"gcr.io/heptio-images/ks-guestbook-demo:0.3","name":"guestbook-ui","ports":[{"containerPort":80}]}]}}}}
+    expected-ready: "false"
+    expected-status: "Rolling Out"
+    expected-health: "warning"
+    expected-message: "1/2 ready, 1 updating"
   creationTimestamp: 2018-07-18T04:40:44Z
   generation: 4
   labels:
@@ -55,7 +56,7 @@ status:
       lastUpdateTime: 2018-07-18T04:48:48Z
       message: Deployment has minimum availability.
       reason: MinimumReplicasAvailable
-      status: "True"
+      status: "false"
       type: Available
     - lastTransitionTime: 2018-07-18T06:29:23Z
       lastUpdateTime: 2018-07-18T06:29:23Z

pkg/health/testdata/deployment-failed.yaml → pkg/health/testdata/Kubernetes/Deployment/deployment-failed.yaml

@@ -7,6 +7,11 @@ metadata:
     control-plane: karina-operator
   namespace: platform-system
   annotations:
+    expected-status: Failed Create
+    expected-ready: "true"
+    expected-health: "unhealthy"
+    expected-message: 'pods "karina-c7585bd87-" is forbidden: error looking up service
+        account platform-system/karina: serviceaccount "karina" not found'
     deployment.kubernetes.io/revision: "1"
   creationTimestamp: 2023-05-10T08:11:03Z
 spec:

pkg/health/testdata/deployment-progressing.yaml → pkg/health/testdata/Kubernetes/Deployment/deployment-progressing.yaml

@@ -2,9 +2,11 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    deployment.kubernetes.io/revision: "4"
-    kubectl.kubernetes.io/last-applied-configuration: |
-      {"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app.kubernetes.io/instance":"guestbook-default"},"name":"guestbook-ui","namespace":"default"},"spec":{"replicas":1,"selector":{"matchLabels":{"app":"guestbook-ui"}},"template":{"metadata":{"labels":{"app":"guestbook-ui","app.kubernetes.io/instance":"guestbook-default"}},"spec":{"containers":[{"image":"gcr.io/heptio-images/ks-guestbook-demo:0.3","name":"guestbook-ui","ports":[{"containerPort":80}]}]}}}}
+    expected-status: Rolling Out
+    expected-health: healthy
+    expected-message: "1/2 ready, 1 updating"
+    expected-replicas: "2"
+    expted-ready: "false"
   creationTimestamp: 2018-07-18T04:40:44Z
   generation: 4
   labels:

pkg/health/testdata/deployment-rollout-failed-unhealthy.yaml → pkg/health/testdata/Kubernetes/Deployment/deployment-rollout-failed-unhealthy.yaml

@@ -2,6 +2,9 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: guestbook-ui
+  annotations:
+    expected-health: warning
+    expected-status: Rollout Failed
 spec:
   progressDeadlineSeconds: 600
   replicas: 2

pkg/health/testdata/deployment-rollout-failed.yaml → pkg/health/testdata/Kubernetes/Deployment/deployment-rollout-failed.yaml

@@ -2,6 +2,11 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: guestbook-ui
+  annotations:
+    expected-ready: "false"
+    expected-status: "Rolling Out"
+    expected-health: "healthy"
+    expected-message: "1/2 ready, 1 updating"
 spec:
   progressDeadlineSeconds: 600
   replicas: 2

pkg/health/testdata/deployment-scaled-up.yaml → pkg/health/testdata/Kubernetes/Deployment/deployment-scaled-up.yaml

@@ -10,9 +10,10 @@ metadata:
     app.kubernetes.io/managed-by: Helm
   namespace: podinfo
   annotations:
-    meta.helm.sh/release-name: podinfo
-    meta.helm.sh/release-namespace: podinfo
-    deployment.kubernetes.io/revision: "1"
+    expected-ready: "true"
+    expected-status: "Running"
+    expected-health: "healthy"
+    expected-message: "3/3 ready"
   creationTimestamp: 2023-12-19T15:50:39Z
 spec:
   replicas: 3

pkg/health/testdata/deployment-scaling-down.yaml → pkg/health/testdata/Kubernetes/Deployment/deployment-scaling-down.yaml

@@ -2,9 +2,10 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    deployment.kubernetes.io/revision: "4"
-    kubectl.kubernetes.io/last-applied-configuration: |
-      {"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app.kubernetes.io/instance":"guestbook-default"},"name":"guestbook-ui","namespace":"default"},"spec":{"replicas":1,"selector":{"matchLabels":{"app":"guestbook-ui"}},"template":{"metadata":{"labels":{"app":"guestbook-ui","app.kubernetes.io/instance":"guestbook-default"}},"spec":{"containers":[{"image":"gcr.io/heptio-images/ks-guestbook-demo:0.3","name":"guestbook-ui","ports":[{"containerPort":80}]}]}}}}
+    expected-ready: "false"
+    expected-status: "Scaling Down"
+    expected-health: "healthy"
+    expected-message: "1/1 ready, 1 updating, 1 terminating"
   creationTimestamp: 2018-07-18T04:40:44Z
   generation: 4
   labels:

pkg/health/testdata/deployment-scaling-up.yaml → pkg/health/testdata/Kubernetes/Deployment/deployment-scaling-up.yaml

@@ -1,9 +1,3 @@
-Config
-Changes1
-Insights0
-Relationships4
-Playbooks1
-Checks0
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -16,9 +10,11 @@ metadata:
     app.kubernetes.io/managed-by: Helm
   namespace: podinfo
   annotations:
-    meta.helm.sh/release-name: podinfo
-    meta.helm.sh/release-namespace: podinfo
-    deployment.kubernetes.io/revision: "1"
+  annotations:
+    expected-ready: "false"
+    expected-status: "Rolling Out"
+    expected-health: "warning"
+    expected-message: "1/3 ready, 1 updating"
   creationTimestamp: 2023-12-19T15:50:39Z
 spec:
   replicas: 3
@@ -120,6 +116,6 @@ status:
       status: "True"
       message: ReplicaSet "podinfo-97c6d4b94" has successfully progressed.
   readyReplicas: 1
-  updatedReplicas: 3
+  updatedReplicas: 2
   availableReplicas: 1
   unavailableReplicas: 2

pkg/health/testdata/deployment-starting.yaml → pkg/health/testdata/Kubernetes/Deployment/deployment-starting.yaml

@@ -5,6 +5,9 @@ metadata:
   labels:
     app.kubernetes.io/instance: guestbook-default
   name: guestbook-ui
+  annotations:
+    expected-status: Starting
+    expected-health: unknown
   namespace: default
 spec:
   progressDeadlineSeconds: 600