chore: deployment/replicaset fixes

flanksource · Dec 5, 2024 · 66d0be9 · 66d0be9
1 parent fa7cb08
commit 66d0be9
Show file tree

Hide file tree

Showing 15 changed files with 121 additions and 123 deletions.
diff --git a/pkg/health/health_deployment.go b/pkg/health/health_deployment.go
@@ -53,7 +53,6 @@ func getReplicaHealth(s ReplicaStatus) *HealthStatus {
 	age := time.Since(s.Object.GetCreationTimestamp().Time).Truncate(time.Minute).Abs()
 
 	gs := GetGenericStatus(s.Object)
-
 	available := gs.FindCondition("Available")
 	isAvailable := s.Ready > 0
 	if available.Status != "" {
@@ -94,11 +93,7 @@ func getReplicaHealth(s ReplicaStatus) *HealthStatus {
 	} else if s.Ready == 0 && isStarting {
 		hs.Status = HealthStatusStarting
 	} else if s.Ready == 0 {
-		if isProgressDeadlineExceeded {
-			hs.Status = HealthStatusCrashLoopBackoff
-		} else if isAvailable {
-			hs.Status = HealthStatusUpdating
-		}
+		hs.Status = lo.Ternary(isAvailable, HealthStatusUpdating, HealthStatusCrashLoopBackoff)
 	}
 
 	if isProgressDeadlineExceeded {
@@ -116,6 +111,10 @@ func getReplicaHealth(s ReplicaStatus) *HealthStatus {
 		hs.Status = HealthStatusScalingUp
 	}
 
+	if s.Replicas != s.Desired || s.Replicas != s.Updated {
+		hs.Ready = false
+	}
+
 	if isStarting && (hs.Health == HealthUnhealthy || hs.Health == HealthWarning) {
 		hs.Health = HealthUnknown
 	}

diff --git a/pkg/health/health_test.go b/pkg/health/health_test.go
@@ -315,66 +315,10 @@ func TestExternalSecrets(t *testing.T) {
 	assertAppHealthMsg(t, b+"healthy.yaml", "SecretSynced", health.HealthHealthy, true)
 }
 
-func TestDeploymentHealth(t *testing.T) {
-	assertAppHealthMsg(t, "./testdata/nginx.yaml", health.HealthStatusRunning, health.HealthHealthy, true, "1/1 ready")
-	assertAppHealthMsg(
-		t,
-		"./deployment-scaled-up.yaml",
-		health.HealthStatusRunning,
-		health.HealthHealthy,
-		true,
-		"3/3 ready",
-	)
-
-	assertAppHealthMsg(
-		t,
-		"./deployment-rollout-failed.yaml",
-		health.HealthStatusRollingOut,
-		health.HealthWarning,
-		true,
-		"1/2 ready, 1 updating",
-	)
-
-	assertAppHealthMsg(
-		t,
-		"./testdata/deployment-suspended.yaml",
-		health.HealthStatusSuspended,
-		health.HealthHealthy,
-		false,
-		"1/1 ready, 1 updating, 1 terminating",
-	)
-	assertAppHealthMsg(
-		t,
-		"./testdata/deployment-degraded.yaml",
-		health.HealthStatusRollingOut,
-		health.HealthWarning,
-		true,
-		"1/2 ready, 1 updating",
-	)
-
-	assertAppHealthMsg(
-		t,
-		"./testdata/deployment-scaling-down.yaml",
-		health.HealthStatusScalingDown,
-		health.HealthHealthy,
-		true,
-		"1/1 ready, 1 updating, 1 terminating",
-	)
-
-}
-
 func TestStatefulSetHealth(t *testing.T) {
+	starting := "./testdata/Kubernetes/StatefulSet/statefulset-starting.yaml"
 	assertAppHealthMsg(
-		t,
-		"./testdata/statefulset.yaml",
-		health.HealthStatusRunning,
-		health.HealthHealthy,
-		true,
-		"1/1 ready",
-	)
-	assertAppHealthMsg(
-		t,
-		"./testdata/statefulset-starting.yaml",
+		t, starting,
 		health.HealthStatusStarting,
 		health.HealthUnknown,
 		true,
@@ -384,7 +328,8 @@ func TestStatefulSetHealth(t *testing.T) {
 	)
 	assertAppHealthMsg(
 		t,
-		"./testdata/statefulset-starting.yaml",
+		starting,
+
 		health.HealthStatusStarting,
 		health.HealthUnknown,
 		true,
@@ -394,7 +339,7 @@ func TestStatefulSetHealth(t *testing.T) {
 	)
 	assertAppHealthMsg(
 		t,
-		"./testdata/statefulset-starting.yaml",
+		starting,
 		health.HealthStatusCrashLoopBackoff,
 		health.HealthUnhealthy,
 		true,
@@ -404,7 +349,7 @@ func TestStatefulSetHealth(t *testing.T) {
 	)
 	assertAppHealthMsg(
 		t,
-		"./testdata/statefulset-starting.yaml",
+		starting,
 		health.HealthStatusCrashLoopBackoff,
 		health.HealthUnhealthy,
 		true,
@@ -417,7 +362,7 @@ func TestStatefulSetHealth(t *testing.T) {
 func TestStatefulSetOnDeleteHealth(t *testing.T) {
 	assertAppHealthMsg(
 		t,
-		"./testdata/statefulset-ondelete.yaml",
+		"./testdata/Kubernetes/StatefulSet/statefulset-ondelete.yaml",
 		"TerminatingStalled",
 		health.HealthWarning,
 		false,
@@ -514,16 +459,6 @@ func TestHPA(t *testing.T) {
 	)
 }
 
-func TestReplicaSet(t *testing.T) {
-	assertAppHealthWithOverwrite(t, "./testdata/replicaset-ittools.yml", map[string]string{
-		"2024-08-03T06:06:18Z": time.Now().Add(-time.Minute * 2).UTC().Format("2006-01-02T15:04:05Z"),
-	}, health.HealthStatusRunning, health.HealthHealthy, false)
-
-	assertAppHealthWithOverwrite(t, "./testdata/replicaset-unhealthy-pods.yaml", map[string]string{
-		"2024-10-21T11:20:19Z": time.Now().Add(-time.Minute * 2).UTC().Format("2006-01-02T15:04:05Z"),
-	}, health.HealthStatusStarting, health.HealthUnknown, false)
-}
-
 // func TestAPIService(t *testing.T) {
 // 	assertAppHealthMsg(t, "./testdata/apiservice-v1-true.yaml", HealthStatusHealthy, health.HealthHealthy, true)
 // 	assertAppHealthMsg(t, "./testdata/apiservice-v1-false.yaml", HealthStatusProgressing, health.HealthHealthy, true)

diff --git a/pkg/health/testdata/deployment-degraded.yaml → ...netes/Deployment/deployment-degraded.yaml b/pkg/health/testdata/deployment-degraded.yaml → ...netes/Deployment/deployment-degraded.yaml
@@ -2,6 +2,10 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
+    expected-ready: "false"
+    expected-status: "Rolling Out"
+    expected-health: "warning"
+    expected-message: "1/2 ready, 1 updating"
   creationTimestamp: 2018-07-18T04:40:44Z
   generation: 4
   labels:

diff --git a/...h/testdata/deployment-rollout-failed.yaml → ...Deployment/deployment-rollout-failed.yaml b/...h/testdata/deployment-rollout-failed.yaml → ...Deployment/deployment-rollout-failed.yaml
@@ -2,6 +2,11 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: guestbook-ui
+  annotations:
+    expected-ready: "false"
+    expected-status: "Rolling Out"
+    expected-health: "healthy"
+    expected-message: "1/2 ready, 1 updating"
 spec:
   progressDeadlineSeconds: 600
   replicas: 2

diff --git a/...health/testdata/deployment-scaled-up.yaml → ...etes/Deployment/deployment-scaled-up.yaml b/...health/testdata/deployment-scaled-up.yaml → ...etes/Deployment/deployment-scaled-up.yaml
@@ -10,9 +10,10 @@ metadata:
     app.kubernetes.io/managed-by: Helm
   namespace: podinfo
   annotations:
-    meta.helm.sh/release-name: podinfo
-    meta.helm.sh/release-namespace: podinfo
-    deployment.kubernetes.io/revision: "1"
+    expected-ready: "true"
+    expected-status: "Running"
+    expected-health: "healthy"
+    expected-message: "3/3 ready"
   creationTimestamp: 2023-12-19T15:50:39Z
 spec:
   replicas: 3

diff --git a/...lth/testdata/deployment-scaling-down.yaml → ...s/Deployment/deployment-scaling-down.yaml b/...lth/testdata/deployment-scaling-down.yaml → ...s/Deployment/deployment-scaling-down.yaml
@@ -2,9 +2,10 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    deployment.kubernetes.io/revision: "4"
-    kubectl.kubernetes.io/last-applied-configuration: |
-      {"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app.kubernetes.io/instance":"guestbook-default"},"name":"guestbook-ui","namespace":"default"},"spec":{"replicas":1,"selector":{"matchLabels":{"app":"guestbook-ui"}},"template":{"metadata":{"labels":{"app":"guestbook-ui","app.kubernetes.io/instance":"guestbook-default"}},"spec":{"containers":[{"image":"gcr.io/heptio-images/ks-guestbook-demo:0.3","name":"guestbook-ui","ports":[{"containerPort":80}]}]}}}}
+    expected-ready: "false"
+    expected-status: "Scaling Down"
+    expected-health: "healthy"
+    expected-message: "1/1 ready, 1 updating, 1 terminating"
   creationTimestamp: 2018-07-18T04:40:44Z
   generation: 4
   labels:

diff --git a/...ealth/testdata/deployment-scaling-up.yaml → ...tes/Deployment/deployment-scaling-up.yaml b/...ealth/testdata/deployment-scaling-up.yaml → ...tes/Deployment/deployment-scaling-up.yaml
@@ -1,9 +1,3 @@
-Config
-Changes1
-Insights0
-Relationships4
-Playbooks1
-Checks0
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -16,9 +10,11 @@ metadata:
     app.kubernetes.io/managed-by: Helm
   namespace: podinfo
   annotations:
-    meta.helm.sh/release-name: podinfo
-    meta.helm.sh/release-namespace: podinfo
-    deployment.kubernetes.io/revision: "1"
+  annotations:
+    expected-ready: "false"
+    expected-status: "Rolling Out"
+    expected-health: "warning"
+    expected-message: "1/3 ready, 1 updating"
   creationTimestamp: 2023-12-19T15:50:39Z
 spec:
   replicas: 3
@@ -120,6 +116,6 @@ status:
       status: "True"
       message: ReplicaSet "podinfo-97c6d4b94" has successfully progressed.
   readyReplicas: 1
-  updatedReplicas: 3
+  updatedReplicas: 2
   availableReplicas: 1
   unavailableReplicas: 2
diff --git a/...health/testdata/deployment-suspended.yaml → ...etes/Deployment/deployment-suspended.yaml b/...health/testdata/deployment-suspended.yaml → ...etes/Deployment/deployment-suspended.yaml
@@ -2,9 +2,10 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   annotations:
-    deployment.kubernetes.io/revision: "4"
-    kubectl.kubernetes.io/last-applied-configuration: |
-      {"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app.kubernetes.io/instance":"guestbook-default"},"name":"guestbook-ui","namespace":"default"},"spec":{"replicas":1,"selector":{"matchLabels":{"app":"guestbook-ui"}},"template":{"metadata":{"labels":{"app":"guestbook-ui","app.kubernetes.io/instance":"guestbook-default"}},"spec":{"containers":[{"image":"gcr.io/heptio-images/ks-guestbook-demo:0.3","name":"guestbook-ui","ports":[{"containerPort":80}]}]}}}}
+    expected-ready: "false"
+    expected-status: "Suspended"
+    expected-health: "healthy"
+    expected-message: "1/1 ready, 1 updating, 1 terminating"
   creationTimestamp: 2018-07-18T04:40:44Z
   generation: 4
   labels:

diff --git a/pkg/health/testdata/nginx.yaml → ...stdata/Kubernetes/Deployment/healthy.yaml b/pkg/health/testdata/nginx.yaml → ...stdata/Kubernetes/Deployment/healthy.yaml
diff --git a/pkg/health/testdata/replicaset-ittools.yml → ...estdata/Kubernetes/ReplicaSet/healthy.yml b/pkg/health/testdata/replicaset-ittools.yml → ...estdata/Kubernetes/ReplicaSet/healthy.yml
@@ -2,11 +2,10 @@ apiVersion: apps/v1
 kind: ReplicaSet
 metadata:
   annotations:
-    deployment.kubernetes.io/desired-replicas: "1"
-    deployment.kubernetes.io/max-replicas: "1"
-    deployment.kubernetes.io/revision: "2"
-    meta.helm.sh/release-name: ittools
-    meta.helm.sh/release-namespace: default
+    expected-ready: "true"
+    expected-status: "Running"
+    expected-health: healthy"
+    expected-message: "1/1 ready"
   creationTimestamp: "2024-08-03T06:06:18Z"
   generation: 52
   labels:
@@ -17,12 +16,12 @@ metadata:
   name: ittools-5fbf458f49
   namespace: default
   ownerReferences:
-  - apiVersion: apps/v1
-    blockOwnerDeletion: true
-    controller: true
-    kind: Deployment
-    name: ittools
-    uid: d2beccff-8da9-42e8-8459-e7ff938b2ffd
+    - apiVersion: apps/v1
+      blockOwnerDeletion: true
+      controller: true
+      kind: Deployment
+      name: ittools
+      uid: d2beccff-8da9-42e8-8459-e7ff938b2ffd
   resourceVersion: "96413911"
   uid: c044b250-2445-4813-b00f-22b696c5fcf2
 spec:
@@ -43,17 +42,17 @@ spec:
     spec:
       automountServiceAccountToken: true
       containers:
-      - image: corentinth/it-tools:latest
-        imagePullPolicy: Always
-        name: it-tools
-        resources:
-          limits:
-            memory: 50Mi
-          requests:
-            cpu: 25m
-            memory: 10Mi
-        terminationMessagePath: /dev/termination-log
-        terminationMessagePolicy: File
+        - image: corentinth/it-tools:latest
+          imagePullPolicy: Always
+          name: it-tools
+          resources:
+            limits:
+              memory: 50Mi
+            requests:
+              cpu: 25m
+              memory: 10Mi
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
       dnsPolicy: ClusterFirst
       enableServiceLinks: false
       restartPolicy: Always

diff --git a/...h/testdata/replicaset-unhealthy-pods.yaml → ...tdata/Kubernetes/ReplicaSet/starting.yaml b/...h/testdata/replicaset-unhealthy-pods.yaml → ...tdata/Kubernetes/ReplicaSet/starting.yaml
diff --git a/pkg/health/testdata/Kubernetes/ReplicaSet/unhealthy.yaml b/pkg/health/testdata/Kubernetes/ReplicaSet/unhealthy.yaml
@@ -0,0 +1,54 @@
+apiVersion: apps/v1
+kind: ReplicaSet
+metadata:
+  uid: f6579017-448f-425a-9645-ea3c93700948
+  name: failing-deployment-866585899d
+  labels:
+    app: failing-app
+    pod-template-hash: 866585899d
+  namespace: default
+  annotations:
+    expected-ready: "false"
+    expected-status: "CrashLoopBackOff"
+    expected-health: "unhealthy"
+    expected-message: "0/1 ready"
+  ownerReferences:
+    - uid: 1ab20b2b-e2c8-4e85-b7b6-5709ba594c0d
+      kind: Deployment
+      name: failing-deployment
+      apiVersion: apps/v1
+      controller: true
+      blockOwnerDeletion: true
+  creationTimestamp: "@now-1d"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: failing-app
+      pod-template-hash: 866585899d
+  template:
+    spec:
+      dnsPolicy: ClusterFirst
+      containers:
+        - args:
+            - -c
+            - sleep 5 && exit 1
+          name: failing-container
+          image: busybox
+          command:
+            - /bin/sh
+          resources: {}
+          imagePullPolicy: Always
+          terminationMessagePath: /dev/termination-log
+          terminationMessagePolicy: File
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext: {}
+      terminationGracePeriodSeconds: 30
+    metadata:
+      labels:
+        app: failing-app
+        pod-template-hash: 866585899d
+status:
+  replicas: 1
+  fullyLabeledReplicas: 1
diff --git a/...health/testdata/statefulset-ondelete.yaml → ...tes/StatefulSet/statefulset-ondelete.yaml b/...health/testdata/statefulset-ondelete.yaml → ...tes/StatefulSet/statefulset-ondelete.yaml
@@ -2,8 +2,9 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   annotations:
-    kubectl.kubernetes.io/last-applied-configuration: |
-      {"apiVersion":"apps/v1beta2","kind":"StatefulSet","metadata":{"annotations":{},"labels":{"app":"redis","app.kubernetes.io/instance":"redis","chart":"redis-3.6.5","heritage":"Tiller","release":"redis"},"name":"redis-master","namespace":"default"},"spec":{"selector":{"matchLabels":{"app":"redis","release":"redis","role":"master"}},"serviceName":"redis-master","template":{"metadata":{"labels":{"app":"redis","app.kubernetes.io/instance":"redis","release":"redis","role":"master"}},"spec":{"containers":[{"env":[{"name":"REDIS_REPLICATION_MODE","value":"master"},{"name":"REDIS_PASSWORD","valueFrom":{"secretKeyRef":{"key":"redis-password","name":"redis"}}},{"name":"REDIS_DISABLE_COMMANDS","value":"FLUSHDB,FLUSHALL"}],"image":"docker.io/bitnami/redis:4.0.10-debian-9","imagePullPolicy":"Always","livenessProbe":{"exec":{"command":["redis-cli","ping"]},"failureThreshold":5,"initialDelaySeconds":30,"periodSeconds":10,"successThreshold":1,"timeoutSeconds":5},"name":"redis","ports":[{"containerPort":6379,"name":"redis"}],"readinessProbe":{"exec":{"command":["redis-cli","ping"]},"failureThreshold":5,"initialDelaySeconds":5,"periodSeconds":10,"successThreshold":1,"timeoutSeconds":1},"resources":{},"volumeMounts":[{"mountPath":"/bitnami/redis/data","name":"redis-data"}]}],"securityContext":{"fsGroup":1001,"runAsUser":1001},"serviceAccountName":"default"}},"updateStrategy":{"type":"OnDelete"},"volumeClaimTemplates":[{"metadata":{"labels":{"app":"redis","chart":"redis-3.6.5","component":"master","heritage":"Tiller","release":"redis"},"name":"redis-data"},"spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"8Gi"}}}}]}}
+    expected-ready: "false"
+    expected-status: "Terminating"
+    expected-message: "1/1 ready, 1 updating"
   creationTimestamp: 2018-07-20T08:23:04Z
   generation: 1
   labels:

diff --git a/...health/testdata/statefulset-starting.yaml → ...tes/StatefulSet/statefulset-starting.yaml b/...health/testdata/statefulset-starting.yaml → ...tes/StatefulSet/statefulset-starting.yaml
diff --git a/pkg/health/testdata/statefulset.yaml → ...a/Kubernetes/StatefulSet/statefulset.yaml b/pkg/health/testdata/statefulset.yaml → ...a/Kubernetes/StatefulSet/statefulset.yaml
@@ -2,8 +2,10 @@ apiVersion: apps/v1
 kind: StatefulSet
 metadata:
   annotations:
-    kubectl.kubernetes.io/last-applied-configuration: |
-      {"apiVersion":"apps/v1beta2","kind":"StatefulSet","metadata":{"annotations":{},"labels":{"app":"redis","app.kubernetes.io/instance":"redis","chart":"redis-3.6.5","heritage":"Tiller","release":"redis"},"name":"redis-master","namespace":"default"},"spec":{"selector":{"matchLabels":{"app":"redis","release":"redis","role":"master"}},"serviceName":"redis-master","template":{"metadata":{"labels":{"app":"redis","app.kubernetes.io/instance":"redis","release":"redis","role":"master"}},"spec":{"containers":[{"env":[{"name":"REDIS_REPLICATION_MODE","value":"master"},{"name":"REDIS_PASSWORD","valueFrom":{"secretKeyRef":{"key":"redis-password","name":"redis"}}},{"name":"REDIS_DISABLE_COMMANDS","value":"FLUSHDB,FLUSHALL"}],"image":"docker.io/bitnami/redis:4.0.10-debian-9","imagePullPolicy":"Always","livenessProbe":{"exec":{"command":["redis-cli","ping"]},"failureThreshold":5,"initialDelaySeconds":30,"periodSeconds":10,"successThreshold":1,"timeoutSeconds":5},"name":"redis","ports":[{"containerPort":6379,"name":"redis"}],"readinessProbe":{"exec":{"command":["redis-cli","ping"]},"failureThreshold":5,"initialDelaySeconds":5,"periodSeconds":10,"successThreshold":1,"timeoutSeconds":1},"resources":{},"volumeMounts":[{"mountPath":"/bitnami/redis/data","name":"redis-data"}]}],"securityContext":{"fsGroup":1001,"runAsUser":1001},"serviceAccountName":"default"}},"updateStrategy":{"type":"OnDelete"},"volumeClaimTemplates":[{"metadata":{"labels":{"app":"redis","chart":"redis-3.6.5","component":"master","heritage":"Tiller","release":"redis"},"name":"redis-data"},"spec":{"accessModes":["ReadWriteOnce"],"resources":{"requests":{"storage":"8Gi"}}}}]}}
+    expected-ready: "true"
+    expected-status: "Running"
+    expected-health: "healthy"
+    expected-message: "1/1 ready"
   creationTimestamp: 2018-07-20T08:23:04Z
   generation: 1
   labels: