Separate kubectl commands into separate jobs

azimuth-cloud · Aug 9, 2024 · 6b17092 · 6b17092
1 parent f95082b
commit 6b17092
Showing 1 changed file with 90 additions and 9 deletions.
diff --git a/danswer-azimuth/templates/hooks.yml b/danswer-azimuth/templates/hooks.yml
@@ -67,22 +67,24 @@ spec:
       restartPolicy: Never
       # TODO: Create a separate service account
       serviceAccountName: default
----
+
 # Until https://github.com/unoplat/vespa-helm-charts/pull/23
 # is merged, we need to patch vespa stateful set after deployment
 # so that service label selectors match correctly.
-# Also, since Danswer API pod gives up on Vespa application package
-# init request after just 5 retries we need to restart the API deployment
-# to trigger a retry on the Vespa setup by the API pod after labels
-# are corrected.
+# Since Danswer API pod gives up on Vespa application package
+# init request after just 5 retries we also need to restart the API
+# deployment to trigger a retry on the Vespa setup by the API pod
+# after labels are corrected.
+# Use three separate hooks with different hook-weights to control ordering.
+---
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: sts-label-updater
+  name: vespa-label-updater
   namespace: {{ .Release.Name }}
   annotations:
     helm.sh/hook: post-install
-    helm.sh/hook-weight: "0"
+    helm.sh/hook-weight: "1"
     helm.sh/hook-delete-policy: hook-succeeded
 spec:
   template:
@@ -98,14 +100,52 @@ spec:
         - statefulset/vespa
         - -p
         - {{ printf "{'spec':{'template':{'metadata':{'labels':{'app.kubernetes.io/instance':'%s'}}}}}" .Release.Name | replace "'" "\"" | squote }}
-        - "&&"
+      restartPolicy: Never
+      # TODO: Create a separate service account
+      serviceAccountName: default
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: vespa-waiter
+  namespace: {{ .Release.Name }}
+  annotations:
+    helm.sh/hook: post-install
+    helm.sh/hook-weight: "2"
+    helm.sh/hook-delete-policy: hook-succeeded
+spec:
+  template:
+    spec:
+      containers:
+      - name: waiter
+        image: gcr.io/google_containers/hyperkube:v1.18.0
+        command:
         - kubectl
         - wait
         - -n
         - {{ .Release.Name }}
         - --for=condition=Ready
         - pod/vespa-0
-        - "&&"
+      restartPolicy: Never
+      # TODO: Create a separate service account
+      serviceAccountName: default
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: danswer-api-restarter
+  namespace: {{ .Release.Name }}
+  annotations:
+    helm.sh/hook: post-install
+    helm.sh/hook-weight: "3"
+    helm.sh/hook-delete-policy: hook-succeeded
+spec:
+  template:
+    spec:
+      containers:
+      - name: restarter
+        image: gcr.io/google_containers/hyperkube:v1.18.0
+        command:
         - kubectl
         - rollout
         - restart
@@ -115,3 +155,44 @@ spec:
       restartPolicy: Never
       # TODO: Create a separate service account
       serviceAccountName: default
+---
+# apiVersion: batch/v1
+# kind: Job
+# metadata:
+#   name: sts-label-updater
+#   namespace: {{ .Release.Name }}
+#   annotations:
+#     helm.sh/hook: post-install
+#     helm.sh/hook-weight: "0"
+#     helm.sh/hook-delete-policy: hook-succeeded
+# spec:
+#   template:
+#     spec:
+#       containers:
+#       - name: label-updater
+#         image: gcr.io/google_containers/hyperkube:v1.18.0
+#         command:
+#         - kubectl
+#         - patch
+#         - -n
+#         - {{ .Release.Name }}
+#         - statefulset/vespa
+#         - -p
+#         - {{ printf "{'spec':{'template':{'metadata':{'labels':{'app.kubernetes.io/instance':'%s'}}}}}" .Release.Name | replace "'" "\"" | squote }}
+#         - "&&"
+#         - kubectl
+#         - wait
+#         - -n
+#         - {{ .Release.Name }}
+#         - --for=condition=Ready
+#         - pod/vespa-0
+#         - "&&"
+#         - kubectl
+#         - rollout
+#         - restart
+#         - -n
+#         - {{ .Release.Name }}
+#         - {{ printf "deployment/%s-danswer-stack-api-deployment" .Release.Name }}
+#       restartPolicy: Never
+#       # TODO: Create a separate service account
+#       serviceAccountName: default