From 6f4e943430357e495b9a30f2689a3c31170599b9 Mon Sep 17 00:00:00 2001 From: Bryce Groff Date: Thu, 19 Dec 2024 06:59:24 -0800 Subject: [PATCH] refactor: refactor the chart to remove a lot of references to busybox, curl, and socat (#14674) Co-authored-by: Alex Buchanan --- airbyte-keycloak-setup/Dockerfile | 3 + airbyte-keycloak-setup/build.gradle.kts | 9 +++ airbyte-keycloak-setup/scripts/entrypoint.sh | 3 + airbyte-pod-sweeper/Dockerfile | 12 +++ airbyte-pod-sweeper/build.gradle.kts | 19 +++++ airbyte-pod-sweeper/scripts/sweep-pod.sh | 61 ++++++++++++++++ .../airbyte-bootloader/templates/_images.tpl | 2 +- charts/airbyte-bootloader/values.yaml | 2 +- .../templates/_images.tpl | 2 +- .../values.yaml | 2 +- .../templates/_images.tpl | 2 +- .../values.yaml | 16 +--- charts/airbyte-cron/templates/_images.tpl | 2 +- charts/airbyte-cron/values.yaml | 2 +- .../templates/_images.tpl | 2 +- charts/airbyte-featureflag-server/values.yaml | 2 +- .../templates/_images.tpl | 2 +- .../airbyte-keycloak-setup/templates/job.yaml | 11 --- charts/airbyte-keycloak-setup/values.yaml | 2 +- charts/airbyte-keycloak/templates/_images.tpl | 2 +- charts/airbyte-keycloak/values.yaml | 2 +- charts/airbyte-metrics/templates/_images.tpl | 2 +- charts/airbyte-metrics/values.yaml | 2 +- .../airbyte-pod-sweeper/templates/_images.tpl | 2 +- .../templates/configmap.yaml | 73 ------------------- .../templates/deployment.yaml | 11 +-- charts/airbyte-pod-sweeper/values.yaml | 5 +- charts/airbyte-server/templates/_images.tpl | 2 +- charts/airbyte-server/values.yaml | 2 +- .../airbyte-temporal-ui/templates/_images.tpl | 2 +- charts/airbyte-temporal-ui/values.yaml | 2 +- charts/airbyte-temporal/templates/_images.tpl | 2 +- charts/airbyte-temporal/values.yaml | 2 +- charts/airbyte-webapp/templates/_images.tpl | 2 +- charts/airbyte-webapp/values.yaml | 2 +- charts/airbyte-worker/README.md | 2 - charts/airbyte-worker/templates/_images.tpl | 2 +- .../airbyte-worker/templates/deployment.yaml | 10 --- charts/airbyte-worker/values.yaml | 12 +-- .../templates/_images.tpl | 2 +- .../airbyte-workload-api-server/values.yaml | 2 +- charts/airbyte-workload-launcher/README.md | 2 - .../templates/_images.tpl | 2 +- .../templates/deployment.yaml | 10 --- charts/airbyte-workload-launcher/values.yaml | 12 +-- charts/airbyte/README.md | 2 - charts/airbyte/templates/_images.tpl | 2 +- charts/airbyte/templates/env-configmap.yaml | 2 - charts/airbyte/templates/minio.yaml | 12 +-- .../airbyte/templates/tests/test-webapp.yaml | 6 +- charts/airbyte/values.yaml | 28 +------ charts/airbyte/values.yaml.test | 10 --- .../helm-tests/tests/basic_template_test.go | 2 - .../tests/enterprise_config_test.go | 11 +-- charts/helm-tests/tests/image_test.go | 35 ++------- charts/test_resources/pro/values.yaml | 10 --- .../templates/airbyte-keycloak-setup/job.yaml | 7 -- .../airbyte-pod-sweeper/configmap.yaml | 73 ------------------- .../airbyte-pod-sweeper/deployment.yaml | 8 -- charts/v2/airbyte/templates/config/_jobs.tpl | 57 --------------- charts/v2/airbyte/templates/minio.yaml | 12 +-- .../airbyte/templates/tests/test-webapp.yaml | 6 +- charts/v2/airbyte/values.yaml | 25 +------ docker/Makefile | 69 +++--------------- docker/airbyte-base-java-image/Dockerfile | 2 +- docker/airbyte-busybox/Dockerfile | 6 -- docker/airbyte-mc/Dockerfile | 15 ---- settings.gradle.kts | 2 + 68 files changed, 191 insertions(+), 538 deletions(-) create mode 100644 airbyte-keycloak-setup/scripts/entrypoint.sh create mode 100644 airbyte-pod-sweeper/Dockerfile create mode 100644 airbyte-pod-sweeper/build.gradle.kts create mode 100755 airbyte-pod-sweeper/scripts/sweep-pod.sh delete mode 100644 charts/airbyte-pod-sweeper/templates/configmap.yaml delete mode 100644 charts/v2/airbyte/templates/airbyte-pod-sweeper/configmap.yaml delete mode 100644 docker/airbyte-busybox/Dockerfile delete mode 100644 docker/airbyte-mc/Dockerfile diff --git a/airbyte-keycloak-setup/Dockerfile b/airbyte-keycloak-setup/Dockerfile index 0a1cee66dfa..77b70035a5b 100644 --- a/airbyte-keycloak-setup/Dockerfile +++ b/airbyte-keycloak-setup/Dockerfile @@ -3,10 +3,13 @@ ARG JDK_IMAGE=airbyte/airbyte-base-java-image:3.3.1 FROM scratch as builder WORKDIR /app ADD airbyte-app.tar /app +ADD entrypoint.sh entrypoint.sh FROM ${JDK_IMAGE} AS keycloak-setup WORKDIR /app COPY --chown=airbyte:airbyte --from=builder /app /app +RUN chmod +x /app/entrypoint.sh USER airbyte:airbyte +ENTRYPOINT "/app/entrypoint.sh" CMD ["/bin/bash", "-c", "airbyte-app/bin/airbyte-keycloak-setup"] diff --git a/airbyte-keycloak-setup/build.gradle.kts b/airbyte-keycloak-setup/build.gradle.kts index fc665f2087e..4e7b4cd8997 100644 --- a/airbyte-keycloak-setup/build.gradle.kts +++ b/airbyte-keycloak-setup/build.gradle.kts @@ -43,3 +43,12 @@ airbyte { imageName = "keycloak-setup" } } + +val copyScripts = tasks.register("copyScripts") { + from("scripts") + into("build/airbyte/docker/") +} + +tasks.named("dockerCopyDistribution") { + dependsOn(copyScripts) +} \ No newline at end of file diff --git a/airbyte-keycloak-setup/scripts/entrypoint.sh b/airbyte-keycloak-setup/scripts/entrypoint.sh new file mode 100644 index 00000000000..8da176da2ab --- /dev/null +++ b/airbyte-keycloak-setup/scripts/entrypoint.sh @@ -0,0 +1,3 @@ +#/bin/bash +until curl --output /dev/null --head --fail http://${KEYCLOAK_INTERNAL_HOST}/auth/health/ready; do sleep 1; done; +/app/airbyte-app/bin/airbyte-keycloak-setup \ No newline at end of file diff --git a/airbyte-pod-sweeper/Dockerfile b/airbyte-pod-sweeper/Dockerfile new file mode 100644 index 00000000000..b44cfc77fe2 --- /dev/null +++ b/airbyte-pod-sweeper/Dockerfile @@ -0,0 +1,12 @@ +FROM bitnami/kubectl:1.31.4 + +# Create the Airbyte User +USER root +RUN groupadd --gid 1000 airbyte && useradd --uid 1000 -g airbyte --create-home --no-log-init airbyte + +USER airbyte:airbyte + +COPY --chown=airbyte:airbyte sweep-pod.sh /app/sweep-pod.sh +RUN chmod +x /app/sweep-pod.sh + +ENTRYPOINT "/app/sweep-pod.sh" \ No newline at end of file diff --git a/airbyte-pod-sweeper/build.gradle.kts b/airbyte-pod-sweeper/build.gradle.kts new file mode 100644 index 00000000000..4d0f24e18ad --- /dev/null +++ b/airbyte-pod-sweeper/build.gradle.kts @@ -0,0 +1,19 @@ +plugins { + id("io.airbyte.gradle.docker") + id("io.airbyte.gradle.publish") +} + +airbyte { + docker { + imageName = "pod-sweeper" + } +} + +val copyScripts = tasks.register("copyScripts") { + from("scripts") + into("build/airbyte/docker/") +} + +tasks.named("dockerCopyDistribution") { + dependsOn(copyScripts) +} \ No newline at end of file diff --git a/airbyte-pod-sweeper/scripts/sweep-pod.sh b/airbyte-pod-sweeper/scripts/sweep-pod.sh new file mode 100755 index 00000000000..aefd271b0f5 --- /dev/null +++ b/airbyte-pod-sweeper/scripts/sweep-pod.sh @@ -0,0 +1,61 @@ +#!/bin/bash +get_job_pods () { + kubectl -n ${KUBE_NAMESPACE} -L airbyte -l airbyte=job-pod \ + get pods \ + -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime} {.status.startTime}{"\n"}{end}' +} +delete_pod() { + printf "From status '%s' since '%s', " $2 $3 + echo "$1" | grep -v "STATUS" | awk '{print $1}' | xargs --no-run-if-empty kubectl -n ${KUBE_NAMESPACE} delete pod +} +while : +do + echo "Starting pod sweeper cycle:" + + if [ -n "${RUNNING_TTL_MINUTES}" ]; then + # Time window for running pods + RUNNING_DATE_STR=`date -d "now - ${RUNNING_TTL_MINUTES} minutes" --utc -Ins` + RUNNING_DATE=`date -d $RUNNING_DATE_STR +%s` + echo "Will sweep running pods from before ${RUNNING_DATE_STR}" + fi + + if [ -n "${SUCCEEDED_TTL_MINUTES}" ]; then + # Shorter time window for succeeded pods + SUCCESS_DATE_STR=`date -d "now - ${SUCCEEDED_TTL_MINUTES} minutes" --utc -Ins` + SUCCESS_DATE=`date -d $SUCCESS_DATE_STR +%s` + echo "Will sweep succeeded pods from before ${SUCCESS_DATE_STR}" + fi + + if [ -n "${UNSUCCESSFUL_TTL_MINUTES}" ]; then + # Longer time window for unsuccessful pods (to debug) + NON_SUCCESS_DATE_STR=`date -d "now - ${UNSUCCESSFUL_TTL_MINUTES} minutes" --utc -Ins` + NON_SUCCESS_DATE=`date -d $NON_SUCCESS_DATE_STR +%s` + echo "Will sweep unsuccessful pods from before ${NON_SUCCESS_DATE_STR}" + fi + ( + IFS=$'\n' + for POD in `get_job_pods`; do + IFS=' ' + POD_NAME=`echo $POD | cut -d " " -f 1` + POD_STATUS=`echo $POD | cut -d " " -f 2` + POD_DATE_STR=`echo $POD | cut -d " " -f 3` + POD_START_DATE_STR=`echo $POD | cut -d " " -f 4` + POD_DATE=`date -d ${POD_DATE_STR:-$POD_START_DATE_STR} '+%s'` + if [ -n "${RUNNING_TTL_MINUTES}" ] && [ "$POD_STATUS" = "Running" ]; then + if [ "$POD_DATE" -lt "$RUNNING_DATE" ]; then + delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + elif [ -n "${SUCCEEDED_TTL_MINUTES}" ] && [ "$POD_STATUS" = "Succeeded" ]; then + if [ "$POD_DATE" -lt "$SUCCESS_DATE" ]; then + delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + elif [ -n "${UNSUCCESSFUL_TTL_MINUTES}" ] && [ "$POD_STATUS" != "Running" ] && [ "$POD_STATUS" != "Succeeded" ]; then + if [ "$POD_DATE" -lt "$NON_SUCCESS_DATE" ]; then + delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + fi + done + ) + echo "Completed pod sweeper cycle. Sleeping for 60 seconds..." + sleep 60 +done diff --git a/charts/airbyte-bootloader/templates/_images.tpl b/charts/airbyte-bootloader/templates/_images.tpl index 6c32ec44171..e5c1d91f5a9 100644 --- a/charts/airbyte-bootloader/templates/_images.tpl +++ b/charts/airbyte-bootloader/templates/_images.tpl @@ -6,7 +6,7 @@ {{/* ensure the registry has a trailing slash, if set */}} {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} -{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}} {{- if (eq (typeOf $img) "string") -}} {{- printf "%s%s" $reg (tpl $img $root) | quote -}} {{- else -}} diff --git a/charts/airbyte-bootloader/values.yaml b/charts/airbyte-bootloader/values.yaml index ee3bc615733..5a5ef74d478 100644 --- a/charts/airbyte-bootloader/values.yaml +++ b/charts/airbyte-bootloader/values.yaml @@ -23,7 +23,7 @@ global: # Docker registry to pull platform images from, e.g. http://my-registry:8000/ registry: "" # Image tag to use for airbyte images. - # Does not include non-airbyte images such as busybox, temporal, minio, etc. + # Does not include non-airbyte images such as temporal, minio, etc. tag: "" diff --git a/charts/airbyte-connector-builder-server/templates/_images.tpl b/charts/airbyte-connector-builder-server/templates/_images.tpl index 6c32ec44171..e5c1d91f5a9 100644 --- a/charts/airbyte-connector-builder-server/templates/_images.tpl +++ b/charts/airbyte-connector-builder-server/templates/_images.tpl @@ -6,7 +6,7 @@ {{/* ensure the registry has a trailing slash, if set */}} {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} -{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}} {{- if (eq (typeOf $img) "string") -}} {{- printf "%s%s" $reg (tpl $img $root) | quote -}} {{- else -}} diff --git a/charts/airbyte-connector-builder-server/values.yaml b/charts/airbyte-connector-builder-server/values.yaml index 64326ab9de1..03af4be7c2d 100644 --- a/charts/airbyte-connector-builder-server/values.yaml +++ b/charts/airbyte-connector-builder-server/values.yaml @@ -19,7 +19,7 @@ global: # Docker registry to pull platform images from, e.g. http://my-registry:8000/ registry: "" # Image tag to use for airbyte images. - # Does not include non-airbyte images such as busybox, temporal, minio, etc. + # Does not include non-airbyte images such as temporal, minio, etc. tag: "" enabled: true diff --git a/charts/airbyte-connector-rollout-worker/templates/_images.tpl b/charts/airbyte-connector-rollout-worker/templates/_images.tpl index 6c32ec44171..e5c1d91f5a9 100644 --- a/charts/airbyte-connector-rollout-worker/templates/_images.tpl +++ b/charts/airbyte-connector-rollout-worker/templates/_images.tpl @@ -6,7 +6,7 @@ {{/* ensure the registry has a trailing slash, if set */}} {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} -{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}} {{- if (eq (typeOf $img) "string") -}} {{- printf "%s%s" $reg (tpl $img $root) | quote -}} {{- else -}} diff --git a/charts/airbyte-connector-rollout-worker/values.yaml b/charts/airbyte-connector-rollout-worker/values.yaml index 85d6bf18445..97ecc45548a 100644 --- a/charts/airbyte-connector-rollout-worker/values.yaml +++ b/charts/airbyte-connector-rollout-worker/values.yaml @@ -18,7 +18,7 @@ global: # Docker registry to pull platform images from, e.g. http://my-registry:8000/ registry: "" # Image tag to use for airbyte images. - # Does not include non-airbyte images such as busybox, temporal, minio, etc. + # Does not include non-airbyte images such as temporal, minio, etc. tag: "" enterprise: @@ -76,20 +76,6 @@ global: ## jobs.kube.main_container_image_pull_secret [string] image pull secret to use for job pod main_container_image_pull_secret: "" - images: - ## JOB_KUBE_BUSYBOX_IMAGE - ## busybox image used by the job pod - ## jobs.kube.images.busybox [string] busybox image used by the job pod - busybox: "" - ## JOB_KUBE_SOCAT_IMAGE - ## socat image used by the job pod - ## jobs.kube.images.socat [string] socat image used by the job pod - socat: "" - ## JOB_KUBE_CURL_IMAGE - ## curl image used by the job pod - ## jobs.kube.images.curl [string] curl image used by the job pod - curl: "" - enabled: true ## connector-rollout-worker.replicaCount Number of connector-rollout-worker replicas replicaCount: 1 diff --git a/charts/airbyte-cron/templates/_images.tpl b/charts/airbyte-cron/templates/_images.tpl index 6c32ec44171..e5c1d91f5a9 100644 --- a/charts/airbyte-cron/templates/_images.tpl +++ b/charts/airbyte-cron/templates/_images.tpl @@ -6,7 +6,7 @@ {{/* ensure the registry has a trailing slash, if set */}} {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} -{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}} {{- if (eq (typeOf $img) "string") -}} {{- printf "%s%s" $reg (tpl $img $root) | quote -}} {{- else -}} diff --git a/charts/airbyte-cron/values.yaml b/charts/airbyte-cron/values.yaml index 1bbace4329f..d630491463d 100644 --- a/charts/airbyte-cron/values.yaml +++ b/charts/airbyte-cron/values.yaml @@ -18,7 +18,7 @@ global: # Docker registry to pull platform images from, e.g. http://my-registry:8000/ registry: "" # Image tag to use for airbyte images. - # Does not include non-airbyte images such as busybox, temporal, minio, etc. + # Does not include non-airbyte images such as temporal, minio, etc. tag: "" enabled: true diff --git a/charts/airbyte-featureflag-server/templates/_images.tpl b/charts/airbyte-featureflag-server/templates/_images.tpl index 6c32ec44171..e5c1d91f5a9 100644 --- a/charts/airbyte-featureflag-server/templates/_images.tpl +++ b/charts/airbyte-featureflag-server/templates/_images.tpl @@ -6,7 +6,7 @@ {{/* ensure the registry has a trailing slash, if set */}} {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} -{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}} {{- if (eq (typeOf $img) "string") -}} {{- printf "%s%s" $reg (tpl $img $root) | quote -}} {{- else -}} diff --git a/charts/airbyte-featureflag-server/values.yaml b/charts/airbyte-featureflag-server/values.yaml index 888929807e6..edf7c764b7b 100644 --- a/charts/airbyte-featureflag-server/values.yaml +++ b/charts/airbyte-featureflag-server/values.yaml @@ -19,7 +19,7 @@ global: # Docker registry to pull platform images from, e.g. http://my-registry:8000/ registry: "" # Image tag to use for airbyte images. - # Does not include non-airbyte images such as busybox, temporal, minio, etc. + # Does not include non-airbyte images such as temporal, minio, etc. tag: "" enterprise: diff --git a/charts/airbyte-keycloak-setup/templates/_images.tpl b/charts/airbyte-keycloak-setup/templates/_images.tpl index 6c32ec44171..e5c1d91f5a9 100644 --- a/charts/airbyte-keycloak-setup/templates/_images.tpl +++ b/charts/airbyte-keycloak-setup/templates/_images.tpl @@ -6,7 +6,7 @@ {{/* ensure the registry has a trailing slash, if set */}} {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} -{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}} {{- if (eq (typeOf $img) "string") -}} {{- printf "%s%s" $reg (tpl $img $root) | quote -}} {{- else -}} diff --git a/charts/airbyte-keycloak-setup/templates/job.yaml b/charts/airbyte-keycloak-setup/templates/job.yaml index 41e534df66e..4f9fd1a05b2 100644 --- a/charts/airbyte-keycloak-setup/templates/job.yaml +++ b/charts/airbyte-keycloak-setup/templates/job.yaml @@ -38,17 +38,6 @@ spec: affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 12 }} {{- end }} initContainers: - - name: keycloak-readiness-check - image: {{ include "imageUrl" (list .Values.initContainers.keycloakReadinessCheck.image $) }} - command: [ "sh", "-c", "until curl --output /dev/null --head --fail http://${KEYCLOAK_INTERNAL_HOST}/auth/health/ready; do sleep 1; done;" ] - env: - - name: KEYCLOAK_INTERNAL_HOST - valueFrom: - configMapKeyRef: - name: {{ .Values.global.configMapName | default (printf "%s-airbyte-env" .Release.Name) }} - key: KEYCLOAK_INTERNAL_HOST - securityContext: - {{- toYaml .Values.initContainerSecurityContext | nindent 14 }} {{- if .Values.extraInitContainers }} {{- toYaml .Values.extraInitContainers | nindent 8 }} {{- end }} diff --git a/charts/airbyte-keycloak-setup/values.yaml b/charts/airbyte-keycloak-setup/values.yaml index f134b437ce1..ba9947eee7d 100644 --- a/charts/airbyte-keycloak-setup/values.yaml +++ b/charts/airbyte-keycloak-setup/values.yaml @@ -24,7 +24,7 @@ global: # Docker registry to pull platform images from, e.g. http://my-registry:8000/ registry: "" # Image tag to use for airbyte images. - # Does not include non-airbyte images such as busybox, temporal, minio, etc. + # Does not include non-airbyte images such as temporal, minio, etc. tag: "" diff --git a/charts/airbyte-keycloak/templates/_images.tpl b/charts/airbyte-keycloak/templates/_images.tpl index 6c32ec44171..0c39b8caf60 100644 --- a/charts/airbyte-keycloak/templates/_images.tpl +++ b/charts/airbyte-keycloak/templates/_images.tpl @@ -6,7 +6,7 @@ {{/* ensure the registry has a trailing slash, if set */}} {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} -{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{/* some images are defined as a string instead of an object ( connector sidecar, etc) */}} {{- if (eq (typeOf $img) "string") -}} {{- printf "%s%s" $reg (tpl $img $root) | quote -}} {{- else -}} diff --git a/charts/airbyte-keycloak/values.yaml b/charts/airbyte-keycloak/values.yaml index 30de3c4bdd3..1ba80beddf9 100644 --- a/charts/airbyte-keycloak/values.yaml +++ b/charts/airbyte-keycloak/values.yaml @@ -26,7 +26,7 @@ global: # Docker registry to pull platform images from, e.g. http://my-registry:8000/ registry: "" # Image tag to use for airbyte images. - # Does not include non-airbyte images such as busybox, temporal, minio, etc. + # Does not include non-airbyte images such as temporal, minio, etc. tag: "" diff --git a/charts/airbyte-metrics/templates/_images.tpl b/charts/airbyte-metrics/templates/_images.tpl index 6c32ec44171..e5c1d91f5a9 100644 --- a/charts/airbyte-metrics/templates/_images.tpl +++ b/charts/airbyte-metrics/templates/_images.tpl @@ -6,7 +6,7 @@ {{/* ensure the registry has a trailing slash, if set */}} {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} -{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}} {{- if (eq (typeOf $img) "string") -}} {{- printf "%s%s" $reg (tpl $img $root) | quote -}} {{- else -}} diff --git a/charts/airbyte-metrics/values.yaml b/charts/airbyte-metrics/values.yaml index 6ccaa691207..f901dec9a1a 100644 --- a/charts/airbyte-metrics/values.yaml +++ b/charts/airbyte-metrics/values.yaml @@ -14,7 +14,7 @@ global: # Docker registry to pull platform images from, e.g. http://my-registry:8000/ registry: "" # Image tag to use for airbyte images. - # Does not include non-airbyte images such as busybox, temporal, minio, etc. + # Does not include non-airbyte images such as temporal, minio, etc. tag: "" enabled: true diff --git a/charts/airbyte-pod-sweeper/templates/_images.tpl b/charts/airbyte-pod-sweeper/templates/_images.tpl index 6c32ec44171..e5c1d91f5a9 100644 --- a/charts/airbyte-pod-sweeper/templates/_images.tpl +++ b/charts/airbyte-pod-sweeper/templates/_images.tpl @@ -6,7 +6,7 @@ {{/* ensure the registry has a trailing slash, if set */}} {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} -{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}} {{- if (eq (typeOf $img) "string") -}} {{- printf "%s%s" $reg (tpl $img $root) | quote -}} {{- else -}} diff --git a/charts/airbyte-pod-sweeper/templates/configmap.yaml b/charts/airbyte-pod-sweeper/templates/configmap.yaml deleted file mode 100644 index 96ce9350871..00000000000 --- a/charts/airbyte-pod-sweeper/templates/configmap.yaml +++ /dev/null @@ -1,73 +0,0 @@ ---- - -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ include "common.names.fullname" . }}-sweep-pod-script - namespace: {{ .Values.namespace | default .Release.Namespace }} - labels: - {{- include "airbyte.labels" . | nindent 4 }} - -data: - sweep-pod.sh: | - #!/bin/bash - get_job_pods () { - kubectl -n ${KUBE_NAMESPACE} -L airbyte -l airbyte=job-pod \ - get pods \ - -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime} {.status.startTime}{"\n"}{end}' - } - delete_pod() { - printf "From status '%s' since '%s', " $2 $3 - echo "$1" | grep -v "STATUS" | awk '{print $1}' | xargs --no-run-if-empty kubectl -n ${KUBE_NAMESPACE} delete pod - } - while : - do - echo "Starting pod sweeper cycle:" - - if [ -n "${RUNNING_TTL_MINUTES}" ]; then - # Time window for running pods - RUNNING_DATE_STR=`date -d "now - ${RUNNING_TTL_MINUTES} minutes" --utc -Ins` - RUNNING_DATE=`date -d $RUNNING_DATE_STR +%s` - echo "Will sweep running pods from before ${RUNNING_DATE_STR}" - fi - - if [ -n "${SUCCEEDED_TTL_MINUTES}" ]; then - # Shorter time window for succeeded pods - SUCCESS_DATE_STR=`date -d "now - ${SUCCEEDED_TTL_MINUTES} minutes" --utc -Ins` - SUCCESS_DATE=`date -d $SUCCESS_DATE_STR +%s` - echo "Will sweep succeeded pods from before ${SUCCESS_DATE_STR}" - fi - - if [ -n "${UNSUCCESSFUL_TTL_MINUTES}" ]; then - # Longer time window for unsuccessful pods (to debug) - NON_SUCCESS_DATE_STR=`date -d "now - ${UNSUCCESSFUL_TTL_MINUTES} minutes" --utc -Ins` - NON_SUCCESS_DATE=`date -d $NON_SUCCESS_DATE_STR +%s` - echo "Will sweep unsuccessful pods from before ${NON_SUCCESS_DATE_STR}" - fi - ( - IFS=$'\n' - for POD in `get_job_pods`; do - IFS=' ' - POD_NAME=`echo $POD | cut -d " " -f 1` - POD_STATUS=`echo $POD | cut -d " " -f 2` - POD_DATE_STR=`echo $POD | cut -d " " -f 3` - POD_START_DATE_STR=`echo $POD | cut -d " " -f 4` - POD_DATE=`date -d ${POD_DATE_STR:-$POD_START_DATE_STR} '+%s'` - if [ -n "${RUNNING_TTL_MINUTES}" ] && [ "$POD_STATUS" = "Running" ]; then - if [ "$POD_DATE" -lt "$RUNNING_DATE" ]; then - delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" - fi - elif [ -n "${SUCCEEDED_TTL_MINUTES}" ] && [ "$POD_STATUS" = "Succeeded" ]; then - if [ "$POD_DATE" -lt "$SUCCESS_DATE" ]; then - delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" - fi - elif [ -n "${UNSUCCESSFUL_TTL_MINUTES}" ] && [ "$POD_STATUS" != "Running" ] && [ "$POD_STATUS" != "Succeeded" ]; then - if [ "$POD_DATE" -lt "$NON_SUCCESS_DATE" ]; then - delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" - fi - fi - done - ) - echo "Completed pod sweeper cycle. Sleeping for 60 seconds..." - sleep 60 - done diff --git a/charts/airbyte-pod-sweeper/templates/deployment.yaml b/charts/airbyte-pod-sweeper/templates/deployment.yaml index 374b9a4f932..648c6f4c733 100644 --- a/charts/airbyte-pod-sweeper/templates/deployment.yaml +++ b/charts/airbyte-pod-sweeper/templates/deployment.yaml @@ -20,7 +20,6 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }} {{- end }} annotations: - checksum/sweep-pod-script: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} {{- if .Values.podAnnotations }} {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} {{- end }} @@ -60,12 +59,8 @@ spec: securityContext: {{- toYaml .Values.containerSecurityContext | nindent 10 }} {{- end }} volumeMounts: - - mountPath: /script/sweep-pod.sh - subPath: sweep-pod.sh - name: sweep-pod-script - mountPath: /.kube name: kube-config - command: ["/bin/bash", "-c", /script/sweep-pod.sh] {{- if .Values.resources }} resources: {{- toYaml .Values.resources | nindent 10 }} {{- end }} @@ -98,8 +93,4 @@ spec: securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} volumes: - name: kube-config - emptyDir: {} - - name: sweep-pod-script - configMap: - name: {{ include "common.names.fullname" . }}-sweep-pod-script - defaultMode: 0755 + emptyDir: {} \ No newline at end of file diff --git a/charts/airbyte-pod-sweeper/values.yaml b/charts/airbyte-pod-sweeper/values.yaml index fa4bb8a6c5b..c4e4d9b0280 100644 --- a/charts/airbyte-pod-sweeper/values.yaml +++ b/charts/airbyte-pod-sweeper/values.yaml @@ -6,7 +6,7 @@ global: # Docker registry to pull platform images from, e.g. http://my-registry:8000/ registry: "" # Image tag to use for airbyte images. - # Does not include non-airbyte images such as busybox, temporal, minio, etc. + # Does not include non-airbyte images such as temporal, minio, etc. tag: "" enabled: true @@ -14,9 +14,8 @@ enabled: true ## podSweeper.image.pullPolicy The pull policy for the pod sweeper image ## podSweeper.image.tag The pod sweeper image tag to use image: - repository: bitnami/kubectl + repository: airbyte/pod-sweeper pullPolicy: IfNotPresent - tag: latest ## podSweeper.podAnnotations [object] Add extra annotations to the podSweeper pod ## diff --git a/charts/airbyte-server/templates/_images.tpl b/charts/airbyte-server/templates/_images.tpl index 6c32ec44171..e5c1d91f5a9 100644 --- a/charts/airbyte-server/templates/_images.tpl +++ b/charts/airbyte-server/templates/_images.tpl @@ -6,7 +6,7 @@ {{/* ensure the registry has a trailing slash, if set */}} {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} -{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}} {{- if (eq (typeOf $img) "string") -}} {{- printf "%s%s" $reg (tpl $img $root) | quote -}} {{- else -}} diff --git a/charts/airbyte-server/values.yaml b/charts/airbyte-server/values.yaml index 390f80a4967..14990cc883f 100644 --- a/charts/airbyte-server/values.yaml +++ b/charts/airbyte-server/values.yaml @@ -23,7 +23,7 @@ global: # Docker registry to pull platform images from, e.g. http://my-registry:8000/ registry: "" # Image tag to use for airbyte images. - # Does not include non-airbyte images such as busybox, temporal, minio, etc. + # Does not include non-airbyte images such as temporal, minio, etc. tag: "" enabled: true diff --git a/charts/airbyte-temporal-ui/templates/_images.tpl b/charts/airbyte-temporal-ui/templates/_images.tpl index 6c32ec44171..e5c1d91f5a9 100644 --- a/charts/airbyte-temporal-ui/templates/_images.tpl +++ b/charts/airbyte-temporal-ui/templates/_images.tpl @@ -6,7 +6,7 @@ {{/* ensure the registry has a trailing slash, if set */}} {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} -{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}} {{- if (eq (typeOf $img) "string") -}} {{- printf "%s%s" $reg (tpl $img $root) | quote -}} {{- else -}} diff --git a/charts/airbyte-temporal-ui/values.yaml b/charts/airbyte-temporal-ui/values.yaml index 9430941dd42..ae4aefe2b01 100644 --- a/charts/airbyte-temporal-ui/values.yaml +++ b/charts/airbyte-temporal-ui/values.yaml @@ -19,7 +19,7 @@ global: # Docker registry to pull platform images from, e.g. http://my-registry:8000/ registry: "" # Image tag to use for airbyte images. - # Does not include non-airbyte images such as busybox, temporal, minio, etc. + # Does not include non-airbyte images such as temporal, minio, etc. tag: "" enabled: false diff --git a/charts/airbyte-temporal/templates/_images.tpl b/charts/airbyte-temporal/templates/_images.tpl index 6c32ec44171..e5c1d91f5a9 100644 --- a/charts/airbyte-temporal/templates/_images.tpl +++ b/charts/airbyte-temporal/templates/_images.tpl @@ -6,7 +6,7 @@ {{/* ensure the registry has a trailing slash, if set */}} {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} -{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}} {{- if (eq (typeOf $img) "string") -}} {{- printf "%s%s" $reg (tpl $img $root) | quote -}} {{- else -}} diff --git a/charts/airbyte-temporal/values.yaml b/charts/airbyte-temporal/values.yaml index 64f90af2137..be0fb3bdf6f 100644 --- a/charts/airbyte-temporal/values.yaml +++ b/charts/airbyte-temporal/values.yaml @@ -18,7 +18,7 @@ global: # Docker registry to pull platform images from, e.g. http://my-registry:8000/ registry: "" # Image tag to use for airbyte images. - # Does not include non-airbyte images such as busybox, temporal, minio, etc. + # Does not include non-airbyte images such as temporal, minio, etc. tag: "" enabled: true diff --git a/charts/airbyte-webapp/templates/_images.tpl b/charts/airbyte-webapp/templates/_images.tpl index 6c32ec44171..e5c1d91f5a9 100644 --- a/charts/airbyte-webapp/templates/_images.tpl +++ b/charts/airbyte-webapp/templates/_images.tpl @@ -6,7 +6,7 @@ {{/* ensure the registry has a trailing slash, if set */}} {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} -{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}} {{- if (eq (typeOf $img) "string") -}} {{- printf "%s%s" $reg (tpl $img $root) | quote -}} {{- else -}} diff --git a/charts/airbyte-webapp/values.yaml b/charts/airbyte-webapp/values.yaml index aedb5076aa5..c673b8420c7 100644 --- a/charts/airbyte-webapp/values.yaml +++ b/charts/airbyte-webapp/values.yaml @@ -17,7 +17,7 @@ global: # Docker registry to pull platform images from, e.g. http://my-registry:8000/ registry: "" # Image tag to use for airbyte images. - # Does not include non-airbyte images such as busybox, temporal, minio, etc. + # Does not include non-airbyte images such as temporal, minio, etc. tag: "" enabled: true diff --git a/charts/airbyte-worker/README.md b/charts/airbyte-worker/README.md index a9d1b210c87..c867085a0b3 100644 --- a/charts/airbyte-worker/README.md +++ b/charts/airbyte-worker/README.md @@ -35,8 +35,6 @@ Helm chart to deploy airbyte-worker | global.extraLabels | object | `{}` | | | global.extraSelectorLabels | object | `{}` | | | global.jobs.kube.annotations | object | `{}` | | -| global.jobs.kube.images.busybox | string | `""` | | -| global.jobs.kube.images.curl | string | `""` | | | global.jobs.kube.labels | object | `{}` | | | global.jobs.kube.main_container_image_pull_secret | string | `""` | | | global.jobs.kube.nodeSelector | object | `{}` | | diff --git a/charts/airbyte-worker/templates/_images.tpl b/charts/airbyte-worker/templates/_images.tpl index 6c32ec44171..e5c1d91f5a9 100644 --- a/charts/airbyte-worker/templates/_images.tpl +++ b/charts/airbyte-worker/templates/_images.tpl @@ -6,7 +6,7 @@ {{/* ensure the registry has a trailing slash, if set */}} {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} -{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}} {{- if (eq (typeOf $img) "string") -}} {{- printf "%s%s" $reg (tpl $img $root) | quote -}} {{- else -}} diff --git a/charts/airbyte-worker/templates/deployment.yaml b/charts/airbyte-worker/templates/deployment.yaml index ad46daeb3dd..9da708036d0 100644 --- a/charts/airbyte-worker/templates/deployment.yaml +++ b/charts/airbyte-worker/templates/deployment.yaml @@ -164,16 +164,6 @@ spec: name: {{ .Release.Name }}-airbyte-env key: JOB_KUBE_TOLERATIONS {{- end }} - - name: JOB_KUBE_BUSYBOX_IMAGE - valueFrom: - configMapKeyRef: - name: {{ .Release.Name }}-airbyte-env - key: JOB_KUBE_BUSYBOX_IMAGE - - name: JOB_KUBE_CURL_IMAGE - valueFrom: - configMapKeyRef: - name: {{ .Release.Name }}-airbyte-env - key: JOB_KUBE_CURL_IMAGE {{- if $.Values.global.jobs.kube.main_container_image_pull_secret }} - name: JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET valueFrom: diff --git a/charts/airbyte-worker/values.yaml b/charts/airbyte-worker/values.yaml index b05d279e669..09cf8e0ccc2 100644 --- a/charts/airbyte-worker/values.yaml +++ b/charts/airbyte-worker/values.yaml @@ -20,7 +20,7 @@ global: # Docker registry to pull platform images from, e.g. http://my-registry:8000/ registry: "" # Image tag to use for airbyte images. - # Does not include non-airbyte images such as busybox, temporal, minio, etc. + # Does not include non-airbyte images such as temporal, minio, etc. tag: "" jobs: @@ -75,16 +75,6 @@ global: ## jobs.kube.main_container_image_pull_secret [string] image pull secret to use for job pod main_container_image_pull_secret: "" - images: - ## JOB_KUBE_BUSYBOX_IMAGE - ## busybox image used by the job pod - ## jobs.kube.images.busybox [string] busybox image used by the job pod - busybox: "" - ## JOB_KUBE_CURL_IMAGE - ## curl image used by the job pod - ## jobs.kube.images.curl [string] curl image used by the job pod - curl: "" - enabled: true ## worker.replicaCount Number of worker replicas replicaCount: 1 diff --git a/charts/airbyte-workload-api-server/templates/_images.tpl b/charts/airbyte-workload-api-server/templates/_images.tpl index 6c32ec44171..e5c1d91f5a9 100644 --- a/charts/airbyte-workload-api-server/templates/_images.tpl +++ b/charts/airbyte-workload-api-server/templates/_images.tpl @@ -6,7 +6,7 @@ {{/* ensure the registry has a trailing slash, if set */}} {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} -{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}} {{- if (eq (typeOf $img) "string") -}} {{- printf "%s%s" $reg (tpl $img $root) | quote -}} {{- else -}} diff --git a/charts/airbyte-workload-api-server/values.yaml b/charts/airbyte-workload-api-server/values.yaml index 2c378abcb3a..c5fc1fe1bed 100644 --- a/charts/airbyte-workload-api-server/values.yaml +++ b/charts/airbyte-workload-api-server/values.yaml @@ -19,7 +19,7 @@ global: # Docker registry to pull platform images from, e.g. http://my-registry:8000/ registry: "" # Image tag to use for airbyte images. - # Does not include non-airbyte images such as busybox, temporal, minio, etc. + # Does not include non-airbyte images such as temporal, minio, etc. tag: "" logs: diff --git a/charts/airbyte-workload-launcher/README.md b/charts/airbyte-workload-launcher/README.md index d636166fd0f..07b5d1275d8 100644 --- a/charts/airbyte-workload-launcher/README.md +++ b/charts/airbyte-workload-launcher/README.md @@ -34,8 +34,6 @@ Helm chart to deploy airbyte-workload-launcher | global.extraLabels | object | `{}` | | | global.extraSelectorLabels | object | `{}` | | | global.jobs.kube.annotations | object | `{}` | | -| global.jobs.kube.images.busybox | string | `""` | | -| global.jobs.kube.images.curl | string | `""` | | | global.jobs.kube.labels | object | `{}` | | | global.jobs.kube.main_container_image_pull_secret | string | `""` | | | global.jobs.kube.nodeSelector | object | `{}` | | diff --git a/charts/airbyte-workload-launcher/templates/_images.tpl b/charts/airbyte-workload-launcher/templates/_images.tpl index 6c32ec44171..e5c1d91f5a9 100644 --- a/charts/airbyte-workload-launcher/templates/_images.tpl +++ b/charts/airbyte-workload-launcher/templates/_images.tpl @@ -6,7 +6,7 @@ {{/* ensure the registry has a trailing slash, if set */}} {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} -{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}} {{- if (eq (typeOf $img) "string") -}} {{- printf "%s%s" $reg (tpl $img $root) | quote -}} {{- else -}} diff --git a/charts/airbyte-workload-launcher/templates/deployment.yaml b/charts/airbyte-workload-launcher/templates/deployment.yaml index f354a6b71b7..7f26ffe6fbc 100644 --- a/charts/airbyte-workload-launcher/templates/deployment.yaml +++ b/charts/airbyte-workload-launcher/templates/deployment.yaml @@ -205,16 +205,6 @@ spec: configMapKeyRef: name: {{ .Release.Name }}-airbyte-env key: CONTAINER_ORCHESTRATOR_IMAGE - - name: JOB_KUBE_BUSYBOX_IMAGE - valueFrom: - configMapKeyRef: - name: {{ .Release.Name }}-airbyte-env - key: JOB_KUBE_BUSYBOX_IMAGE - - name: JOB_KUBE_CURL_IMAGE - valueFrom: - configMapKeyRef: - name: {{ .Release.Name }}-airbyte-env - key: JOB_KUBE_CURL_IMAGE - name: JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET valueFrom: configMapKeyRef: diff --git a/charts/airbyte-workload-launcher/values.yaml b/charts/airbyte-workload-launcher/values.yaml index 486760a56d6..b5bbe549e10 100644 --- a/charts/airbyte-workload-launcher/values.yaml +++ b/charts/airbyte-workload-launcher/values.yaml @@ -21,7 +21,7 @@ global: # Docker registry to pull platform images from, e.g. http://my-registry:8000/ registry: "" # Image tag to use for airbyte images. - # Does not include non-airbyte images such as busybox, temporal, minio, etc. + # Does not include non-airbyte images such as temporal, minio, etc. tag: "" jobs: @@ -76,16 +76,6 @@ global: ## jobs.kube.main_container_image_pull_secret [string] image pull secret to use for job pod main_container_image_pull_secret: "" - images: - ## JOB_KUBE_BUSYBOX_IMAGE - ## busybox image used by the job pod - ## jobs.kube.images.busybox [string] busybox image used by the job pod - busybox: "" - ## JOB_KUBE_CURL_IMAGE - ## curl image used by the job pod - ## jobs.kube.images.curl [string] curl image used by the job pod - curl: "" - enabled: true ## workload-launcher.replicaCount Number of workload launcher replicas replicaCount: 1 diff --git a/charts/airbyte/README.md b/charts/airbyte/README.md index 1cb93a2638b..94b500ec099 100644 --- a/charts/airbyte/README.md +++ b/charts/airbyte/README.md @@ -154,8 +154,6 @@ Helm chart to deploy airbyte | global.enterprise.secretName | string | `"airbyte-config-secrets"` | Secret name where an Airbyte license key is stored | | global.env_vars | object | `{}` | Environment variables | | global.jobs.kube.annotations | object | `{}` | key/value annotations applied to kube jobs | -| global.jobs.kube.images.busybox | string | `""` | busybox image used by the job pod | -| global.jobs.kube.images.curl | string | `""` | curl image used by the job pod | | global.jobs.kube.labels | object | `{}` | key/value labels applied to kube jobs | | global.jobs.kube.main_container_image_pull_secret | string | `""` | image pull secret to use for job pod | | global.jobs.kube.nodeSelector | object | `{}` | Node labels for pod assignment | diff --git a/charts/airbyte/templates/_images.tpl b/charts/airbyte/templates/_images.tpl index 6c32ec44171..e5c1d91f5a9 100644 --- a/charts/airbyte/templates/_images.tpl +++ b/charts/airbyte/templates/_images.tpl @@ -6,7 +6,7 @@ {{/* ensure the registry has a trailing slash, if set */}} {{- $reg = (ternary $reg (printf "%s/" (trimSuffix "/" $reg)) (empty $reg)) -}} -{{/* some images are defined as a string instead of an object (busybox, curl, connector sidecar, etc) */}} +{{/* some images are defined as a string instead of an object (connector sidecar, etc) */}} {{- if (eq (typeOf $img) "string") -}} {{- printf "%s%s" $reg (tpl $img $root) | quote -}} {{- else -}} diff --git a/charts/airbyte/templates/env-configmap.yaml b/charts/airbyte/templates/env-configmap.yaml index a28b65f205c..282f5cbda8f 100644 --- a/charts/airbyte/templates/env-configmap.yaml +++ b/charts/airbyte/templates/env-configmap.yaml @@ -89,8 +89,6 @@ data: JOB_KUBE_TOLERATIONS: {{ $.Values.global.jobs.kube.tolerations | include "airbyte.flattenArrayMap" | quote }} {{- end }} - JOB_KUBE_BUSYBOX_IMAGE: {{ include "imageUrl" (list $.Values.global.jobs.kube.images.busybox $) }} - JOB_KUBE_CURL_IMAGE: {{ include "imageUrl" (list $.Values.global.jobs.kube.images.curl $) }} {{ $workloadLauncher := (index .Values "workload-launcher") }} CONTAINER_ORCHESTRATOR_IMAGE: {{ include "imageUrl" (list $workloadLauncher.containerOrchestrator.image $) }} WORKLOAD_INIT_IMAGE: {{ include "imageUrl" (list $workloadLauncher.workloadInit.image $) }} diff --git a/charts/airbyte/templates/minio.yaml b/charts/airbyte/templates/minio.yaml index 4c6bd8cbe3d..94f552887a4 100644 --- a/charts/airbyte/templates/minio.yaml +++ b/charts/airbyte/templates/minio.yaml @@ -133,13 +133,13 @@ spec: fsGroup: 1000 containers: - name: minio-mc - image: {{ include "imageUrl" (list .Values.minio.mcImage $) }} + image: {{ include "imageUrl" (list .Values.minio.image $) }} command: ["/bin/sh", "-c", - "until (/usr/bin/mc config host add myminio $MINIO_ENDPOINT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY) do echo '...waiting...' && sleep 1; done; - /usr/bin/mc mb --ignore-existing myminio/state-storage; - /usr/bin/mc policy set public myminio/state-storage; - /usr/bin/mc mb --ignore-existing myminio/airbyte-dev-logs; - /usr/bin/mc policy set public myminio/airbyte-dev-logs;"] + "until (/bin/mc config host add myminio $MINIO_ENDPOINT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY) do echo '...waiting...' && sleep 1; done; + /bin/mc mb --ignore-existing myminio/state-storage; + /bin/mc policy set public myminio/state-storage; + /bin/mc mb --ignore-existing myminio/airbyte-dev-logs; + /bin/mc policy set public myminio/airbyte-dev-logs;"] securityContext: allowPrivilegeEscalation: false runAsNonRoot: true diff --git a/charts/airbyte/templates/tests/test-webapp.yaml b/charts/airbyte/templates/tests/test-webapp.yaml index 8b33ae24b04..99d4c75bb9d 100644 --- a/charts/airbyte/templates/tests/test-webapp.yaml +++ b/charts/airbyte/templates/tests/test-webapp.yaml @@ -15,10 +15,10 @@ spec: {{- end }} {{- end }} containers: - - name: wget + - name: webapp-test image: {{ include "imageUrl" (list .Values.testWebapp.image $) }} - command: ['wget'] - args: ['{{ .Release.Name }}-airbyte-webapp-svc:{{ .Values.webapp.service.port }}'] + command: ['curl'] + args: ['-s', '{{ .Release.Name }}-airbyte-webapp-svc:{{ .Values.webapp.service.port }}', '>', '/dev/null'] resources: requests: memory: "64Mi" diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml index 1f20859a511..d98b9473570 100644 --- a/charts/airbyte/values.yaml +++ b/charts/airbyte/values.yaml @@ -26,7 +26,7 @@ global: # Docker registry to pull platform images from, e.g. http://my-registry:8000/ registry: "" # Image tag to use for airbyte images. - # Does not include non-airbyte images such as busybox, temporal, minio, etc. + # Does not include non-airbyte images such as temporal, minio, etc. tag: "" # Docker image pull secret @@ -185,14 +185,6 @@ global: # -- image pull secret to use for job pod main_container_image_pull_secret: "" - images: - ## JOB_KUBE_BUSYBOX_IMAGE - # -- busybox image used by the job pod - busybox: "busybox:1.35" - ## JOB_KUBE_CURL_IMAGE - # -- curl image used by the job pod - curl: "curlimages/curl:8.1.1" - ## @section Common Parameters # -- String to partially override airbyte.fullname template with a string (will prepend the release name) @@ -447,11 +439,7 @@ pod-sweeper: enabled: true image: # -- The image repository to use for the pod sweeper - repository: bitnami/kubectl - # -- The pull policy for the pod sweeper image - pullPolicy: IfNotPresent - # -- The pod sweeper image tag to use - tag: 1.28.9 + repository: airbyte/pod-sweeper # -- Add extra annotations to the podSweeper pod podAnnotations: {} @@ -1732,10 +1720,6 @@ minio: # -- Minio tag image tag: RELEASE.2023-11-20T22-40-07Z - mcImage: - repository: airbyte/mc - tag: latest - storage: volumeClaimValue: 500Mi @@ -2106,10 +2090,6 @@ keycloak-setup: seccompProfile: type: RuntimeDefault - initContainers: - keycloakReadinessCheck: - image: "curlimages/curl:8.1.1" - containerSecurityContext: allowPrivilegeEscalation: false runAsNonRoot: true @@ -2373,5 +2353,5 @@ featureflag-server: testWebapp: image: - repository: busybox - tag: latest + repository: airbyte/airbyte-base-java-image + tag: 3.3.1 diff --git a/charts/airbyte/values.yaml.test b/charts/airbyte/values.yaml.test index 2c9b6c11ab9..3818aa09b17 100644 --- a/charts/airbyte/values.yaml.test +++ b/charts/airbyte/values.yaml.test @@ -129,16 +129,6 @@ global: ## jobs.kube.main_container_image_pull_secret [string] image pull secret to use for job pod main_container_image_pull_secret: "" - images: - ## JOB_KUBE_BUSYBOX_IMAGE - ## busybox image used by the job pod - ## jobs.kube.images.busybox [string] busybox image used by the job pod - busybox: "" - ## JOB_KUBE_CURL_IMAGE - ## curl image used by the job pod - ## jobs.kube.images.curl [string] curl image used by the job pod - curl: "" - ## @section Common Parameters ## nameOverride -- String to partially override airbyte.fullname template with a string (will prepend the release name) diff --git a/charts/helm-tests/tests/basic_template_test.go b/charts/helm-tests/tests/basic_template_test.go index 68d1515b163..6b6bad021df 100644 --- a/charts/helm-tests/tests/basic_template_test.go +++ b/charts/helm-tests/tests/basic_template_test.go @@ -126,8 +126,6 @@ var commonConfigMapKeys = []string{ "GOOGLE_APPLICATION_CREDENTIALS", "INTERNAL_API_HOST", "JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION", - "JOB_KUBE_BUSYBOX_IMAGE", - "JOB_KUBE_CURL_IMAGE", "JOB_KUBE_CONNECTOR_IMAGE_REGISTRY", "JOB_KUBE_LOCAL_VOLUME_ENABLED", "JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET", diff --git a/charts/helm-tests/tests/enterprise_config_test.go b/charts/helm-tests/tests/enterprise_config_test.go index c1199afae62..98c8aa071e2 100644 --- a/charts/helm-tests/tests/enterprise_config_test.go +++ b/charts/helm-tests/tests/enterprise_config_test.go @@ -250,7 +250,7 @@ func TestBasicEnterpriseConfigWithHelmValues(t *testing.T) { } func TestKeycloakInitContainerOverride(t *testing.T) { - t.Run("default keycloak readiness image is curlimages/curl", func(t *testing.T) { + t.Run("default keycloak readiness image is airbyte/utils", func(t *testing.T) { helmOpts := BaseHelmOptionsForEnterpriseWithValues() helmOpts.SetValues["global.auth.instanceAdmin.firstName"] = "Octavia" helmOpts.SetValues["global.auth.instanceAdmin.lastName"] = "Squidington" @@ -262,16 +262,13 @@ func TestKeycloakInitContainerOverride(t *testing.T) { helmOpts.SetValues["global.auth.identityProvider.oidc.clientSecretSecretKey"] = "client-secret" chartYaml := renderChart(t, helmOpts) - keycloakSetupJob := getJob(chartYaml, "airbyte-keycloak-setup") keycloakStatefulSet := getStatefulSet(chartYaml, "airbyte-keycloak") - setupInitContainers := keycloakSetupJob.Spec.Template.Spec.InitContainers keycloakInitContainers := keycloakStatefulSet.Spec.Template.Spec.InitContainers - assert.Equal(t, "curlimages/curl:8.1.1", setupInitContainers[0].Image) assert.Equal(t, "postgres:13-alpine", keycloakInitContainers[0].Image) }) - t.Run("override init container image ", func(t *testing.T) { + t.Run("override init container image", func(t *testing.T) { helmOpts := BaseHelmOptionsForEnterpriseWithValues() helmOpts.SetValues["global.auth.instanceAdmin.firstName"] = "Octavia" helmOpts.SetValues["global.auth.instanceAdmin.lastName"] = "Squidington" @@ -281,16 +278,12 @@ func TestKeycloakInitContainerOverride(t *testing.T) { helmOpts.SetValues["global.auth.identityProvider.oidc.appName"] = "example-app" helmOpts.SetValues["global.auth.identityProvider.oidc.clientIdSecretKey"] = "client-id" helmOpts.SetValues["global.auth.identityProvider.oidc.clientSecretSecretKey"] = "client-secret" - helmOpts.SetValues["keycloak-setup.initContainers.keycloakReadinessCheck.image"] = "airbyte/custom-curl-image" helmOpts.SetValues["keycloak.initContainers.initDb.image"] = "airbyte/custom-postgres-image" chartYaml := renderChart(t, helmOpts) - keycloakSetupJob := getJob(chartYaml, "airbyte-keycloak-setup") keycloakStatefulSet := getStatefulSet(chartYaml, "airbyte-keycloak") - setupInitContainers := keycloakSetupJob.Spec.Template.Spec.InitContainers keycloakInitContainers := keycloakStatefulSet.Spec.Template.Spec.InitContainers - assert.Equal(t, "airbyte/custom-curl-image", setupInitContainers[0].Image) assert.Equal(t, "airbyte/custom-postgres-image", keycloakInitContainers[0].Image) }) } diff --git a/charts/helm-tests/tests/image_test.go b/charts/helm-tests/tests/image_test.go index 8a2e2bed65b..a00f279a23e 100644 --- a/charts/helm-tests/tests/image_test.go +++ b/charts/helm-tests/tests/image_test.go @@ -19,22 +19,19 @@ func TestImages_Default(t *testing.T) { assert.ElementsMatch(t, images, []string{ "airbyte/connector-builder-server:dev", "airbyte/cron:dev", - "bitnami/kubectl:1.28.9", "airbyte/server:dev", + "airbyte/pod-sweeper:dev", + "airbyte/airbyte-base-java-image:3.3.1", "temporalio/auto-setup:1.23.0", "airbyte/webapp:dev", "airbyte/worker:dev", "airbyte/workload-api-server:dev", "airbyte/workload-launcher:dev", "airbyte/bootloader:dev", - "airbyte/mc:latest", - "busybox:latest", "airbyte/db:dev", "minio/minio:RELEASE.2023-11-20T22-40-07Z", "airbyte/workload-init-container:dev", - "curlimages/curl:8.1.1", "airbyte/connector-sidecar:dev", - "busybox:1.35", "airbyte/container-orchestrator:dev", }) } @@ -48,16 +45,15 @@ func TestImages_DefaultAllEnabled(t *testing.T) { assert.ElementsMatch(t, images, []string{ "airbyte/connector-builder-server:dev", "airbyte/cron:dev", - "bitnami/kubectl:1.28.9", "airbyte/server:dev", "temporalio/auto-setup:1.23.0", "airbyte/webapp:dev", "airbyte/worker:dev", + "airbyte/pod-sweeper:dev", + "airbyte/airbyte-base-java-image:3.3.1", "airbyte/workload-api-server:dev", "airbyte/workload-launcher:dev", "airbyte/bootloader:dev", - "airbyte/mc:latest", - "busybox:latest", "airbyte/db:dev", "minio/minio:RELEASE.2023-11-20T22-40-07Z", "airbyte/metrics-reporter:dev", @@ -65,11 +61,9 @@ func TestImages_DefaultAllEnabled(t *testing.T) { "postgres:13-alpine", "airbyte/keycloak:dev", "airbyte/keycloak-setup:dev", - "curlimages/curl:8.1.1", "airbyte/container-orchestrator:dev", "airbyte/workload-init-container:dev", "airbyte/connector-sidecar:dev", - "busybox:1.35", }) } @@ -83,22 +77,19 @@ func TestImages_GlobalTag(t *testing.T) { assert.ElementsMatch(t, images, []string{ "airbyte/connector-builder-server:test-tag", "airbyte/cron:test-tag", - "bitnami/kubectl:1.28.9", "airbyte/server:test-tag", "temporalio/auto-setup:1.23.0", "airbyte/webapp:test-tag", "airbyte/worker:test-tag", "airbyte/workload-api-server:test-tag", "airbyte/workload-launcher:test-tag", + "airbyte/pod-sweeper:test-tag", + "airbyte/airbyte-base-java-image:3.3.1", "airbyte/bootloader:test-tag", - "airbyte/mc:latest", - "busybox:latest", "airbyte/db:test-tag", "minio/minio:RELEASE.2023-11-20T22-40-07Z", "airbyte/metrics-reporter:test-tag", "temporalio/ui:2.30.1", - "curlimages/curl:8.1.1", - "busybox:1.35", "postgres:13-alpine", "airbyte/connector-sidecar:test-tag", "airbyte/workload-init-container:test-tag", @@ -135,8 +126,6 @@ func TestImages_GlobalRegistry(t *testing.T) { } // The loop above checks these too, but these are important core images, so they're tested explicitly here. - assert.Equal(t, "http://my-registry/busybox:1.35", env.Data["JOB_KUBE_BUSYBOX_IMAGE"]) - assert.Equal(t, "http://my-registry/curlimages/curl:8.1.1", env.Data["JOB_KUBE_CURL_IMAGE"]) assert.Equal(t, "http://my-registry/airbyte/container-orchestrator:dev", env.Data["CONTAINER_ORCHESTRATOR_IMAGE"]) assert.Equal(t, "http://my-registry/airbyte/connector-sidecar:dev", env.Data["CONNECTOR_SIDECAR_IMAGE"]) assert.Equal(t, "http://my-registry/airbyte/workload-init-container:dev", env.Data["WORKLOAD_INIT_IMAGE"]) @@ -166,30 +155,26 @@ func TestImages_AppTag(t *testing.T) { for _, app := range slices.Concat(allApps, moreApps) { setAppOpt(opts, app, "image.tag", "app-tag") } - opts.SetValues["minio.mcImage.tag"] = "mc-app-tag" chart := renderChart(t, opts) images := findAllImages(chart) assert.ElementsMatch(t, images, []string{ "airbyte/connector-builder-server:app-tag", "airbyte/cron:app-tag", - "bitnami/kubectl:app-tag", "airbyte/server:app-tag", "temporalio/auto-setup:app-tag", + "airbyte/pod-sweeper:app-tag", + "airbyte/airbyte-base-java-image:app-tag", "airbyte/webapp:app-tag", "airbyte/worker:app-tag", "airbyte/workload-api-server:app-tag", "airbyte/workload-launcher:app-tag", "airbyte/bootloader:app-tag", - "airbyte/mc:mc-app-tag", - "busybox:app-tag", "airbyte/db:app-tag", "minio/minio:app-tag", "airbyte/metrics-reporter:app-tag", "temporalio/ui:app-tag", - "busybox:1.35", "postgres:13-alpine", - "curlimages/curl:8.1.1", "airbyte/keycloak:app-tag", "airbyte/keycloak-setup:app-tag", // these don't support app tags due to backwards compat. @@ -230,16 +215,12 @@ func TestImages_StringImages(t *testing.T) { // Make sure the imageUrl helper handles the cases where the image value // is a string instead of an object. opts := BaseHelmOptions() - opts.SetValues["global.jobs.kube.images.busybox"] = "my-busybox" - opts.SetValues["global.jobs.kube.images.curl"] = "my-curl" opts.SetValues["workload-launcher.containerOrchestrator.image"] = "my-oc" opts.SetValues["workload-launcher.connectorSidecar.image"] = "my-cs" opts.SetValues["workload-launcher.workloadInit.image"] = "my-wi" chart := renderChart(t, opts) env := getConfigMap(chart, "airbyte-airbyte-env") - assert.Equal(t, "my-busybox", env.Data["JOB_KUBE_BUSYBOX_IMAGE"]) - assert.Equal(t, "my-curl", env.Data["JOB_KUBE_CURL_IMAGE"]) assert.Equal(t, "my-oc", env.Data["CONTAINER_ORCHESTRATOR_IMAGE"]) assert.Equal(t, "my-cs", env.Data["CONNECTOR_SIDECAR_IMAGE"]) assert.Equal(t, "my-wi", env.Data["WORKLOAD_INIT_IMAGE"]) diff --git a/charts/test_resources/pro/values.yaml b/charts/test_resources/pro/values.yaml index 74acfa3f122..a5e27c0a4d4 100644 --- a/charts/test_resources/pro/values.yaml +++ b/charts/test_resources/pro/values.yaml @@ -130,16 +130,6 @@ global: ## jobs.kube.main_container_image_pull_secret [string] image pull secret to use for job pod main_container_image_pull_secret: "" - images: - ## JOB_KUBE_BUSYBOX_IMAGE - ## busybox image used by the job pod - ## jobs.kube.images.busybox [string] busybox image used by the job pod - busybox: "" - ## JOB_KUBE_CURL_IMAGE - ## curl image used by the job pod - ## jobs.kube.images.curl [string] curl image used by the job pod - curl: "" - ## @section Common Parameters ## nameOverride -- String to partially override airbyte.fullname template with a string (will prepend the release name) diff --git a/charts/v2/airbyte/templates/airbyte-keycloak-setup/job.yaml b/charts/v2/airbyte/templates/airbyte-keycloak-setup/job.yaml index 3b5a3295328..90dcdee4e87 100644 --- a/charts/v2/airbyte/templates/airbyte-keycloak-setup/job.yaml +++ b/charts/v2/airbyte/templates/airbyte-keycloak-setup/job.yaml @@ -38,13 +38,6 @@ spec: affinity: {{- include "common.tplvalues.render" (dict "value" .Values.keycloakSetup.affinity "context" $) | nindent 6 }} {{- end }} initContainers: - - name: keycloak-readiness-check - image: {{ include "imageUrl" (list .Values.keycloakSetup.initContainers.keycloakReadinessCheck.image $) }} - command: [ "sh", "-c", "until curl --output /dev/null --head --fail http://${KEYCLOAK_INTERNAL_HOST}/auth/health/ready; do sleep 1; done;" ] - env: - {{- include "airbyte.keycloak.admin.client.internalHost.env" . | nindent 12 }} - securityContext: - {{- toYaml .Values.keycloakSetup.initContainerSecurityContext | nindent 12 }} {{- if .Values.keycloakSetup.extraInitContainers }} {{- toYaml .Values.keycloakSetup.extraInitContainers | nindent 8 }} {{- end }} diff --git a/charts/v2/airbyte/templates/airbyte-pod-sweeper/configmap.yaml b/charts/v2/airbyte/templates/airbyte-pod-sweeper/configmap.yaml deleted file mode 100644 index faaec781d91..00000000000 --- a/charts/v2/airbyte/templates/airbyte-pod-sweeper/configmap.yaml +++ /dev/null @@ -1,73 +0,0 @@ ---- - -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-pod-sweeper-sweep-pod-script - namespace: {{ .Values.podSweeper.namespace | default .Release.Namespace }} - labels: - {{- include "airbyte.labels" . | nindent 4 }} - -data: - sweep-pod.sh: | - #!/bin/bash - get_job_pods () { - kubectl -n ${KUBE_NAMESPACE} -L airbyte -l airbyte=job-pod \ - get pods \ - -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime} {.status.startTime}{"\n"}{end}' - } - delete_pod() { - printf "From status '%s' since '%s', " $2 $3 - echo "$1" | grep -v "STATUS" | awk '{print $1}' | xargs --no-run-if-empty kubectl -n ${KUBE_NAMESPACE} delete pod - } - while : - do - echo "Starting pod sweeper cycle:" - - if [ -n "${RUNNING_TTL_MINUTES}" ]; then - # Time window for running pods - RUNNING_DATE_STR=`date -d "now - ${RUNNING_TTL_MINUTES} minutes" --utc -Ins` - RUNNING_DATE=`date -d $RUNNING_DATE_STR +%s` - echo "Will sweep running pods from before ${RUNNING_DATE_STR}" - fi - - if [ -n "${SUCCEEDED_TTL_MINUTES}" ]; then - # Shorter time window for succeeded pods - SUCCESS_DATE_STR=`date -d "now - ${SUCCEEDED_TTL_MINUTES} minutes" --utc -Ins` - SUCCESS_DATE=`date -d $SUCCESS_DATE_STR +%s` - echo "Will sweep succeeded pods from before ${SUCCESS_DATE_STR}" - fi - - if [ -n "${UNSUCCESSFUL_TTL_MINUTES}" ]; then - # Longer time window for unsuccessful pods (to debug) - NON_SUCCESS_DATE_STR=`date -d "now - ${UNSUCCESSFUL_TTL_MINUTES} minutes" --utc -Ins` - NON_SUCCESS_DATE=`date -d $NON_SUCCESS_DATE_STR +%s` - echo "Will sweep unsuccessful pods from before ${NON_SUCCESS_DATE_STR}" - fi - ( - IFS=$'\n' - for POD in `get_job_pods`; do - IFS=' ' - POD_NAME=`echo $POD | cut -d " " -f 1` - POD_STATUS=`echo $POD | cut -d " " -f 2` - POD_DATE_STR=`echo $POD | cut -d " " -f 3` - POD_START_DATE_STR=`echo $POD | cut -d " " -f 4` - POD_DATE=`date -d ${POD_DATE_STR:-$POD_START_DATE_STR} '+%s'` - if [ -n "${RUNNING_TTL_MINUTES}" ] && [ "$POD_STATUS" = "Running" ]; then - if [ "$POD_DATE" -lt "$RUNNING_DATE" ]; then - delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" - fi - elif [ -n "${SUCCEEDED_TTL_MINUTES}" ] && [ "$POD_STATUS" = "Succeeded" ]; then - if [ "$POD_DATE" -lt "$SUCCESS_DATE" ]; then - delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" - fi - elif [ -n "${UNSUCCESSFUL_TTL_MINUTES}" ] && [ "$POD_STATUS" != "Running" ] && [ "$POD_STATUS" != "Succeeded" ]; then - if [ "$POD_DATE" -lt "$NON_SUCCESS_DATE" ]; then - delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" - fi - fi - done - ) - echo "Completed pod sweeper cycle. Sleeping for 60 seconds..." - sleep 60 - done diff --git a/charts/v2/airbyte/templates/airbyte-pod-sweeper/deployment.yaml b/charts/v2/airbyte/templates/airbyte-pod-sweeper/deployment.yaml index b04f7e85b1a..ba008eab907 100644 --- a/charts/v2/airbyte/templates/airbyte-pod-sweeper/deployment.yaml +++ b/charts/v2/airbyte/templates/airbyte-pod-sweeper/deployment.yaml @@ -20,7 +20,6 @@ spec: {{- include "common.tplvalues.render" (dict "value" .Values.podSweeper.podLabels "context" $) | nindent 8 }} {{- end }} annotations: - checksum/sweep-pod-script: {{ include (print $.Template.BasePath "/airbyte-pod-sweeper/configmap.yaml") . | sha256sum }} {{- if .Values.podSweeper.podAnnotations }} {{- include "common.tplvalues.render" (dict "value" .Values.podSweeper.podAnnotations "context" $) | nindent 8 }} {{- end }} @@ -60,9 +59,6 @@ spec: securityContext: {{- toYaml .Values.podSweeper.containerSecurityContext | nindent 10 }} {{- end }} volumeMounts: - - mountPath: /script/sweep-pod.sh - subPath: sweep-pod.sh - name: sweep-pod-script - mountPath: /.kube name: kube-config command: ["/bin/bash", "-c", /script/sweep-pod.sh] @@ -99,7 +95,3 @@ spec: volumes: - name: kube-config emptyDir: {} - - name: sweep-pod-script - configMap: - name: {{ .Release.Name }}-pod-sweeper-sweep-pod-script - defaultMode: 0755 diff --git a/charts/v2/airbyte/templates/config/_jobs.tpl b/charts/v2/airbyte/templates/config/_jobs.tpl index c06335434c1..b2b4a2eee44 100644 --- a/charts/v2/airbyte/templates/config/_jobs.tpl +++ b/charts/v2/airbyte/templates/config/_jobs.tpl @@ -70,60 +70,6 @@ Renders the jobs.kube.localVolume.enabled environment variable key: JOB_KUBE_LOCAL_VOLUME_ENABLED {{- end }} -{{/* -Renders the global.jobs.kube.images.busybox value -*/}} -{{- define "airbyte.jobs.kube.images.busybox" }} - {{- include "imageUrl" (list .Values.global.jobs.kube.images.busybox $) }} -{{- end }} - -{{/* -Renders the jobs.kube.images.busybox environment variable -*/}} -{{- define "airbyte.jobs.kube.images.busybox.env" }} -- name: JOB_KUBE_BUSYBOX_IMAGE - valueFrom: - configMapKeyRef: - name: {{ .Release.Name }}-airbyte-env - key: JOB_KUBE_BUSYBOX_IMAGE -{{- end }} - -{{/* -Renders the global.jobs.kube.images.socat value -*/}} -{{- define "airbyte.jobs.kube.images.socat" }} - {{- include "imageUrl" (list .Values.global.jobs.kube.images.socat $) }} -{{- end }} - -{{/* -Renders the jobs.kube.images.socat environment variable -*/}} -{{- define "airbyte.jobs.kube.images.socat.env" }} -- name: JOB_KUBE_SOCAT_IMAGE - valueFrom: - configMapKeyRef: - name: {{ .Release.Name }}-airbyte-env - key: JOB_KUBE_SOCAT_IMAGE -{{- end }} - -{{/* -Renders the global.jobs.kube.images.curl value -*/}} -{{- define "airbyte.jobs.kube.images.curl" }} - {{- include "imageUrl" (list .Values.global.jobs.kube.images.curl $) }} -{{- end }} - -{{/* -Renders the jobs.kube.images.curl environment variable -*/}} -{{- define "airbyte.jobs.kube.images.curl.env" }} -- name: JOB_KUBE_CURL_IMAGE - valueFrom: - configMapKeyRef: - name: {{ .Release.Name }}-airbyte-env - key: JOB_KUBE_CURL_IMAGE -{{- end }} - {{/* Renders the global.jobs.kube.main_container_image_pull_secret value */}} @@ -255,9 +201,6 @@ Renders the set of all jobs config map variables JOB_KUBE_SERVICEACCOUNT: {{ .Values.global.serviceAccountName | quote }} JOB_KUBE_NAMESPACE: {{ include "airbyte.jobs.kube.namespace" . | quote }} JOB_KUBE_LOCAL_VOLUME_ENABLED: {{ include "airbyte.jobs.kube.localVolume.enabled" . | quote }} -JOB_KUBE_BUSYBOX_IMAGE: {{ include "imageUrl" (list .Values.global.jobs.kube.images.busybox $) | quote }} -JOB_KUBE_SOCAT_IMAGE: {{ include "imageUrl" (list .Values.global.jobs.kube.images.socat $) | quote }} -JOB_KUBE_CURL_IMAGE: {{ include "imageUrl" (list .Values.global.jobs.kube.images.curl $) | quote }} JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET: {{ include "airbyte.jobs.kube.main_container_image_pull_secret" . | quote }} JOB_KUBE_ANNOTATIONS: {{ .Values.global.jobs.kube.annotations | include "airbyte.flattenMap" | quote }} JOB_KUBE_LABELS: {{ .Values.global.jobs.kube.labels | include "airbyte.flattenMap" | quote }} diff --git a/charts/v2/airbyte/templates/minio.yaml b/charts/v2/airbyte/templates/minio.yaml index 0f9e9835ae0..68edf24e11c 100644 --- a/charts/v2/airbyte/templates/minio.yaml +++ b/charts/v2/airbyte/templates/minio.yaml @@ -123,13 +123,13 @@ spec: fsGroup: 1000 containers: - name: minio-mc - image: {{ include "imageUrl" (list .Values.minio.mcImage $) }} + image: {{ include "imageUrl" (list .Values.minio.image $) }} command: ["/bin/sh", "-c", - "until (/usr/bin/mc config host add myminio $MINIO_ENDPOINT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY) do echo '...waiting...' && sleep 1; done; - /usr/bin/mc mb --ignore-existing myminio/state-storage; - /usr/bin/mc policy set public myminio/state-storage; - /usr/bin/mc mb --ignore-existing myminio/airbyte-dev-logs; - /usr/bin/mc policy set public myminio/airbyte-dev-logs;"] + "until (/bin/mc config host add myminio $MINIO_ENDPOINT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY) do echo '...waiting...' && sleep 1; done; + /bin/mc mb --ignore-existing myminio/state-storage; + /bin/mc policy set public myminio/state-storage; + /bin/mc mb --ignore-existing myminio/airbyte-dev-logs; + /bin/mc policy set public myminio/airbyte-dev-logs;"] securityContext: allowPrivilegeEscalation: false runAsNonRoot: true diff --git a/charts/v2/airbyte/templates/tests/test-webapp.yaml b/charts/v2/airbyte/templates/tests/test-webapp.yaml index 7d4f0f5f18b..35a8e648664 100644 --- a/charts/v2/airbyte/templates/tests/test-webapp.yaml +++ b/charts/v2/airbyte/templates/tests/test-webapp.yaml @@ -15,10 +15,10 @@ spec: {{- end }} {{- end }} containers: - - name: wget + - name: webapp-test image: {{ include "imageUrl" (list .Values.testWebapp.image $) }} - command: ['wget'] - args: ['{{ .Release.Name }}-airbyte-webapp-svc:{{ .Values.webapp.service.port }}'] + command: ['curl'] + args: ['-s', '{{ .Release.Name }}-airbyte-webapp-svc:{{ .Values.webapp.service.port }}', '>', '/dev/null'] resources: requests: memory: "64Mi" diff --git a/charts/v2/airbyte/values.yaml b/charts/v2/airbyte/values.yaml index 3a0a174d47d..276d4e1bf83 100644 --- a/charts/v2/airbyte/values.yaml +++ b/charts/v2/airbyte/values.yaml @@ -25,7 +25,7 @@ global: # Docker registry to pull platform images from, e.g. http://my-registry:8000/ registry: "" # Image tag to use for airbyte images. - # Does not include non-airbyte images such as busybox, temporal, minio, etc. + # Does not include non-airbyte images such as temporal, minio, etc. tag: "" # Docker image pull secret @@ -336,14 +336,6 @@ global: localVolume: enabled: false - images: - ## JOB_KUBE_BUSYBOX_IMAGE - # -- busybox image used by the job pod - busybox: "busybox:1.35" - ## JOB_KUBE_CURL_IMAGE - # -- curl image used by the job pod - curl: "curlimages/curl:8.1.1" - topology: nodeSelectorLabel: "airbyte/node-pool" nodeSelectors: @@ -682,11 +674,9 @@ podSweeper: enabled: true image: # -- The image repository to use for the pod sweeper - repository: bitnami/kubectl + repository: airbyte/pod-sweeper # -- The pull policy for the pod sweeper image pullPolicy: IfNotPresent - # -- The pod sweeper image tag to use - tag: 1.28.9 replicaCount: 1 @@ -2037,10 +2027,6 @@ minio: rootUser: "" rootPassword: "" - mcImage: - repository: airbyte/mc - tag: latest - storage: volumeClaimValue: 500Mi @@ -2506,10 +2492,6 @@ keycloakSetup: seccompProfile: type: RuntimeDefault - initContainers: - keycloakReadinessCheck: - image: "curlimages/curl:8.1.1" - containerSecurityContext: allowPrivilegeEscalation: false runAsNonRoot: true @@ -2790,5 +2772,4 @@ featureflagServer: testWebapp: image: - repository: busybox - tag: latest + repository: airbyte/utils diff --git a/docker/Makefile b/docker/Makefile index 12d9c15290b..899cf2e9276 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -7,9 +7,8 @@ OS ?= $(shell uname | tr '[:upper:]' '[:lower:]') PUBLISH ?= false BASE_JAVA_IMAGE_VERSION ?= 3.3.1 +NGINX_VERSION = 1.27.3 BASE_JAVA_IMAGE = airbyte/airbyte-base-java-image:$(BASE_JAVA_IMAGE_VERSION) -BASE_BUSYBOX_IMAGE_VERSION ?= 1.28 -ALPINE_IMAGE_VERSION ?= 3.18 image.base-images: ## Build all the base images image.base-images: buildx.start @@ -22,9 +21,9 @@ image.airbyte-base-java-image: buildx.start --builder airbyte-image-builder \ --platform linux/amd64,linux/arm64 \ --push \ - -f $(IMAGES_DIR)/airbyte-base-java-image/Dockerfile . ; \ + $(IMAGES_DIR)/airbyte-base-java-image ; \ else \ - docker build -t airbyte/airbyte-base-java-image:$(VERSION) -f $(IMAGES_DIR)/airbyte-base-java-image/Dockerfile . ; \ + docker build -t airbyte/airbyte-base-java-image:$(VERSION) $(IMAGES_DIR)/airbyte-base-java-image ; \ fi image.airbyte-base-java-python-image: ## Build the airbyte/airbyte-base-java-python-image @@ -35,11 +34,11 @@ image.airbyte-base-java-python-image: buildx.start --build-arg JDK_IMAGE=$(BASE_JAVA_IMAGE) \ --platform linux/amd64,linux/arm64 \ --push \ - -f $(IMAGES_DIR)/airbyte-base-java-python-image/Dockerfile . ; \ + $(IMAGES_DIR)/airbyte-base-java-python-image ; \ else \ docker build -t airbyte/airbyte-base-java-python-image:$(VERSION) \ --build-arg JDK_IMAGE=$(BASE_JAVA_IMAGE) \ - -f $(IMAGES_DIR)/airbyte-base-java-python-image/Dockerfile . ; \ + $(IMAGES_DIR)/airbyte-base-java-python-image ; \ fi image.airbyte-base-java-worker-image: ## Build the airbyte/airbyte-base-java-worker-image @@ -49,71 +48,25 @@ image.airbyte-base-java-worker-image: buildx.start --build-arg JDK_IMAGE=$(BASE_JAVA_IMAGE) \ --platform linux/amd64,linux/arm64 \ --push \ - -f $(IMAGES_DIR)/airbyte-base-java-worker-image/Dockerfile . ; \ + $(IMAGES_DIR)/airbyte-base-java-worker-image ; \ else \ docker build -t airbyte/airbyte-base-java-worker-image:$(VERSION) \ --build-arg JDK_IMAGE=$(BASE_JAVA_IMAGE) \ - -f $(IMAGES_DIR)/airbyte-base-java-worker-image/Dockerfile . ; \ - fi - -image.airbyte-busybox: ## Build the airbyte/busybox image -image.airbyte-busybox: buildx.start - @if [ "$(PUBLISH)" = "true" ]; then \ - docker buildx build -t airbyte/busybox:$(VERSION) \ - --build-arg BASE_BUSYBOX_IMAGE_VERSION=$(BASE_BUSYBOX_IMAGE_VERSION) \ - --platform linux/amd64,linux/arm64 \ - --push \ - -f $(IMAGES_DIR)/airbyte-busybox/Dockerfile . ; \ - else \ - docker build -t airbyte/busybox:$(VERSION) \ - --build-arg BASE_BUSYBOX_IMAGE_VERSION=$(BASE_BUSYBOX_IMAGE_VERSION) \ - -f $(IMAGES_DIR)/airbyte-busybox/Dockerfile . ; \ - fi - -image.airbyte-mc: ## Build the airbyte/mc image -image.airbyte-mc: buildx.start - @if [ "$(PUBLISH)" = "true" ]; then \ - docker buildx build -t airbyte/mc:$(VERSION) \ - --build-arg ALPINE_IMAGE_VERSION=$(ALPINE_IMAGE_VERSION) \ - --platform linux/amd64,linux/arm64 \ - --push \ - -f $(IMAGES_DIR)/airbyte-mc/Dockerfile . ; \ - else \ - docker build -t airbyte/mc:$(VERSION) \ - --build-arg ALPINE_IMAGE_VERSION=$(ALPINE_IMAGE_VERSION) \ - -f $(IMAGES_DIR)/airbyte-mc/Dockerfile . ; \ + $(IMAGES_DIR)/airbyte-base-java-worker-image ; \ fi image.airbyte-nginx-slim: ## Build the airbyte/nginx-unprivileged image image.airbyte-nginx-slim: buildx.start @if [ "$(PUBLISH)" = "true" ]; then \ - docker buildx build -t airbyte/nginx-alpine-slim:$(VERSION) \ - --build-arg UID="1000" \ - --build-arg GID="1000" \ - --platform linux/amd64,linux/arm64 \ - --push \ - -f $(IMAGES_DIR)/airbyte-nginx/alpine-slim/Dockerfile $(IMAGES_DIR)/airbyte-nginx/alpine-slim/. ; \ - else \ - docker build -t airbyte/nginx-alpine-slim:$(VERSION) \ - --build-arg UID="1000" \ - --build-arg GID="1000" \ - -f $(IMAGES_DIR)/airbyte-nginx/alpine-slim/Dockerfile $(IMAGES_DIR)/airbyte-nginx/alpine-slim/. ; \ - fi - -image.airbyte-nginx-unprivileged: ## Build the airbyte/nginx-unprivileged image -image.airbyte-nginx-unprivileged: buildx.start - @if [ "$(PUBLISH)" = "true" ]; then \ - docker buildx build -t airbyte/nginx-unprivileged:$(VERSION) \ - --build-arg IMAGE="airbyte/nginx-alpine-slim:1.27.2-alpine-slim-2" \ + docker buildx build -t airbyte/nginx-alpine-slim:$(NGINX_VERSION) \ --build-arg UID="1000" \ --build-arg GID="1000" \ --platform linux/amd64,linux/arm64 \ --push \ - -f $(IMAGES_DIR)/airbyte-nginx/unprivileged/Dockerfile $(IMAGES_DIR)/airbyte-nginx/unprivileged/. ; \ + https://github.com/nginxinc/docker-nginx-unprivileged.git\#$(NGINX_VERSION):mainline/alpine-slim ; \ else \ - docker build -t airbyte/nginx-unprivileged:$(VERSION) \ - --build-arg IMAGE="airbyte/nginx-alpine-slim:1.27.2-alpine-slim-2" \ + docker build -t airbyte/nginx-alpine-slim:$(NGINX_VERSION) \ --build-arg UID="1000" \ --build-arg GID="1000" \ - -f $(IMAGES_DIR)/airbyte-nginx/unprivileged/Dockerfile $(IMAGES_DIR)/airbyte-nginx/unprivileged/. ; \ + https://github.com/nginxinc/docker-nginx-unprivileged.git\#$(NGINX_VERSION):mainline/alpine-slim ; \ fi diff --git a/docker/airbyte-base-java-image/Dockerfile b/docker/airbyte-base-java-image/Dockerfile index 36e0f484881..0f6f9297451 100644 --- a/docker/airbyte-base-java-image/Dockerfile +++ b/docker/airbyte-base-java-image/Dockerfile @@ -25,7 +25,7 @@ ADD --chown=airbyte:airbyte https://dtdg.co/latest-java-tracer dd-java-agent.jar # Add the OpenTelemetry Java APM agent ADD --chown=airbyte:airbyte https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/latest/download/opentelemetry-javaagent.jar opentelemetry-javaagent.jar -ADD --chown=airbyte:airbyte oss/docker/airbyte-base-java-image/entrypoint.sh /entrypoint.sh +ADD --chown=airbyte:airbyte entrypoint.sh /entrypoint.sh USER airbyte:airbyte diff --git a/docker/airbyte-busybox/Dockerfile b/docker/airbyte-busybox/Dockerfile deleted file mode 100644 index 7ceda929f41..00000000000 --- a/docker/airbyte-busybox/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -ARG BUSYBOX_IMAGE_VERSION=1.28 -FROM busybox:${BUSYBOX_IMAGE_VERSION} - -RUN adduser -u 1000 -s /bin/sh -D airbyte - -USER airbyte:airbyte diff --git a/docker/airbyte-mc/Dockerfile b/docker/airbyte-mc/Dockerfile deleted file mode 100644 index b3ab9430d04..00000000000 --- a/docker/airbyte-mc/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -ARG ALPINE_IMAGE_VERSION=3.18 -FROM alpine:${ALPINE_IMAGE_VERSION} - -RUN adduser -u 1000 -s /bin/sh -D airbyte - -RUN apkArch="$(apk --print-arch)"; \ - case "$apkArch" in \ - aarch64) export PLATFORM='arm64' ;; \ - *) export PLATFORM='amd64' ;; \ - esac; \ - wget https://dl.min.io/client/mc/release/linux-${PLATFORM}/mc && \ - mv mc /usr/bin/mc && \ - chmod +x /usr/bin/mc - -USER airbyte:airbyte diff --git a/settings.gradle.kts b/settings.gradle.kts index 2f116a6c375..fce494fe845 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -127,6 +127,7 @@ include(":oss:airbyte-keycloak") include(":oss:airbyte-keycloak-setup") include(":oss:airbyte-mappers") include(":oss:airbyte-metrics:reporter") +include(":oss:airbyte-pod-sweeper") include(":oss:airbyte-server") include(":oss:airbyte-temporal") include(":oss:airbyte-tests") @@ -190,6 +191,7 @@ project(":oss:airbyte-keycloak").projectDir = file("airbyte-keycloak") project(":oss:airbyte-keycloak-setup").projectDir = file("airbyte-keycloak-setup") project(":oss:airbyte-mappers").projectDir = file("airbyte-mappers") project(":oss:airbyte-metrics:reporter").projectDir = file("airbyte-metrics/reporter") +project(":oss:airbyte-pod-sweeper").projectDir = file("airbyte-pod-sweeper") project(":oss:airbyte-server").projectDir = file("airbyte-server") project(":oss:airbyte-temporal").projectDir = file("airbyte-temporal") project(":oss:airbyte-tests").projectDir = file("airbyte-tests")