chore(helm): add logging and storage templates (#13807)

Co-authored-by: perangel <[email protected]>
airbytehq · Sep 5, 2024 · 2531253 · 2531253
1 parent 0c753eb
commit 2531253
Show file tree

Hide file tree

Showing 12 changed files with 279 additions and 341 deletions.
diff --git a/charts/airbyte-server/templates/deployment.yaml b/charts/airbyte-server/templates/deployment.yaml
@@ -74,6 +74,7 @@ spec:
           value: "true"
         {{- end }}
         {{- if eq .Values.global.deploymentMode "oss"  }}
+        {{- include "airbyte.logging.envs" . | nindent 8 }}
         - name: AIRBYTE_API_HOST
           valueFrom:
             configMapKeyRef:
@@ -99,11 +100,6 @@ spec:
             configMapKeyRef:
               name: {{ .Release.Name }}-airbyte-env
               key: CONFIG_ROOT
-        - name: LOG4J_CONFIGURATION_FILE
-          valueFrom:
-            configMapKeyRef:
-              name: {{ .Release.Name }}-airbyte-env
-              key: LOG4J_CONFIGURATION_FILE
         - name: MICROMETER_METRICS_ENABLED
           valueFrom:
             configMapKeyRef:
@@ -159,8 +155,6 @@ spec:
             configMapKeyRef:
               name: {{ .Release.Name }}-airbyte-env
               key: TEMPORAL_HOST
-        - name: LOG_LEVEL
-          value: "{{ .Values.log.level }}"
         - name: JOB_MAIN_CONTAINER_CPU_REQUEST
           valueFrom:
             configMapKeyRef:
@@ -256,7 +250,7 @@ spec:
           value: {{ .Values.global.auth.cookieSameSiteSetting | default "Strict" | quote }}
         {{- end }}
 
-        # SECRETS MANAGER
+        # Secrets Manager
         - name: SECRET_PERSISTENCE
           value: {{ include "airbyte.secretPersistence" . }}
         # Values for AwsSecretsManager
@@ -276,7 +270,6 @@ spec:
         - name: AWS_SECRET_MANAGER_REGION
           value: {{ (((.Values.global).secretsManager).awsSecretManager).region }}
         {{- end }}
-
         {{- if ((((.Values.global).secretsManager).awsSecretManager).tags) }}
         - name: AWS_SECRET_MANAGER_SECRET_TAGS
           value: {{ include "airbyte.tagsToString" .Values.global.secretsManager.awsSecretManager.tags }}
@@ -310,107 +303,8 @@ spec:
               key: {{ include "airbyte.vaultAuthTokenSecretKey" .Values.global.secretsManager.vault.authTokenSecretKey }}
         {{- end }}
 
-        # STORAGE
-        - name: STORAGE_TYPE
-          valueFrom:
-            configMapKeyRef:
-              name: {{ .Release.Name }}-airbyte-env
-              key: STORAGE_TYPE
-        - name: STORAGE_BUCKET_ACTIVITY_PAYLOAD
-          valueFrom:
-            configMapKeyRef:
-              name: {{ .Release.Name }}-airbyte-env
-              key: STORAGE_BUCKET_ACTIVITY_PAYLOAD
-        - name: STORAGE_BUCKET_LOG
-          valueFrom:
-            configMapKeyRef:
-              name: {{ .Release.Name }}-airbyte-env
-              key: STORAGE_BUCKET_LOG
-        - name: STORAGE_BUCKET_STATE
-          valueFrom:
-            configMapKeyRef:
-              name: {{ .Release.Name }}-airbyte-env
-              key: STORAGE_BUCKET_STATE
-        - name: STORAGE_BUCKET_WORKLOAD_OUTPUT
-          valueFrom:
-            configMapKeyRef:
-              name: {{ .Release.Name }}-airbyte-env
-              key: STORAGE_BUCKET_WORKLOAD_OUTPUT
-
-        # local
-        {{- if eq (lower (default "" .Values.global.storage.type)) "local" }}
-        - name: LOCAL_ROOT
-          valueFrom:
-            configMapKeyRef:
-              name: {{ .Release.Name }}-airbyte-env
-              key: LOCAL_ROOT
-        {{- end }}
-
-        # S3
-        {{- if eq ((((.Values.global).storage).s3).authenticationType) "credentials" }}
-        - name: AWS_ACCESS_KEY_ID # todo: change to S3_ACCESS_KEY_ID, changes need to be pushed to the applications.yaml files
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "airbyte.secretStoreName" .Values.global.storage.storageSecretName }}
-              key: {{ include "airbyte.s3AccessKeyIdSecretKey" .Values.global.storage.s3.accessKeyIdSecretKey }}
-        - name: AWS_SECRET_ACCESS_KEY # todo: change to S3_SECRET_ACCESS_KEY, changes need to be pushed to the applications.yaml files
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "airbyte.secretStoreName" .Values.global.storage.storageSecretName }}
-              key: {{ include "airbyte.s3SecretAccessKeySecretKey" .Values.global.storage.s3.secretAccessKeySecretKey }}
-        {{- end }}
-        {{- if ((((.Values.global).storage).s3).region) }}
-        - name: AWS_DEFAULT_REGION # todo: change to S3_DEFAULT_REGION, changes need to be pushed to the applications.yaml files
-          valueFrom:
-            configMapKeyRef:
-              name: {{ .Release.Name }}-airbyte-env
-              key: AWS_DEFAULT_REGION
-        {{- end }}
-        - name: S3_PATH_STYLE_ACCESS
-          valueFrom:
-            configMapKeyRef:
-              name: {{ .Release.Name }}-airbyte-env
-              key: S3_PATH_STYLE_ACCESS
-
-        # MINIO
-        {{- if and (eq (lower (default "" .Values.global.storage.type)) "minio") (((.Values.global).storage).minio)}}
-        - name: AWS_ACCESS_KEY_ID # todo: change to MINIO_ACCESS_KEY_ID, changes need to be pushed to the applications.yaml files
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "airbyte.secretStoreName" .Values.global.storage.storageSecretName }}
-              key: {{ include "airbyte.minioAccessKeyIdSecretKey" .Values.global.storage.minio.accessKeyIdSecretKey }}
-        - name: AWS_SECRET_ACCESS_KEY # todo: change to MINIO_SECRET_ACCESS_KEY, changes need to be pushed to the applications.yaml files
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "airbyte.secretStoreName" .Values.global.storage.storageSecretName }}
-              key: {{ include "airbyte.minioSecretAccessKeySecretKey" .Values.global.storage.minio.secretAccessKeySecretKey }}
-        {{- else if (ne (lower (default "" .Values.global.storage.type)) "s3") }} # default behavior if none supplied is internal minio deployment
-        # this is for the internally deployed minio
-        - name: AWS_ACCESS_KEY_ID # todo: change to MINIO_ACCESS_KEY_ID, changes need to be pushed to the applications.yaml files
-          valueFrom:
-            secretKeyRef:
-              name: {{ .Release.Name }}-airbyte-secrets
-              key: DEFAULT_MINIO_ACCESS_KEY
-        - name: AWS_SECRET_ACCESS_KEY # todo: change to MINIO_SECRET_ACCESS_KEY, changes need to be pushed to the applications.yaml files
-          valueFrom:
-            secretKeyRef:
-              name: {{ .Release.Name }}-airbyte-secrets
-              key: DEFAULT_MINIO_SECRET_KEY
-        {{- end }}
-        - name: MINIO_ENDPOINT
-          valueFrom:
-            configMapKeyRef:
-              name: {{ .Release.Name }}-airbyte-env
-              key: MINIO_ENDPOINT
-
-        #GCS
-        {{- if eq (lower (default "" .Values.global.storage.type)) "gcs" }}
-        - name: GOOGLE_APPLICATION_CREDENTIALS
-          valueFrom:
-            configMapKeyRef:
-              name: {{ .Release.Name }}-airbyte-env
-              key: GOOGLE_APPLICATION_CREDENTIALS
-        {{- end }}
+        # Storage
+        {{- include "airbyte.storage.envs" . | nindent 8 }}
 
         {{- include "airbyte.database.envs" . | nindent 8 }}
         {{- include "airbyte.enterprise.instanceAdmin" . | nindent 8 }} 

diff --git a/charts/airbyte-worker/templates/deployment.yaml b/charts/airbyte-worker/templates/deployment.yaml
@@ -77,11 +77,7 @@ spec:
         - name: CONTAINER_ORCHESTRATOR_SECRET_NAME
           value: {{ include "airbyte.secretStoreName" .Values.global.storage.storageSecretName }}
         {{- end }}
-        - name: LOG4J_CONFIGURATION_FILE
-          valueFrom:
-            configMapKeyRef:
-              name: {{ .Release.Name }}-airbyte-env
-              key: LOG4J_CONFIGURATION_FILE
+        {{- include "airbyte.logging.envs" . | nindent 8 }}
         - name: MICROMETER_METRICS_ENABLED
           valueFrom:
             configMapKeyRef:
@@ -139,8 +135,6 @@ spec:
             configMapKeyRef:
               name: {{ .Release.Name }}-airbyte-env
               key: TEMPORAL_WORKER_PORTS
-        - name: LOG_LEVEL
-          value: "{{ .Values.log.level }}"
         - name: JOB_KUBE_NAMESPACE
           valueFrom:
             fieldRef:
@@ -365,106 +359,8 @@ spec:
               key: {{ include "airbyte.vaultAuthTokenSecretKey" .Values.global.secretsManager.vault.authTokenSecretKey }}
         {{- end }}
 
-        # STORAGE
-        - name: STORAGE_TYPE
-          valueFrom:
-            configMapKeyRef:
-              name: {{ .Release.Name }}-airbyte-env
-              key: STORAGE_TYPE
-        - name: STORAGE_BUCKET_ACTIVITY_PAYLOAD
-          valueFrom:
-            configMapKeyRef:
-              name: {{ .Release.Name }}-airbyte-env
-              key: STORAGE_BUCKET_ACTIVITY_PAYLOAD
-        - name: STORAGE_BUCKET_LOG
-          valueFrom:
-            configMapKeyRef:
-              name: {{ .Release.Name }}-airbyte-env
-              key: STORAGE_BUCKET_LOG
-        - name: STORAGE_BUCKET_STATE
-          valueFrom:
-            configMapKeyRef:
-              name: {{ .Release.Name }}-airbyte-env
-              key: STORAGE_BUCKET_STATE
-        - name: STORAGE_BUCKET_WORKLOAD_OUTPUT
-          valueFrom:
-            configMapKeyRef:
-              name: {{ .Release.Name }}-airbyte-env
-              key: STORAGE_BUCKET_WORKLOAD_OUTPUT
-
-        # local
-        {{- if eq (lower (default "" .Values.global.storage.type)) "local" }}
-        - name: LOCAL_ROOT
-          valueFrom:
-              name: {{ .Release.Name }}-airbyte-env
-              key: LOCAL_ROOT
-        {{- end }}
-
-        # S3
-        {{- if eq ((((.Values.global).storage).s3).authenticationType) "credentials" }}
-        - name: AWS_ACCESS_KEY_ID # todo: change to S3_ACCESS_KEY_ID, changes need to be pushed to the applications.yaml files
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "airbyte.secretStoreName" .Values.global.storage.storageSecretName }}
-              key: {{ include "airbyte.s3AccessKeyIdSecretKey" .Values.global.storage.s3.accessKeyIdSecretKey }}
-        - name: AWS_SECRET_ACCESS_KEY # todo: change to S3_SECRET_ACCESS_KEY, changes need to be pushed to the applications.yaml files
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "airbyte.secretStoreName" .Values.global.storage.storageSecretName }}
-              key: {{ include "airbyte.s3SecretAccessKeySecretKey" .Values.global.storage.s3.secretAccessKeySecretKey }}
-        {{- end }}
-        {{- if ((((.Values.global).storage).s3).region) }}
-        - name: AWS_DEFAULT_REGION # todo: change to S3_DEFAULT_REGION, changes need to be pushed to the applications.yaml files
-          valueFrom:
-            configMapKeyRef:
-              name: {{ .Release.Name }}-airbyte-env
-              key: AWS_DEFAULT_REGION
-        {{- end }}
-        - name: S3_PATH_STYLE_ACCESS
-          valueFrom:
-            configMapKeyRef:
-              name: {{ .Release.Name }}-airbyte-env
-              key: S3_PATH_STYLE_ACCESS
-
-        # MINIO
-        {{- if and (eq (lower (default "" .Values.global.storage.type)) "minio") (((.Values.global).storage).minio)}}
-        - name: AWS_ACCESS_KEY_ID # todo: change to MINIO_ACCESS_KEY_ID, changes need to be pushed to the applications.yaml files
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "airbyte.secretStoreName" .Values.global.storage.storageSecretName }}
-              key: {{ include "airbyte.minioAccessKeyIdSecretKey" .Values.global.storage.minio.accessKeyIdSecretKey }}
-        - name: AWS_SECRET_ACCESS_KEY # todo: change to MINIO_SECRET_ACCESS_KEY, changes need to be pushed to the applications.yaml files
-          valueFrom:
-            secretKeyRef:
-              name: {{ include "airbyte.secretStoreName" .Values.global.storage.storageSecretName }}
-              key: {{ include "airbyte.minioSecretAccessKeySecretKey" .Values.global.storage.minio.secretAccessKeySecretKey }}
-        {{- else if (ne (lower (default "" .Values.global.storage.type)) "s3") }} # default behavior if none supplied is internal minio deployment
-        # this is for the internally deployed minio
-        - name: AWS_ACCESS_KEY_ID # todo: change to MINIO_ACCESS_KEY_ID, changes need to be pushed to the applications.yaml files
-          valueFrom:
-            secretKeyRef:
-              name: {{ .Release.Name }}-airbyte-secrets
-              key: DEFAULT_MINIO_ACCESS_KEY
-        - name: AWS_SECRET_ACCESS_KEY # todo: change to MINIO_SECRET_ACCESS_KEY, changes need to be pushed to the applications.yaml files
-          valueFrom:
-            secretKeyRef:
-              name: {{ .Release.Name }}-airbyte-secrets
-              key: DEFAULT_MINIO_SECRET_KEY
-        {{- end }}
-        - name: MINIO_ENDPOINT
-          valueFrom:
-            configMapKeyRef:
-              name: {{ .Release.Name }}-airbyte-env
-              key: MINIO_ENDPOINT
-
-        #GCS
-        {{- if eq (lower (default "" .Values.global.storage.type)) "gcs" }}
-        - name: GOOGLE_APPLICATION_CREDENTIALS
-          valueFrom:
-            configMapKeyRef:
-              name: {{ .Release.Name }}-airbyte-env
-              key: GOOGLE_APPLICATION_CREDENTIALS
-        {{- end }}
+        # Storage
+        {{- include "airbyte.storage.envs" . | nindent 8 }}
 
         # Database
         {{- include "airbyte.database.envs" . | nindent 8 }}
@@ -490,7 +386,7 @@ spec:
         {{- end }}
         {{- end }}
 
-        # Values from extraEnv for more compability(if you want to use external secret source or other stuff)
+        # Values from extraEnv for more compatibility (if you want to use external secret source or other stuff)
         {{- if .Values.extraEnv }}
         {{- toYaml .Values.extraEnv | nindent 8 }}
         {{- end }}