Skip to content

Commit

Permalink
Swap internal RPC server for API server in the helm chart (apache#44463)
Browse files Browse the repository at this point in the history 
* Swap internal RPC server for API server in the helm chart

Previously the PRC server was possible to disable/enable, where as now the new
api-server is a required component in Airflow 3.0+

This also does a little bit of drive-by refactoring of the helper templates
for service account name generation

* Mark the LocalExecutor test that uses example_xcom dag as xfail for now

We merged a change to swap the LocalExecutor over to the new AIP-72 TaskSDK
code, and it wasn't complete enough to work with xcom. To unblock main we are
temporarirly marking that test as xfail, with a link to the issue to fix it.
  • Loading branch information
@ashb
ashb authored Nov 29, 2024
1 parent 35000c9 commit 288c87d
Show file tree
Hide file tree
Showing 16 changed files with 417 additions and 675 deletions.
92 changes: 25 additions & 67 deletions chart/templates/_helpers.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -590,122 +590,80 @@ server_tls_key_file = /etc/pgbouncer/server.key
{{- end }}
{{- end }}

{{/* Create the name of the webserver service account to use */}}
{{- define "webserver.serviceAccountName" -}}
{{- if .Values.webserver.serviceAccount.create }}
{{- default (printf "%s-webserver" (include "airflow.serviceAccountName" .)) .Values.webserver.serviceAccount.name }}
{{/* Helper to generate service account name respecting .Values.$section.serviceAccount flags */}}
{{- define "_serviceAccountName" -}}
{{- $sa := get (get .Values .key) "serviceAccount" }}
{{- if $sa.create }}
{{- default (printf "%s-%s" (include "airflow.serviceAccountName" .) (default .key .nameSuffix )) $sa.name | quote }}
{{- else }}
{{- default "default" .Values.webserver.serviceAccount.name }}
{{- default "default" $sa.name | quote }}
{{- end }}
{{- end }}

{{/* Create the name of the webserver service account to use */}}
{{- define "webserver.serviceAccountName" -}}
{{- include "_serviceAccountName" (merge (dict "key" "webserver") .) -}}
{{- end }}


{{/* Create the name of the RPC server service account to use */}}
{{- define "rpcServer.serviceAccountName" -}}
{{- if .Values._rpcServer.serviceAccount.create }}
{{- default (printf "%s-rpc-server" (include "airflow.serviceAccountName" .)) .Values._rpcServer.serviceAccount.name }}
{{- else }}
{{- default "default" .Values._rpcServer.serviceAccount.name }}
{{- end }}
{{/* Create the name of the API server service account to use */}}
{{- define "apiServer.serviceAccountName" -}}
{{- include "_serviceAccountName" (merge (dict "key" "apiServer" "nameSuffix" "api-server" ) .) -}}
{{- end }}

{{/* Create the name of the redis service account to use */}}
{{- define "redis.serviceAccountName" -}}
{{- if .Values.redis.serviceAccount.create }}
{{- default (printf "%s-redis" (include "airflow.serviceAccountName" .)) .Values.redis.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.redis.serviceAccount.name }}
{{- end }}
{{- include "_serviceAccountName" (merge (dict "key" "redis") .) -}}
{{- end }}

{{/* Create the name of the flower service account to use */}}
{{- define "flower.serviceAccountName" -}}
{{- if .Values.flower.serviceAccount.create }}
{{- default (printf "%s-flower" (include "airflow.serviceAccountName" .)) .Values.flower.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.flower.serviceAccount.name }}
{{- end }}
{{- include "_serviceAccountName" (merge (dict "key" "flower") .) -}}
{{- end }}

{{/* Create the name of the scheduler service account to use */}}
{{- define "scheduler.serviceAccountName" -}}
{{- if .Values.scheduler.serviceAccount.create }}
{{- default (printf "%s-scheduler" (include "airflow.serviceAccountName" .)) .Values.scheduler.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.scheduler.serviceAccount.name }}
{{- end }}
{{- include "_serviceAccountName" (merge (dict "key" "scheduler") .) -}}
{{- end }}

{{/* Create the name of the StatsD service account to use */}}
{{- define "statsd.serviceAccountName" -}}
{{- if .Values.statsd.serviceAccount.create }}
{{- default (printf "%s-statsd" (include "airflow.serviceAccountName" .)) .Values.statsd.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.statsd.serviceAccount.name }}
{{- end }}
{{- include "_serviceAccountName" (merge (dict "key" "statsd") .) -}}
{{- end }}

{{/* Create the name of the create user job service account to use */}}
{{- define "createUserJob.serviceAccountName" -}}
{{- if .Values.createUserJob.serviceAccount.create }}
{{- default (printf "%s-create-user-job" (include "airflow.serviceAccountName" .)) .Values.createUserJob.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.createUserJob.serviceAccount.name }}
{{- end }}
{{- include "_serviceAccountName" (merge (dict "key" "createUserJob" "nameSuffix" "create-user-job") .) -}}
{{- end }}

{{/* Create the name of the migrate database job service account to use */}}
{{- define "migrateDatabaseJob.serviceAccountName" -}}
{{- if .Values.migrateDatabaseJob.serviceAccount.create }}
{{- default (printf "%s-migrate-database-job" (include "airflow.serviceAccountName" .)) .Values.migrateDatabaseJob.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.migrateDatabaseJob.serviceAccount.name }}
{{- end }}
{{- include "_serviceAccountName" (merge (dict "key" "migrateDatabaseJob" "nameSuffix" "migrate-database-job") .) -}}
{{- end }}

{{/* Create the name of the worker service account to use */}}
{{- define "worker.serviceAccountName" -}}
{{- if .Values.workers.serviceAccount.create }}
{{- default (printf "%s-worker" (include "airflow.serviceAccountName" .)) .Values.workers.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.workers.serviceAccount.name }}
{{- end }}
{{- include "_serviceAccountName" (merge (dict "key" "workers" "nameSuffix" "worker") .) -}}
{{- end }}

{{/* Create the name of the triggerer service account to use */}}
{{- define "triggerer.serviceAccountName" -}}
{{- if .Values.triggerer.serviceAccount.create }}
{{- default (printf "%s-triggerer" (include "airflow.serviceAccountName" .)) .Values.triggerer.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.triggerer.serviceAccount.name }}
{{- end }}
{{- include "_serviceAccountName" (merge (dict "key" "triggerer") .) -}}
{{- end }}

{{/* Create the name of the dag processor service account to use */}}
{{- define "dagProcessor.serviceAccountName" -}}
{{- if .Values.dagProcessor.serviceAccount.create }}
{{- default (printf "%s-dag-processor" (include "airflow.serviceAccountName" .)) .Values.dagProcessor.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.dagProcessor.serviceAccount.name }}
{{- end }}
{{- include "_serviceAccountName" (merge (dict "key" "dagProcessor" "nameSuffix" "dag-processor") .) -}}
{{- end }}

{{/* Create the name of the pgbouncer service account to use */}}
{{- define "pgbouncer.serviceAccountName" -}}
{{- if .Values.pgbouncer.serviceAccount.create }}
{{- default (printf "%s-pgbouncer" (include "airflow.serviceAccountName" .)) .Values.pgbouncer.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.pgbouncer.serviceAccount.name }}
{{- end }}
{{- include "_serviceAccountName" (merge (dict "key" "pgbouncer") .) -}}
{{- end }}

{{/* Create the name of the cleanup service account to use */}}
{{- define "cleanup.serviceAccountName" -}}
{{- if .Values.cleanup.serviceAccount.create }}
{{- default (printf "%s-cleanup" (include "airflow.serviceAccountName" .)) .Values.cleanup.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.cleanup.serviceAccount.name }}
{{- end }}
{{- include "_serviceAccountName" (merge (dict "key" "cleanup") .) -}}
{{- end }}

{{- define "wait-for-migrations-command" -}}
Expand Down
Loading

0 comments on commit 288c87d

Please sign in to comment.