diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 129ebe39..9c2b1bf8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,8 +10,6 @@ jobs: # digests should be appended in image versions according to the KIND release notes # https://github.com/kubernetes-sigs/kind/releases/tag/v0.14.0 kindest-image-tag: - - 'v1.16.15' - - 'v1.17.17' - 'v1.18.20@sha256:738cdc23ed4be6cc0b7ea277a2ebcc454c8373d7d8fb991a7fcdbd126188e6d7' - 'v1.19.16@sha256:a146f9819fece706b337d34125bbd5cb8ae4d25558427bf2fa3ee8ad231236f2' - 'v1.20.15@sha256:45d0194a8069c46483a0e509088ab9249302af561ebee76a1281a1f08ecb4ed3' diff --git a/README.md b/README.md index 7b9e606b..31b4af37 100644 --- a/README.md +++ b/README.md @@ -11,14 +11,61 @@ Learn more about ZITADEL by checking out the [source repository on GitHub](https ## What's in the Chart -By default, this chart installs: -- A highly available ZITADEL deployment. -- A highly available and secure CockroachDB statefulset. +By default, this chart installs a highly available ZITADEL deployment. ## Install the Chart Follow the [guide for deploying ZITADEL on Kubernetes](https://docs.zitadel.com/docs/guides/deploy/kubernetes). +## Upgrading from v3 + +:::Note +Apart from breaking changes in this chart, v4 also update the +default ZITADEL version to v2.14.2. For upgrading +ZITADEL, please refer to the +[ZITADEL release notes](https://github.com/zitadel/zitadel/releases/tag/v2.14.0). + +This section is only relevant for existing releases with the +values property cockroachdb.enabled not set to false. + +In v4, the cockroachdb chart dependency is removed. +We decided to go this way because: +- Maintaining two separate releases is easier, especially in production. +- We can use Helm hooks specific to ZITADEL. +- ZITADEL doesn't only support CockroachDB. + +If you have cockroachdb.enabled=true in your values.yaml, +you need to make sure, that the cockroachdb chart is not +managed by the zitadel release anymore. The following +example for doing so uninstalls your entire zitadel +release, reinstalls cockroach using a dedicated release, +and then installs the new zitadel chart version. +The new cockroach release will take over the PersistentVolumeClaims +from the uninstalled chart, so no data migration is needed. +Nevertheless, we highly recommend making and testing a backup before upgrading. +Also note, that you will have downtime when +following the example while zitadel is uninstalled. + +```bash +helm repo add cockroachdb https://charts.cockroachdb.com/ +helm repo update cockroachdb zitadel +helm uninstall my-zitadel +helm install crdb cockroachdb/cockroachdb --version 8.1.8 --set fullnameOverride=crdb +helm install my-zitadel zitadel/zitadel --values ./my-zitadel-values.yaml +``` + +## Uninstalling the Chart + +The ZITADEL chart uses Helm hooks, +[which are not garbage collected by helm uninstall, yet](https://helm.sh/docs/topics/charts_hooks/#hook-resources-are-not-managed-with-corresponding-releases). +Therefore, to also remove hooks installed by the ZITADEL Helm chart, +delete them manually: + +```bash +helm uninstall my-zitadel +kubectl delete job --selector app.kubernetes.io/name=zitadel,app.kubernetes.io/managed-by=Helm +``` + ## Contributors diff --git a/charts/zitadel/Chart.lock b/charts/zitadel/Chart.lock deleted file mode 100644 index 37640026..00000000 --- a/charts/zitadel/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: cockroachdb - repository: https://charts.cockroachdb.com/ - version: 8.1.7 -digest: sha256:029f534dc34a839b4d4e45bfcdf3d1d0868e85c7acc8ddda7a77b68085c3f7a1 -generated: "2022-10-18T10:19:40.057964275+02:00" diff --git a/charts/zitadel/Chart.yaml b/charts/zitadel/Chart.yaml index cafb3e2b..727c6b03 100644 --- a/charts/zitadel/Chart.yaml +++ b/charts/zitadel/Chart.yaml @@ -2,16 +2,11 @@ apiVersion: v2 name: zitadel description: A Helm chart for ZITADEL v2 type: application -appVersion: "v2.13.1" -version: 3.5.0 -kubeVersion: '>= 1.16.15-0' +appVersion: "v2.15.0" +version: 4.0.0 +kubeVersion: ">= 1.18.20-0" icon: https://zitadel.zitadel.cloud/ui/login/resources/themes/zitadel/logo-dark.svg -dependencies: -- name: cockroachdb - version: ~8.1.0 - repository: https://charts.cockroachdb.com/ - condition: cockroachdb.enabled maintainers: -- name: zitadel - email: support@zitadel.com - url: https://zitadel.com + - name: zitadel + email: support@zitadel.com + url: https://zitadel.com diff --git a/charts/zitadel/templates/configmap.yaml b/charts/zitadel/templates/configmap.yaml index 988082ca..7f17adfc 100644 --- a/charts/zitadel/templates/configmap.yaml +++ b/charts/zitadel/templates/configmap.yaml @@ -2,6 +2,10 @@ apiVersion: v1 kind: ConfigMap metadata: name: zitadel-config-yaml + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation + helm.sh/hook-weight: "0" data: zitadel-config-yaml: |- {{ .Values.zitadel.configmapConfig | toYaml | nindent 4 }} diff --git a/charts/zitadel/templates/deployment.yaml b/charts/zitadel/templates/deployment.yaml index fa6837a1..40d735b6 100644 --- a/charts/zitadel/templates/deployment.yaml +++ b/charts/zitadel/templates/deployment.yaml @@ -4,6 +4,7 @@ metadata: name: {{ include "zitadel.fullname" . }} labels: {{- include "zitadel.labels" . | nindent 4 }} + app.kubernetes.io/component: start spec: replicas: {{ .Values.replicaCount }} selector: @@ -19,6 +20,7 @@ spec: checksum/secret-db-ssl-root-crt: {{ include (print $.Template.BasePath "/secret_db-ssl-root-crt.yaml") . | sha256sum }} checksum/secret-zitadel-secrets: {{ include (print $.Template.BasePath "/secret_zitadel-secrets.yaml") . | sha256sum }} labels: + app.kubernetes.io/component: start {{- include "zitadel.selectorLabels" . | nindent 8 }} spec: {{- with .Values.imagePullSecrets }} @@ -36,7 +38,7 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} args: - - start-from-init + - start - --config - /config/zitadel-config-yaml {{- if .Values.zitadel.secretConfig }} @@ -47,8 +49,6 @@ spec: - --config - /.secrets/config-yaml {{- end }} - - --steps - - /config/zitadel-config-yaml - --masterkeyFile - /.secrets/masterkey env: diff --git a/charts/zitadel/templates/initjob.yaml b/charts/zitadel/templates/initjob.yaml new file mode 100644 index 00000000..d14b8aa8 --- /dev/null +++ b/charts/zitadel/templates/initjob.yaml @@ -0,0 +1,166 @@ +{{- if .Values.initJob.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: "{{ include "zitadel.fullname" . }}-init" + labels: + {{- include "zitadel.labels" . | nindent 4 }} + app.kubernetes.io/component: init + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation + helm.sh/hook-weight: "1" +spec: + backoffLimit: 5 + activeDeadlineSeconds: {{ .Values.initJob.activeDeadlineSeconds }} + template: + metadata: + labels: + {{- include "zitadel.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: init + {{- with .Values.initJob.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "zitadel.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + enableServiceLinks: false + restartPolicy: OnFailure + containers: + - name: "{{ .Chart.Name }}-init" + securityContext: + {{- toYaml .Values.securityContext | nindent 14 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - init + - --config + - /config/zitadel-config-yaml + {{- if .Values.zitadel.secretConfig }} + - --config + - /.secrets/zitadel-secrets-yaml + {{- end }} + {{- if .Values.zitadel.configSecretName }} + - --config + - /.secrets/config-yaml + {{- end }} + env: + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + {{- if (or .Values.zitadel.dbSslRootCrt .Values.zitadel.dbSslRootCrtSecret) }} + - name: ZITADEL_DATABASE_COCKROACH_USER_SSL_ROOTCERT + value: /.secrets/ca.crt + - name: ZITADEL_DATABASE_COCKROACH_ADMIN_SSL_ROOTCERT + value: /.secrets/ca.crt + {{- end}} + {{- if .Values.zitadel.dbSslClientCrtSecret }} + - name: ZITADEL_DATABASE_COCKROACH_ADMIN_SSL_CERT + value: /.secrets/tls.crt + - name: ZITADEL_DATABASE_COCKROACH_ADMIN_SSL_KEY + value: /.secrets/tls.key + {{- end}} + {{- with .Values.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: zitadel-config-yaml + mountPath: /config + - name: chowned-secrets + mountPath: /.secrets + resources: + {{- toYaml .Values.initJob.resources | nindent 14 }} + {{- if .Values.initJob.extraContainers }} + {{- toYaml .Values.initJob.extraContainers | nindent 8 }} + {{- end }} + initContainers: + - args: + - "{{ include "zitadel.joincpcommands" (dict "commands" (list + (include "zitadel.makecpcommand" (dict "value" .Values.zitadel.secretConfig "path" "/zitadel-secrets-yaml/*" )) + (include "zitadel.makecpcommand" (dict "value" (or .Values.zitadel.dbSslRootCrt .Values.zitadel.dbSslRootCrtSecret) "path" "/db-ssl-root-crt/*" )) + (include "zitadel.makecpcommand" (dict "value" .Values.zitadel.dbSslClientCrtSecret "path" "/db-ssl-client-crt/*" )) + (include "zitadel.makecpcommand" (dict "value" .Values.zitadel.configSecretName "path" "/zitadel-config-yaml/*" )) + )) }} chown -R 1000:1000 /chowned-secrets/* && chmod 400 /chowned-secrets/*" + command: + - sh + - -c + image: "{{ .Values.chownImage.repository }}:{{ .Values.chownImage.tag }}" + imagePullPolicy: {{ .Values.chownImage.pullPolicy }} + name: chown + volumeMounts: + - name: chowned-secrets + mountPath: /chowned-secrets + {{- if .Values.zitadel.secretConfig }} + - name: zitadel-secrets-yaml + mountPath: /zitadel-secrets-yaml + {{- end }} + {{- if .Values.zitadel.configSecretName }} + - name: zitadel-secret-config-yaml + mountPath: /zitadel-config-yaml + {{- end }} + {{- if (or .Values.zitadel.dbSslRootCrt .Values.zitadel.dbSslRootCrtSecret) }} + - name: db-ssl-root-crt + mountPath: /db-ssl-root-crt + {{- end }} + {{- if .Values.zitadel.dbSslClientCrtSecret }} + - name: db-ssl-client-crt + mountPath: /db-ssl-client-crt + {{- end }} + securityContext: + runAsNonRoot: false + runAsUser: 0 + volumes: + - name: zitadel-config-yaml + configMap: + name: zitadel-config-yaml + {{- if .Values.zitadel.secretConfig }} + - name: zitadel-secrets-yaml + secret: + secretName: zitadel-secrets-yaml + {{- end }} + {{- if .Values.zitadel.configSecretName }} + - name: zitadel-secret-config-yaml + secret: + secretName: {{ .Values.zitadel.configSecretName }} + {{- end }} + {{- if .Values.zitadel.dbSslRootCrt }} + - name: db-ssl-root-crt + secret: + secretName: db-ssl-root-crt + {{- end }} + {{- if .Values.zitadel.dbSslRootCrtSecret }} + - name: db-ssl-root-crt + secret: + secretName: {{ .Values.zitadel.dbSslRootCrtSecret }} + {{- end }} + {{- if .Values.zitadel.dbSslClientCrtSecret }} + - name: db-ssl-client-crt + secret: + secretName: {{ .Values.zitadel.dbSslClientCrtSecret }} + {{- end }} +{{- if (or (and .Values.zitadel.masterkey .Values.zitadel.masterkeySecretName) (and (not .Values.zitadel.masterkey) (not .Values.zitadel.masterkeySecretName)) ) }} +{{- fail "Eighter set .Values.zitadel.masterkey or .Values.zitadel.masterkeySecretName exclusively" }} +{{- end }} + - name: chowned-secrets + emptyDir: {} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{ end }} \ No newline at end of file diff --git a/charts/zitadel/templates/secret_db-ssl-root-crt.yaml b/charts/zitadel/templates/secret_db-ssl-root-crt.yaml index d36b427e..09f052c7 100644 --- a/charts/zitadel/templates/secret_db-ssl-root-crt.yaml +++ b/charts/zitadel/templates/secret_db-ssl-root-crt.yaml @@ -4,6 +4,10 @@ kind: Secret type: Opaque metadata: name: db-ssl-root-crt + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation + helm.sh/hook-weight: "0" stringData: ca.crt: |- {{ .Values.zitadel.dbSslRootCrt | default "" | nindent 6 }} diff --git a/charts/zitadel/templates/secret_zitadel-masterkey.yaml b/charts/zitadel/templates/secret_zitadel-masterkey.yaml index 7d872f25..a5932fd0 100644 --- a/charts/zitadel/templates/secret_zitadel-masterkey.yaml +++ b/charts/zitadel/templates/secret_zitadel-masterkey.yaml @@ -4,6 +4,10 @@ kind: Secret type: Opaque metadata: name: zitadel-masterkey + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation + helm.sh/hook-weight: "0" stringData: masterkey: {{ .Values.zitadel.masterkey }} {{- end -}} diff --git a/charts/zitadel/templates/secret_zitadel-secrets.yaml b/charts/zitadel/templates/secret_zitadel-secrets.yaml index b92c4bac..c0959142 100644 --- a/charts/zitadel/templates/secret_zitadel-secrets.yaml +++ b/charts/zitadel/templates/secret_zitadel-secrets.yaml @@ -1,11 +1,12 @@ -{{- if (and .Values.cockroachdb.enabled .Values.cockroachdb.tls.enabled (not ((((.Values.zitadel.secretConfig).Database).cockroach).User).Password)) }} -{{- fail ".Values.zitadel.secretConfig.Database.cockroach.User.Password is mandatory for tls enabled cockroach" }} -{{- end }} apiVersion: v1 kind: Secret type: Opaque metadata: name: zitadel-secrets-yaml + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation + helm.sh/hook-weight: "0" stringData: zitadel-secrets-yaml: |- {{ .Values.zitadel.secretConfig | toYaml | nindent 4 }} diff --git a/charts/zitadel/templates/serviceaccount.yaml b/charts/zitadel/templates/serviceaccount.yaml index 9defb0fa..f6ad5b7d 100644 --- a/charts/zitadel/templates/serviceaccount.yaml +++ b/charts/zitadel/templates/serviceaccount.yaml @@ -5,8 +5,11 @@ metadata: name: {{ include "zitadel.serviceAccountName" . }} labels: {{- include "zitadel.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} annotations: - {{- toYaml . | nindent 4 }} - {{- end }} + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation + helm.sh/hook-weight: "0" + {{- with .Values.serviceAccount.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} diff --git a/charts/zitadel/templates/setupjob.yaml b/charts/zitadel/templates/setupjob.yaml new file mode 100644 index 00000000..8e3f1abe --- /dev/null +++ b/charts/zitadel/templates/setupjob.yaml @@ -0,0 +1,175 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: "{{ include "zitadel.fullname" . }}-setup" + labels: + {{- include "zitadel.labels" . | nindent 4 }} + app.kubernetes.io/component: setup + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation + helm.sh/hook-weight: "2" +spec: + backoffLimit: 5 + activeDeadlineSeconds: {{ .Values.setupJob.activeDeadlineSeconds }} + template: + metadata: + labels: + {{- include "zitadel.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: setup + {{- with .Values.setupJob.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "zitadel.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + enableServiceLinks: false + restartPolicy: OnFailure + containers: + - name: "{{ .Chart.Name }}-setup" + securityContext: + {{- toYaml .Values.securityContext | nindent 14 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - setup + - --config + - /config/zitadel-config-yaml + {{- if .Values.zitadel.secretConfig }} + - --config + - /.secrets/zitadel-secrets-yaml + {{- end }} + {{- if .Values.zitadel.configSecretName }} + - --config + - /.secrets/config-yaml + {{- end }} + - --steps + - /config/zitadel-config-yaml + - --masterkeyFile + - /.secrets/masterkey + env: + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + {{- if (or .Values.zitadel.dbSslRootCrt .Values.zitadel.dbSslRootCrtSecret) }} + - name: ZITADEL_DATABASE_COCKROACH_USER_SSL_ROOTCERT + value: /.secrets/ca.crt + - name: ZITADEL_DATABASE_COCKROACH_ADMIN_SSL_ROOTCERT + value: /.secrets/ca.crt + {{- end}} + {{- if .Values.zitadel.dbSslClientCrtSecret }} + - name: ZITADEL_DATABASE_COCKROACH_ADMIN_SSL_CERT + value: /.secrets/tls.crt + - name: ZITADEL_DATABASE_COCKROACH_ADMIN_SSL_KEY + value: /.secrets/tls.key + {{- end}} + {{- with .Values.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: zitadel-config-yaml + mountPath: /config + - name: chowned-secrets + mountPath: /.secrets + resources: + {{- toYaml .Values.setupJob.resources | nindent 14 }} + {{- if .Values.setupJob.extraContainers }} + {{- toYaml .Values.setupJob.extraContainers | nindent 8 }} + {{- end }} + initContainers: + - args: + - "{{ include "zitadel.joincpcommands" (dict "commands" (list + (include "zitadel.makecpcommand" (dict "value" .Values.zitadel.secretConfig "path" "/zitadel-secrets-yaml/*" )) + (include "zitadel.makecpcommand" (dict "value" .Values.zitadel.masterkey "path" "/masterkey/*" )) + (include "zitadel.makecpcommand" (dict "value" .Values.zitadel.masterkeySecretName "path" "/masterkey/*" )) + (include "zitadel.makecpcommand" (dict "value" (or .Values.zitadel.dbSslRootCrt .Values.zitadel.dbSslRootCrtSecret) "path" "/db-ssl-root-crt/*" )) + (include "zitadel.makecpcommand" (dict "value" .Values.zitadel.dbSslClientCrtSecret "path" "/db-ssl-client-crt/*" )) + (include "zitadel.makecpcommand" (dict "value" .Values.zitadel.configSecretName "path" "/zitadel-config-yaml/*" )) + )) }} chown -R 1000:1000 /chowned-secrets/* && chmod 400 /chowned-secrets/*" + command: + - sh + - -c + image: "{{ .Values.chownImage.repository }}:{{ .Values.chownImage.tag }}" + imagePullPolicy: {{ .Values.chownImage.pullPolicy }} + name: chown + volumeMounts: + - name: chowned-secrets + mountPath: /chowned-secrets + - name: masterkey + mountPath: /masterkey + {{- if .Values.zitadel.secretConfig }} + - name: zitadel-secrets-yaml + mountPath: /zitadel-secrets-yaml + {{- end }} + {{- if .Values.zitadel.configSecretName }} + - name: zitadel-secret-config-yaml + mountPath: /zitadel-config-yaml + {{- end }} + {{- if (or .Values.zitadel.dbSslRootCrt .Values.zitadel.dbSslRootCrtSecret) }} + - name: db-ssl-root-crt + mountPath: /db-ssl-root-crt + {{- end }} + {{- if .Values.zitadel.dbSslClientCrtSecret }} + - name: db-ssl-client-crt + mountPath: /db-ssl-client-crt + {{- end }} + securityContext: + runAsNonRoot: false + runAsUser: 0 + volumes: + - name: zitadel-config-yaml + configMap: + name: zitadel-config-yaml + {{- if .Values.zitadel.secretConfig }} + - name: zitadel-secrets-yaml + secret: + secretName: zitadel-secrets-yaml + {{- end }} + {{- if .Values.zitadel.configSecretName }} + - name: zitadel-secret-config-yaml + secret: + secretName: {{ .Values.zitadel.configSecretName }} + {{- end }} + {{- if .Values.zitadel.dbSslRootCrt }} + - name: db-ssl-root-crt + secret: + secretName: db-ssl-root-crt + {{- end }} + {{- if .Values.zitadel.dbSslRootCrtSecret }} + - name: db-ssl-root-crt + secret: + secretName: {{ .Values.zitadel.dbSslRootCrtSecret }} + {{- end }} + {{- if .Values.zitadel.dbSslClientCrtSecret }} + - name: db-ssl-client-crt + secret: + secretName: {{ .Values.zitadel.dbSslClientCrtSecret }} + {{- end }} +{{- if (or (and .Values.zitadel.masterkey .Values.zitadel.masterkeySecretName) (and (not .Values.zitadel.masterkey) (not .Values.zitadel.masterkeySecretName)) ) }} +{{- fail "Eighter set .Values.zitadel.masterkey or .Values.zitadel.masterkeySecretName exclusively" }} +{{- end }} + - name: masterkey + secret: + secretName: {{ default "zitadel-masterkey" .Values.zitadel.masterkeySecretName }} + - name: chowned-secrets + emptyDir: {} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/zitadel/test/installation/accessibility.go b/charts/zitadel/test/installation/accessibility.go index 5559b72a..f8e30053 100644 --- a/charts/zitadel/test/installation/accessibility.go +++ b/charts/zitadel/test/installation/accessibility.go @@ -21,34 +21,46 @@ type checkOptions struct { test func(response *http.Response) error } -func (c *checkOptions) execute(ctx context.Context, t *testing.T, wg *sync.WaitGroup) { +func (c *checkOptions) try(ctx context.Context, t *testing.T, wg *sync.WaitGroup, try int) { + + err := c.execute(ctx, t) + if err == nil { + wg.Done() + return + } + + if try == 0 { + t.Fatal(err) + } + time.Sleep(time.Second) + c.try(ctx, t, wg, try-1) +} + +func (c *checkOptions) execute(ctx context.Context, t *testing.T) (err error) { checkCtx, checkCancel := context.WithTimeout(ctx, 5*time.Second) defer checkCancel() - defer wg.Done() req, err := http.NewRequestWithContext(checkCtx, http.MethodGet, c.getUrl, nil) if err != nil { - t.Fatalf("creating request for url %s failed: %s", c.getUrl, err.Error()) - return + return fmt.Errorf("creating request for url %s failed: %s", c.getUrl, err.Error()) } resp, err := http.DefaultClient.Do(req) if err != nil { - t.Fatalf("sending request %+v failed: %s", *req, err) - return + return fmt.Errorf("sending request %+v failed: %s", *req, err) } defer resp.Body.Close() if err = c.test(resp); err != nil { - t.Fatalf("checking response to request %+v failed: %s", *req, err) - return + return fmt.Errorf("checking response to request %+v failed: %s", *req, err) } + return nil } func (s *configurationTest) checkAccessibility(pods []corev1.Pod) { ctx, cancel := context.WithTimeout(s.context, time.Minute) defer cancel() - serviceTunnel := k8s.NewTunnel(s.options.KubectlOptions, k8s.ResourceTypeService, s.release, 8080, 8080) + serviceTunnel := k8s.NewTunnel(s.kubeOptions, k8s.ResourceTypeService, s.zitadelRelease, 8080, 8080) serviceTunnel.ForwardPort(s.T()) tunnels := []*k8s.Tunnel{serviceTunnel} @@ -88,7 +100,7 @@ func (s *configurationTest) checkAccessibility(pods []corev1.Pod) { pod := pods[i] port := 8081 + i - podTunnel := k8s.NewTunnel(s.options.KubectlOptions, k8s.ResourceTypePod, pod.Name, port, 8080) + podTunnel := k8s.NewTunnel(s.kubeOptions, k8s.ResourceTypePod, pod.Name, port, 8080) podTunnel.ForwardPort(s.T()) tunnels = append(tunnels, podTunnel) checks = append(checks, zitadelStatusChecks(port)...) @@ -97,7 +109,7 @@ func (s *configurationTest) checkAccessibility(pods []corev1.Pod) { wg := sync.WaitGroup{} for _, check := range checks { wg.Add(1) - go check.execute(ctx, s.T(), &wg) + go check.try(ctx, s.T(), &wg, 60) } wait(ctx, s.T(), &wg, "accessibility") } diff --git a/charts/zitadel/test/installation/after.go b/charts/zitadel/test/installation/after.go index 6c9f8dd8..ad28fc7e 100644 --- a/charts/zitadel/test/installation/after.go +++ b/charts/zitadel/test/installation/after.go @@ -1,13 +1,21 @@ package installation import ( + "os" + "github.com/gruntwork-io/terratest/modules/k8s" ) func (s *configurationTest) TearDownTest() { + + if _, exists := os.LookupEnv("GITHUB_SHA"); exists { + s.log.Logf(s.T(), "Not running cleanup tasks, as tests were run on a throwaway runner") + return + } + if !s.T().Failed() { - k8s.DeleteNamespace(s.T(), s.options.KubectlOptions, s.namespace) + k8s.DeleteNamespace(s.T(), s.kubeOptions, s.kubeOptions.Namespace) } else { - s.log.Logf(s.T(), "Test failed on namespace %s. Omitting cleanup.", s.namespace) + s.log.Logf(s.T(), "Test failed on namespace %s. Omitting cleanup.", s.kubeOptions.Namespace) } } diff --git a/charts/zitadel/test/installation/availability.go b/charts/zitadel/test/installation/availability.go index b8d8034f..58159dcc 100644 --- a/charts/zitadel/test/installation/availability.go +++ b/charts/zitadel/test/installation/availability.go @@ -17,7 +17,7 @@ func (s *configurationTest) awaitReadiness(pods []corev1.Pod) { for _, p := range pods { wg.Add(1) go func(pod corev1.Pod) { - k8s.WaitUntilPodAvailable(s.T(), s.options.KubectlOptions, pod.Name, 300, time.Second) + k8s.WaitUntilPodAvailable(s.T(), s.kubeOptions, pod.Name, 300, time.Second) wg.Done() }(p) } diff --git a/charts/zitadel/test/installation/before.go b/charts/zitadel/test/installation/before.go index e86aa28d..29e4518c 100644 --- a/charts/zitadel/test/installation/before.go +++ b/charts/zitadel/test/installation/before.go @@ -8,7 +8,7 @@ import ( func (s *configurationTest) SetupTest() { clusterKubectl := new(k8s.KubectlOptions) - if err := copier.Copy(clusterKubectl, s.options.KubectlOptions); err != nil { + if err := copier.Copy(clusterKubectl, s.kubeOptions); err != nil { s.T().Fatal(err) } @@ -30,22 +30,22 @@ metadata: name: crdb `) - if _, err := k8s.GetNamespaceE(s.T(), s.options.KubectlOptions, s.namespace); err != nil { - k8s.CreateNamespace(s.T(), s.options.KubectlOptions, s.namespace) + if _, err := k8s.GetNamespaceE(s.T(), s.kubeOptions, s.kubeOptions.Namespace); err != nil { + k8s.CreateNamespace(s.T(), s.kubeOptions, s.kubeOptions.Namespace) } else { - s.log.Logf(s.T(), "Namespace: %s already exist!", s.namespace) + s.log.Logf(s.T(), "Namespace: %s already exist!", s.kubeOptions.Namespace) } if s.beforeFunc == nil { return } - clientset, err := k8s.GetKubernetesClientFromOptionsE(s.T(), s.options.KubectlOptions) + clientset, err := k8s.GetKubernetesClientFromOptionsE(s.T(), s.kubeOptions) if err != nil { s.T().Fatal(err) } - if err = s.beforeFunc(s.context, s.options.KubectlOptions.Namespace, clientset); err != nil { + if err = s.beforeFunc(s.context, s.kubeOptions.Namespace, clientset); err != nil { s.T().Fatal(err) } } diff --git a/charts/zitadel/test/installation/config.go b/charts/zitadel/test/installation/config.go index 694d2a48..98b86bb4 100644 --- a/charts/zitadel/test/installation/config.go +++ b/charts/zitadel/test/installation/config.go @@ -2,51 +2,64 @@ package installation import ( "context" - "github.com/gruntwork-io/terratest/modules/helm" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "github.com/gruntwork-io/terratest/modules/k8s" "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/random" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "k8s.io/client-go/kubernetes" - "os" - "path/filepath" - "strings" - "testing" ) type beforeFunc func(ctx context.Context, namespace string, k8sClient *kubernetes.Clientset) error type configurationTest struct { suite.Suite - context context.Context - log *logger.Logger - chartPath string - release string - namespace string - options *helm.Options - beforeFunc beforeFunc + context context.Context + log *logger.Logger + kubeOptions *k8s.KubectlOptions + zitadelValues map[string]string + crdbValues map[string]string + zitadelChartPath string + zitadelRelease string + crdbRepoName string + crdbRepoURL string + crdbChart string + crdbVersion string + crdbRelease string + beforeFunc beforeFunc } -func TestConfiguration(t *testing.T, before beforeFunc, values map[string]string) { +func TestConfiguration(t *testing.T, before beforeFunc, zitadelValues map[string]string) { chartPath, err := filepath.Abs("../../") require.NoError(t, err) namespace := createNamespaceName() + crdbRepoName := fmt.Sprintf("crdb-%s", strings.TrimPrefix(namespace, "zitadel-helm-")) kubeOptions := k8s.NewKubectlOptions("", "", namespace) it := &configurationTest{ - context: context.Background(), - log: logger.New(logger.Terratest), - chartPath: chartPath, - release: "zitadel-test", - namespace: namespace, - options: &helm.Options{ - KubectlOptions: kubeOptions, - SetValues: values, + context: context.Background(), + log: logger.New(logger.Terratest), + kubeOptions: kubeOptions, + zitadelValues: zitadelValues, + crdbValues: map[string]string{ + "fullnameOverride": "crdb", }, - beforeFunc: before, + zitadelChartPath: chartPath, + zitadelRelease: "zitadel-test", + crdbRepoURL: "https://charts.cockroachdb.com/", + crdbRepoName: crdbRepoName, + crdbChart: fmt.Sprintf("%s/cockroachdb", crdbRepoName), + crdbRelease: "crdb", + crdbVersion: "10.0.0", + beforeFunc: before, } suite.Run(t, it) } diff --git a/charts/zitadel/test/installation/run.go b/charts/zitadel/test/installation/run.go index 07eb5996..c3d3c756 100644 --- a/charts/zitadel/test/installation/run.go +++ b/charts/zitadel/test/installation/run.go @@ -1,6 +1,9 @@ package installation import ( + "testing" + "time" + "github.com/gruntwork-io/terratest/modules/helm" "github.com/gruntwork-io/terratest/modules/k8s" corev1 "k8s.io/api/core/v1" @@ -8,15 +11,23 @@ import ( ) func (s *configurationTest) TestZITADELInstallation() { - // given - options := s.options - // when - helm.Install(s.T(), options, s.chartPath, s.release) + helm.AddRepo(s.T(), &helm.Options{}, s.crdbRepoName, s.crdbRepoURL) + helm.Install(s.T(), &helm.Options{ + KubectlOptions: s.kubeOptions, + SetValues: s.crdbValues, + Version: s.crdbVersion, + }, s.crdbChart, s.crdbRelease) + + helm.Install(s.T(), &helm.Options{ + KubectlOptions: s.kubeOptions, + SetValues: s.zitadelValues, + }, s.zitadelChartPath, s.zitadelRelease) + + k8s.WaitUntilJobSucceed(s.T(), s.kubeOptions, "zitadel-test-init", 300, time.Second) + k8s.WaitUntilJobSucceed(s.T(), s.kubeOptions, "zitadel-test-setup", 300, time.Second) - // then - // await that all zitadel related pods become ready - pods := k8s.ListPods(s.T(), s.options.KubectlOptions, metav1.ListOptions{LabelSelector: `app.kubernetes.io/instance=zitadel-test, app.kubernetes.io/component notin (init)`}) + pods := listPods(s.T(), 5, s.kubeOptions) s.awaitReadiness(pods) zitadelPods := make([]corev1.Pod, 0) for i := range pods { @@ -28,3 +39,18 @@ func (s *configurationTest) TestZITADELInstallation() { s.log.Logf(s.T(), "ZITADEL pods are ready") s.checkAccessibility(zitadelPods) } + +// listPods retries until all three start pods are returned from the kubeapi +func listPods(t *testing.T, try int, kubeOptions *k8s.KubectlOptions) []corev1.Pod { + + if try == 0 { + t.Fatal("no trials left") + } + + pods := k8s.ListPods(t, kubeOptions, metav1.ListOptions{LabelSelector: `app.kubernetes.io/instance=zitadel-test, app.kubernetes.io/component=start`}) + if len(pods) == 3 { + return pods + } + time.Sleep(time.Second) + return listPods(t, try-1, kubeOptions) +} diff --git a/charts/zitadel/values.yaml b/charts/zitadel/values.yaml index e37d35ee..9d01599e 100644 --- a/charts/zitadel/values.yaml +++ b/charts/zitadel/values.yaml @@ -1,11 +1,9 @@ # Default values for zitadel. zitadel: - # The ZITADEL config under configmapConfig is written to a Kubernetes ConfigMap # See all defaults here: # https://github.com/zitadel/zitadel/blob/main/cmd/defaults.yaml configmapConfig: - Database: cockroach: Host: "crdb-public" @@ -34,19 +32,19 @@ zitadel: # ZITADEL uses the masterkey for symmetric encryption. # You can generate it for example with tr -dc A-Za-z0-9