diff --git a/charts/cloudstack-csi/templates/_helpers.tpl b/charts/cloudstack-csi/templates/_helpers.tpl index 8d5dd4d..0985bf6 100644 --- a/charts/cloudstack-csi/templates/_helpers.tpl +++ b/charts/cloudstack-csi/templates/_helpers.tpl @@ -2,7 +2,7 @@ {{/* Expand the name of the chart. */}} -{{- define "csi.name" -}} +{{- define "cloudstack-csi-driver.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} @@ -11,7 +11,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "csi.fullname" -}} +{{- define "cloudstack-csi-driver.fullname" -}} {{- if .Values.fullnameOverride -}} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} {{- else -}} @@ -27,73 +27,36 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "csi.chart" -}} +{{- define "cloudstack-csi-driver.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} {{/* Common labels */}} -{{- define "csi.labels" -}} -app.kubernetes.io/name: {{ include "csi.name" . }} -helm.sh/chart: {{ include "csi.chart" . }} -app.kubernetes.io/instance: {{ .Release.Name }} +{{- define "cloudstack-csi-driver.labels" -}} +{{ include "cloudstack-csi-driver.selectorLabels" . }} +helm.sh/chart: {{ include "cloudstack-csi-driver.chart" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} +app.kubernetes.io/component: csi-driver app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end -}} - - -{{/* -Create the name of the service account to use -*/}} -{{- define "csi.serviceAccountName" -}} -{{- if .Values.serviceAccount.create -}} - {{ default (include "csi.fullname" .) .Values.serviceAccount.name }} -{{- else -}} - {{ default "default" .Values.serviceAccount.name }} -{{- end -}} +{{- if .Values.customLabels }} +{{ toYaml .Values.extraLabels }} +{{- end }} {{- end -}} {{/* -Create unified labels for csi components +Common selector labels */}} -{{- define "csi.common.matchLabels" -}} -app: {{ template "csi.name" . }} -release: {{ .Release.Name }} -{{- end -}} - -{{- define "csi.common.metaLabels" -}} -chart: {{ template "csi.chart" . }} -heritage: {{ .Release.Service }} -{{- if .Values.extraLabels }} -{{ toYaml .Values.extraLabels -}} -{{- end }} -{{- end -}} - -{{- define "csi.controller.matchLabels" -}} -component: controller -{{ include "csi.common.matchLabels" . }} -{{- end -}} - -{{- define "csi.controller.labels" -}} -{{ include "csi.controller.matchLabels" . }} -{{ include "csi.common.metaLabels" . }} -{{- end -}} - -{{- define "csi.node.matchLabels" -}} -component: node -{{ include "csi.common.matchLabels" . }} -{{- end -}} - -{{- define "csi.node.labels" -}} -{{ include "csi.node.matchLabels" . }} -{{ include "csi.common.metaLabels" . }} +{{- define "cloudstack-csi-driver.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cloudstack-csi-driver.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} {{- end -}} {{/* -Create cloud-config makro. +Create cloud-config macro. */}} {{- define "cloudConfig" -}} [Global] diff --git a/charts/cloudstack-csi/templates/csi-clusterrole.yaml b/charts/cloudstack-csi/templates/csi-clusterrole.yaml index 2c956a7..f82003d 100644 --- a/charts/cloudstack-csi/templates/csi-clusterrole.yaml +++ b/charts/cloudstack-csi/templates/csi-clusterrole.yaml @@ -1,18 +1,17 @@ {{ if .Values.controller.enabled }} {{ if .Values.rbac.create }} -# This YAML file contains Cluster Role objects, -# which are necessary to run cloudstack-csi-controller -{{ $enableLeaderElection := gt (int .Values.controller.replicaCount) 1 }} --- # external Attacher kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-attacher-role + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} rules: - apiGroups: [""] resources: ["persistentvolumes"] @@ -26,22 +25,18 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments/status"] verbs: ["patch"] - {{- if $enableLeaderElection}} - # leader election - - apiGroups: [coordination.k8s.io] - resources: [leases] - verbs: ["get", "watch", "list", "delete", "update", "create"] - {{- end}} --- # external Provisioner kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-provisioner-role + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} rules: - apiGroups: [""] resources: ["persistentvolumes"] @@ -70,18 +65,18 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["get", "list", "watch"] - {{- if $enableLeaderElection}} - # leader election - - apiGroups: [coordination.k8s.io] - resources: [leases] - verbs: ["get", "watch", "list", "delete", "update", "create"] - {{- end}} --- # external Resizer kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-resizer-role + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} rules: # The following rule should be uncommented for plugins that require secrets # for provisioning. @@ -103,11 +98,5 @@ rules: - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - {{- if $enableLeaderElection}} - # leader election - - apiGroups: [coordination.k8s.io] - resources: [leases] - verbs: ["get", "watch", "list", "delete", "update", "create"] - {{- end}} {{- end}} {{- end}} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/csi-clusterrolebinding.yaml b/charts/cloudstack-csi/templates/csi-controller-clusterrolebinding.yaml similarity index 66% rename from charts/cloudstack-csi/templates/csi-clusterrolebinding.yaml rename to charts/cloudstack-csi/templates/csi-controller-clusterrolebinding.yaml index 4420bc8..5b54315 100644 --- a/charts/cloudstack-csi/templates/csi-clusterrolebinding.yaml +++ b/charts/cloudstack-csi/templates/csi-controller-clusterrolebinding.yaml @@ -1,19 +1,19 @@ {{ if .Values.controller.enabled }} {{ if .Values.rbac.create }} -# This YAML file contains Cluster Role Binding objects, -# which are necessary to run cloudstack-csi-controller --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-attacher-binding + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} subjects: - kind: ServiceAccount - name: {{ .Values.serviceAccount.name }} + name: {{ .Values.controller.serviceAccount.name }} namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole @@ -24,13 +24,15 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-provisioner-binding + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} subjects: - kind: ServiceAccount - name: {{ .Values.serviceAccount.name }} + name: {{ .Values.controller.serviceAccount.name }} namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole @@ -41,13 +43,15 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-resizer-binding + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} subjects: - kind: ServiceAccount - name: {{ .Values.serviceAccount.name }} + name: {{ .Values.controller.serviceAccount.name }} namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole diff --git a/charts/cloudstack-csi/templates/csi-controller-deploy.yaml b/charts/cloudstack-csi/templates/csi-controller-deploy.yaml index 5d751ad..5e0dc98 100644 --- a/charts/cloudstack-csi/templates/csi-controller-deploy.yaml +++ b/charts/cloudstack-csi/templates/csi-controller-deploy.yaml @@ -1,192 +1,307 @@ {{ if .Values.controller.enabled }} -{{ $enableLeaderElection := gt (int .Values.controller.replicaCount) 1 }} - apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "csi.name" . }}-controller + name: {{ include "cloudstack-csi-driver.name" . }}-controller namespace: {{ .Release.Namespace }} labels: - {{- include "csi.controller.labels" . | nindent 4 }} + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.controller.deploymentAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} spec: replicas: {{ .Values.controller.replicaCount }} + {{- if or (kindIs "float64" .Values.controller.revisionHistoryLimit) (kindIs "int64" .Values.controller.revisionHistoryLimit) }} + revisionHistoryLimit: {{ .Values.controller.revisionHistoryLimit }} + {{- end }} + {{- with .Values.controller.updateStrategy }} strategy: - type: {{ .Values.controller.strategy.type }} -{{- if eq .Values.controller.strategy.type "RollingUpdate" }} - rollingUpdate: - maxUnavailable: {{ .Values.controller.strategy.rollingUpdate.maxUnavailable }} - maxSurge: {{ .Values.controller.strategy.rollingUpdate.maxSurge }} -{{- end }} + {{- toYaml . | nindent 4 }} + {{- end }} selector: matchLabels: - {{- include "csi.controller.matchLabels" . | nindent 6 }} + app: {{ include "cloudstack-csi-driver.name" . }}-controller + {{- include "cloudstack-csi-driver.selectorLabels" . | nindent 6 }} template: metadata: labels: - {{- include "csi.controller.labels" . | nindent 8 }} + app: {{ include "cloudstack-csi-driver.name" . }}-controller + {{- include "cloudstack-csi-driver.labels" . | nindent 8 }} + {{- if .Values.controller.podLabels }} + {{- toYaml .Values.controller.podLabels | nindent 8 }} + {{- end }} + {{- if .Values.controller.podAnnotations }} annotations: - {{- with .Values.commonAnnotations }} + {{- tpl ( .Values.controller.podAnnotations | toYaml ) . | nindent 8 }} + {{- end }} + spec: + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.controller.nodeSelector }} {{- toYaml . | nindent 8 }} {{- end }} - spec: - serviceAccount: cloudstack-csi-controller + serviceAccount: {{ .Values.controller.serviceAccount.name }} + priorityClassName: {{ .Values.controller.priorityClassName }} + {{- with default .Values.controller.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + tolerations: + {{- with .Values.controller.tolerations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.controller.podSecurityContext }} securityContext: - {{- toYaml .Values.controller.podSecurityContext | nindent 8 }} + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: cloudstack-csi-controller - securityContext: - {{- toYaml .Values.controller.securityContext | nindent 12 }} - image: "{{ .Values.controller.csiDriverController.image.repository }}:{{ .Values.controller.csiDriverController.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.controller.csiDriverController.image.pullPolicy }} + image: "{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.controller.image.pullPolicy }} args: - "controller" - "--endpoint=$(CSI_ENDPOINT)" - "--cloudstack-config=$(CLOUD_CONFIG)" - - "--logging-format={{ .Values.logFormat }}" - - "--v={{ .Values.logVerbosityLevel }}" - {{- if .Values.controller.csiDriverController.extraArgs }} - {{- with .Values.controller.csiDriverController.extraArgs }} - {{- tpl . $ | trim | nindent 12 }} - {{- end }} + - "--logging-format={{ .Values.controller.loggingFormat }}" + - "--v={{ .Values.controller.logLevel }}" + {{- range .Values.controller.extraArgs }} + - {{ . }} {{- end }} env: - name: CSI_ENDPOINT value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - name: CLOUD_CONFIG - value: /etc/config/cloud.conf + value: /etc/config/cloud-config + {{- with .Values.controller.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: cloud-config mountPath: /etc/config/ - {{- with .Values.controller.csiDriverController.volumeMounts }} - {{- toYaml . | nindent 12 }} + {{- with .Values.controller.volumeMounts }} + {{- toYaml . | nindent 12 }} {{- end }} ports: - containerPort: 9808 name: healthz protocol: TCP - {{ if .Values.livenessProbe.enable }} - livenessProbe: {{- toYaml .Values.livenessProbe.livenessProbeSpec | nindent 12 }} + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- with .Values.controller.resources }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} - - name: external-provisioner + {{- with .Values.controller.securityContext }} securityContext: - {{- toYaml .Values.controller.securityContext | nindent 12 }} - image: "{{ .Values.controller.provisioner.image.repository }}:{{ .Values.controller.provisioner.image.tag }}" - imagePullPolicy: {{ .Values.controller.provisioner.image.pullPolicy }} + {{- toYaml . | nindent 12 }} + {{- end }} + - name: external-provisioner + image: "{{ .Values.sidecars.provisioner.image.repository }}:{{ .Values.sidecars.provisioner.image.tag }}" + imagePullPolicy: {{ .Values.sidecars.provisioner.image.pullPolicy }} args: - - "--v={{ .Values.logVerbosityLevel }}" - - "--csi-address=$(ADDRESS)" - - "--timeout={{ .Values.timeout }}" - - "--feature-gates=Topology={{ .Values.controller.provisioner.topology }}" - - "--timeout=300s" - - "--kube-api-qps=100" - - "--kube-api-burst=100" - {{- if $enableLeaderElection }} - - "--leader-election=true" - - "--leader-election-lease-duration=120s" - - "--leader-election-renew-deadline=60s" - - "--leader-election-retry-period=30s" - {{- end}} - - "--default-fstype=ext4" - - "--strict-topology" - {{- if .Values.controller.provisioner.extraArgs }} - {{- with .Values.controller.provisioner.extraArgs }} - {{- tpl . $ | trim | nindent 12 }} + {{- if not (regexMatch "(-timeout)" (join " " .Values.sidecars.provisioner.extraArgs)) }} + - --timeout=60s + {{- end }} + - --csi-address=$(ADDRESS) + - --v={{ .Values.sidecars.provisioner.logLevel }} + {{- if .Values.sidecars.provisioner.featureGates }} + - --feature-gates={{ .Values.sidecars.provisioner.featureGates }} + {{- end }} + {{- if .Values.controller.extraCreateMetadata }} + - --extra-create-metadata + {{- end }} + - --leader-election={{ .Values.sidecars.provisioner.leaderElection.enabled | required "leader election state for csi-provisioner is required, must be set to true || false." }} + {{- if .Values.sidecars.provisioner.leaderElection.enabled }} + {{- if .Values.sidecars.provisioner.leaderElection.leaseDuration }} + - --leader-election-lease-duration={{ .Values.sidecars.provisioner.leaderElection.leaseDuration }} + {{- end }} + {{- if .Values.sidecars.provisioner.leaderElection.renewDeadline}} + - --leader-election-renew-deadline={{ .Values.sidecars.provisioner.leaderElection.renewDeadline }} + {{- end }} + {{- if .Values.sidecars.provisioner.leaderElection.retryPeriod }} + - --leader-election-retry-period={{ .Values.sidecars.provisioner.leaderElection.retryPeriod }} + {{- end }} + {{- end }} + - --default-fstype={{ .Values.controller.defaultFsType }} + {{- if not (regexMatch "(-kube-api-qps)|(-kube-api-burst)|(-worker-threads)" (join " " .Values.sidecars.provisioner.extraArgs)) }} + - --kube-api-qps=20 + - --kube-api-burst=100 + - --worker-threads=100 + {{- end }} + {{- if not (regexMatch "(-retry-interval-max)" (join " " .Values.sidecars.provisioner.extraArgs)) }} + - --retry-interval-max=30m {{- end }} + - --strict-topology + {{- range .Values.sidecars.provisioner.extraArgs }} + - {{ . }} {{- end }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock + {{- with .Values.sidecars.provisioner.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: {{ toYaml .Values.controller.provisioner.resources | nindent 12 }} - - name: external-attacher + {{- with default .Values.controller.resources .Values.sidecars.provisioner.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.provisioner.securityContext }} securityContext: - {{- toYaml .Values.controller.securityContext | nindent 12 }} - image: "{{ .Values.controller.attacher.image.repository }}:{{ .Values.controller.attacher.image.tag }}" - imagePullPolicy: {{ .Values.controller.attacher.image.pullPolicy }} + {{- toYaml . | nindent 12 }} + {{- end }} + - name: external-attacher + image: "{{ .Values.sidecars.attacher.image.repository }}:{{ .Values.sidecars.attacher.image.tag }}" + imagePullPolicy: {{ .Values.sidecars.attacher.image.pullPolicy }} args: - - "--v={{ .Values.logVerbosityLevel }}" - - "--csi-address=$(ADDRESS)" - - "--timeout={{ .Values.timeout }}" - - "--default-fstype=ext4" - {{- if $enableLeaderElection }} - - "--leader-election=true" - - "--leader-election-lease-duration=120s" - - "--leader-election-renew-deadline=60s" - - "--leader-election-retry-period=30s" - {{- end}} - - "--kube-api-qps=100" - - "--kube-api-burst=100" - {{- if .Values.controller.attacher.extraArgs }} - {{- with .Values.controller.attacher.extraArgs }} - {{- tpl . $ | trim | nindent 12 }} + {{- if not (regexMatch "(-timeout)" (join " " .Values.sidecars.attacher.extraArgs)) }} + - --timeout=60s + {{- end }} + - --csi-address=$(ADDRESS) + - --v={{ .Values.sidecars.attacher.logLevel }} + - --leader-election={{ .Values.sidecars.attacher.leaderElection.enabled | required "leader election state for csi-attacher is required, must be set to true || false." }} + {{- if .Values.sidecars.attacher.leaderElection.enabled }} + {{- if .Values.sidecars.attacher.leaderElection.leaseDuration }} + - --leader-election-lease-duration={{ .Values.sidecars.attacher.leaderElection.leaseDuration }} {{- end }} + {{- if .Values.sidecars.attacher.leaderElection.renewDeadline}} + - --leader-election-renew-deadline={{ .Values.sidecars.attacher.leaderElection.renewDeadline }} + {{- end }} + {{- if .Values.sidecars.attacher.leaderElection.retryPeriod }} + - --leader-election-retry-period={{ .Values.sidecars.attacher.leaderElection.retryPeriod }} + {{- end }} + {{- end }} + {{- if not (regexMatch "(-kube-api-qps)|(-kube-api-burst)|(-worker-threads)" (join " " .Values.sidecars.attacher.extraArgs)) }} + - --kube-api-qps=20 + - --kube-api-burst=100 + - --worker-threads=100 + {{- end }} + {{- if not (regexMatch "(-retry-interval-max)" (join " " .Values.sidecars.attacher.extraArgs)) }} + - --retry-interval-max=5m + {{- end }} + {{- range .Values.sidecars.attacher.extraArgs }} + - {{ . }} {{- end }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock + {{- with .Values.sidecars.attacher.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: {{ toYaml .Values.controller.attacher.resources | nindent 12 }} - - name: external-resizer + {{- with default .Values.controller.resources .Values.sidecars.attacher.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.attacher.securityContext }} securityContext: - {{- toYaml .Values.controller.securityContext | nindent 12 }} - image: "{{ .Values.controller.resizer.image.repository }}:{{ .Values.controller.resizer.image.tag }}" - imagePullPolicy: {{ .Values.controller.resizer.image.pullPolicy }} + {{- toYaml . | nindent 12 }} + {{- end }} + - name: external-resizer + image: "{{ .Values.sidecars.resizer.image.repository }}:{{ .Values.sidecars.resizer.image.tag }}" + imagePullPolicy: {{ .Values.sidecars.resizer.image.pullPolicy }} args: - - "--v={{ .Values.logVerbosityLevel }}" - - "--csi-address=$(ADDRESS)" - - "--timeout={{ .Values.timeout }}" - {{- if $enableLeaderElection }} - - "--leader-election=true" - - "--leader-election-lease-duration=120s" - - "--leader-election-renew-deadline=60s" - - "--leader-election-retry-period=30s" - {{- end}} - - "--kube-api-qps=100" - - "--kube-api-burst=100" - {{- if .Values.controller.resizer.extraArgs }} - {{- with .Values.controller.resizer.extraArgs }} - {{- tpl . $ | trim | nindent 12 }} + {{- if not (regexMatch "(-timeout)" (join " " .Values.sidecars.resizer.extraArgs)) }} + - --timeout=60s + {{- end }} + - --csi-address=$(ADDRESS) + - --v={{ .Values.sidecars.resizer.logLevel }} + - --handle-volume-inuse-error=false + {{- with .Values.sidecars.resizer.leaderElection }} + - --leader-election={{ .enabled | default true }} + {{- if .leaseDuration }} + - --leader-election-lease-duration={{ .leaseDuration }} + {{- end }} + {{- if .renewDeadline }} + - --leader-election-renew-deadline={{ .renewDeadline }} {{- end }} + {{- if .retryPeriod }} + - --leader-election-retry-period={{ .retryPeriod }} + {{- end }} + {{- end }} + {{- if not (regexMatch "(-kube-api-qps)|(-kube-api-burst)|(-workers)" (join " " .Values.sidecars.resizer.extraArgs)) }} + - --kube-api-qps=20 + - --kube-api-burst=100 + - --workers=100 + {{- end }} + {{- if not (regexMatch "(-retry-interval-max)" (join " " .Values.sidecars.resizer.extraArgs)) }} + - --retry-interval-max=30m + {{- end }} + {{- range .Values.sidecars.resizer.extraArgs }} + - {{ . }} {{- end }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock + {{- with .Values.sidecars.resizer.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: {{ toYaml .Values.controller.resizer.resources | nindent 12 }} - {{ if .Values.livenessProbe.enable }} - - name: liveness-probe + {{- with default .Values.controller.resources .Values.sidecars.resizer.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.resizer.securityContext }} securityContext: - {{- toYaml .Values.node.securityContext | nindent 12 }} - image: "{{ .Values.livenessProbe.image.repository }}:{{ .Values.livenessProbe.image.tag }}" - imagePullPolicy: {{ .Values.livenessProbe.image.pullPolicy }} + {{- toYaml . | nindent 12 }} + {{- end }} + - name: liveness-probe + image: "{{ .Values.sidecars.livenessProbe.image.repository }}:{{ .Values.sidecars.livenessProbe.image.tag }}" + imagePullPolicy: {{ .Values.sidecars.livenessProbe.image.pullPolicy }} args: - - "--v={{ .Values.logVerbosityLevel }}" - - "--csi-address=$(ADDRESS)" - {{- if .Values.livenessProbe.extraArgs }} - {{- with .Values.livenessProbe.extraArgs }} - {{- tpl . $ | trim | nindent 12 }} - {{- end }} + - --csi-address=/csi/csi.sock + {{- range .Values.sidecars.livenessProbe.extraArgs }} + - {{ . }} {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ - name: socket-dir - resources: {{ toYaml .Values.livenessProbe.resources | nindent 12 }} - {{- end }} + - name: socket-dir + mountPath: /csi + {{- with default .Values.controller.resources .Values.sidecars.livenessProbe.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.livenessProbe.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} volumes: - name: socket-dir emptyDir: {} @@ -202,18 +317,14 @@ spec: {{- with .Values.controller.volumes }} {{- toYaml . | nindent 8 }} {{- end }} - affinity: {{ toYaml .Values.controller.affinity | nindent 8 }} - nodeSelector: {{ toYaml .Values.controller.nodeSelector | nindent 8 }} - tolerations: {{ toYaml .Values.controller.tolerations | nindent 8 }} {{- with .Values.controller.hostAliases }} hostAliases: {{- toYaml . | nindent 8 }} {{- end }} - {{- if .Values.controller.priorityClassName }} - priorityClassName: {{ .Values.controller.priorityClassName }} - {{- end }} - {{- with .Values.imagePullSecrets }} + {{- if .Values.imagePullSecrets }} imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} {{- end }} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/csi-controller-role-leases.yaml b/charts/cloudstack-csi/templates/csi-controller-role-leases.yaml new file mode 100644 index 0000000..2684d4f --- /dev/null +++ b/charts/cloudstack-csi/templates/csi-controller-role-leases.yaml @@ -0,0 +1,20 @@ +{{ if .Values.controller.enabled }} +{{ if .Values.rbac.create }} +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: {{ .Release.Namespace }} + name: csi-leases-role + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +{{- end}} +{{- end}} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/csi-controller-rolebinding-leases.yaml b/charts/cloudstack-csi/templates/csi-controller-rolebinding-leases.yaml new file mode 100644 index 0000000..5fb443c --- /dev/null +++ b/charts/cloudstack-csi/templates/csi-controller-rolebinding-leases.yaml @@ -0,0 +1,24 @@ +{{ if .Values.controller.enabled }} +{{ if .Values.rbac.create }} +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-leases-rolebinding + namespace: {{ .Release.Namespace }} + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + name: {{ .Values.controller.serviceAccount.name }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: csi-leases-role + apiGroup: rbac.authorization.k8s.io +{{- end}} +{{- end}} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/csi-controller-sa.yaml b/charts/cloudstack-csi/templates/csi-controller-sa.yaml new file mode 100644 index 0000000..c819c3d --- /dev/null +++ b/charts/cloudstack-csi/templates/csi-controller-sa.yaml @@ -0,0 +1,16 @@ +{{- if .Values.controller.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.controller.serviceAccount.name }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.controller.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if .Values.controller.serviceAccount.automountServiceAccountToken }} +automountServiceAccountToken: {{ .Values.controller.serviceAccount.automountServiceAccountToken }} +{{- end }} +{{- end -}} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/csi-driver.yaml b/charts/cloudstack-csi/templates/csi-driver.yaml index 323eaa5..0e35864 100644 --- a/charts/cloudstack-csi/templates/csi-driver.yaml +++ b/charts/cloudstack-csi/templates/csi-driver.yaml @@ -1,16 +1,16 @@ -{{- if or (.Values.controller.enabled) (.Values.node.enabled) }} apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: name: csi.cloudstack.apache.org + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} spec: attachRequired: true - podInfoOnMount: true + podInfoOnMount: false # Supports only persistent volumes. volumeLifecycleModes: - - Persistent -{{- end }} + - Persistent diff --git a/charts/cloudstack-csi/templates/csi-node-clusterrole.yaml b/charts/cloudstack-csi/templates/csi-node-clusterrole.yaml new file mode 100644 index 0000000..82c46b1 --- /dev/null +++ b/charts/cloudstack-csi/templates/csi-node-clusterrole.yaml @@ -0,0 +1,25 @@ +{{ if .Values.node.enabled }} +{{ if .Values.rbac.create }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cloudstack-csi-node-role + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get"] +{{- end}} +{{- end}} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/csi-node-clusterrolebinding.yaml b/charts/cloudstack-csi/templates/csi-node-clusterrolebinding.yaml new file mode 100644 index 0000000..0debd55 --- /dev/null +++ b/charts/cloudstack-csi/templates/csi-node-clusterrolebinding.yaml @@ -0,0 +1,23 @@ +{{ if .Values.node.enabled }} +{{ if .Values.rbac.create }} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cloudstack-csi-node-binding + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + name: {{ .Values.node.serviceAccount.name }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: cloudstack-csi-node-role + apiGroup: rbac.authorization.k8s.io +{{- end}} +{{- end}} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/csi-node-ds.yaml b/charts/cloudstack-csi/templates/csi-node-ds.yaml index 125e3ea..fda2410 100644 --- a/charts/cloudstack-csi/templates/csi-node-ds.yaml +++ b/charts/cloudstack-csi/templates/csi-node-ds.yaml @@ -3,158 +3,241 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: {{ include "csi.name" . }}-node + name: {{ include "cloudstack-csi-driver.name" . }}-node namespace: {{ .Release.Namespace }} labels: - {{- include "csi.node.labels" . | nindent 4 }} + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.node.daemonSetAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} spec: + {{- if or (kindIs "float64" .Values.node.revisionHistoryLimit) (kindIs "int64" .Values.node.revisionHistoryLimit) }} + revisionHistoryLimit: {{ .Values.node.revisionHistoryLimit }} + {{- end }} selector: matchLabels: - {{- include "csi.node.matchLabels" . | nindent 6 }} + app: {{ include "cloudstack-csi-driver.name" . }}-node + {{- include "cloudstack-csi-driver.selectorLabels" . | nindent 6 }} updateStrategy: - type: RollingUpdate + {{- toYaml .Values.node.updateStrategy | nindent 4 }} template: metadata: labels: - {{- include "csi.node.labels" . | nindent 8 }} + app: {{ include "cloudstack-csi-driver.name" . }}-node + {{- include "cloudstack-csi-driver.selectorLabels" . | nindent 8 }} + {{- if .Values.node.podLabels }} + {{- toYaml .Values.node.podLabels | nindent 8 }} + {{- end }} + {{- with .Values.node.podAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 8 }} - {{- end }} + {{- end }} spec: + {{- with .Values.node.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.node.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ .Values.node.serviceAccount.name }} + terminationGracePeriodSeconds: {{ .Values.node.terminationGracePeriodSeconds }} + priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }} + tolerations: + {{- with .Values.node.tolerations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.node.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: cloudstack-csi-node - image: "{{ .Values.node.csiDriver.image.repository }}:{{ .Values.node.csiDriver.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.node.csiDriver.image.pullPolicy }} + image: "{{ .Values.node.image.repository }}:{{ .Values.node.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.node.image.pullPolicy }} args: - - "node" - - "--endpoint=$(CSI_ENDPOINT)" - - "--cloudstack-config=$(CLOUD_CONFIG)" - - "--logging-format={{ .Values.logFormat }}" - - "--node-name=$(NODE_NAME)" - - "--v={{ .Values.logVerbosityLevel }}" - {{- if .Values.node.csiDriver.extraArgs }} - {{- with .Values.node.csiDriver.extraArgs }} - {{- tpl . $ | trim | nindent 12 }} + - node + - --endpoint=$(CSI_ENDPOINT) + - --cloudstack-config=$(CLOUD_CONFIG) + {{- with .Values.node.volumeAttachLimit }} + - --volume-attach-limit={{ . }} {{- end }} + - --node-name=$(NODE_NAME) + {{- with .Values.node.loggingFormat }} + - --logging-format={{ . }} + {{- end }} + - --v={{ .Values.node.logLevel }} + {{- range .Values.node.extraArgs }} + - {{ . }} {{- end }} env: - name: CSI_ENDPOINT value: unix:///csi/csi.sock - name: CLOUD_CONFIG - value: /etc/config/cloud.conf + value: /etc/config/cloud-config - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - {{ if .Values.livenessProbe.enable }} - livenessProbe: {{- toYaml .Values.livenessProbe.livenessProbeSpec | nindent 12 }} - {{ end }} - ports: - - containerPort: 9808 - name: healthz - protocol: TCP - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true + {{- with .Values.node.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - - name: socket-dir - mountPath: /csi - name: kubelet-dir - mountPath: {{ .Values.node.kubeletDir }} - # needed so that any mounts setup inside this container are - # propagated back to the host machine. + mountPath: {{ .Values.node.kubeletPath }} mountPropagation: "Bidirectional" - - name: pods-probe-dir + - name: plugin-dir + mountPath: /csi + - name: device-dir mountPath: /dev - mountPropagation: "HostToContainer" - name: cloud-config mountPath: /etc/config - {{- with .Values.node.csiDriver.volumeMounts }} + {{- if eq .Values.node.metadataSource "cloud-init" }} + - name: cloud-init-dir + mountPath: /run/cloud-init + {{- end }} + {{- if eq .Values.node.metadataSource "ignition" }} + - name: ignition-dir + mountPath: /run/metadata + {{- end }} + {{- with .Values.node.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- with .Values.node.resources }} + resources: {{- toYaml . | nindent 12 }} {{- end }} - resources: {{ toYaml .Values.node.csiDriver.resources | nindent 12 }} - - name: node-driver-registrar + {{- with .Values.node.securityContext }} securityContext: - {{- toYaml .Values.node.securityContext | nindent 12 }} - image: "{{ .Values.node.nodeDriverRegistrar.image.repository }}:{{ .Values.node.nodeDriverRegistrar.image.tag }}" - imagePullPolicy: {{ .Values.node.nodeDriverRegistrar.image.pullPolicy }} + {{- toYaml . | nindent 12 }} + {{- end }} + - name: node-driver-registrar + image: "{{ .Values.sidecars.nodeDriverRegistrar.image.repository }}:{{ .Values.sidecars.nodeDriverRegistrar.image.tag }}" + imagePullPolicy: {{ .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} args: - - "-v={{ .Values.logVerbosityLevel }}" - - "--csi-address=$(ADDRESS)" - - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" - - "--health-port=9809" - {{- if .Values.node.nodeDriverRegistrar.extraArgs }} - {{- with .Values.node.nodeDriverRegistrar.extraArgs }} - {{- tpl . $ | trim | nindent 12 }} - {{- end }} + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --health-port=9809 + - --v={{ .Values.sidecars.nodeDriverRegistrar.logLevel }} + {{- range .Values.sidecars.nodeDriverRegistrar.extraArgs }} + - {{ . }} {{- end }} - ports: - - containerPort: 9809 - name: healthz - {{ if .Values.livenessProbe.enable }} - livenessProbe: {{- toYaml .Values.livenessProbe.livenessProbeSpec | nindent 12 }} - {{- end }} env: - name: ADDRESS value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH - value: {{ .Values.node.kubeletDir }}/plugins/csi.cloudstack.apache.org/csi.sock - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName + value: {{ printf "%s/plugins/csi.cloudstack.apache.org/csi.sock" (trimSuffix "/" .Values.node.kubeletPath) }} + {{- with .Values.sidecars.nodeDriverRegistrar.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} + ports: + - containerPort: 9809 + name: healthz + {{- with .Values.sidecars.nodeDriverRegistrar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} volumeMounts: - - mountPath: /csi - name: socket-dir + - name: plugin-dir + mountPath: /csi - name: registration-dir mountPath: /registration - resources: {{ toYaml .Values.node.nodeDriverRegistrar.resources | nindent 12 }} - {{ if .Values.livenessProbe.enable }} - - name: liveness-probe + - name: probe-dir + mountPath: {{ printf "%s/plugins/csi.cloudstack.apache.org/" (trimSuffix "/" .Values.node.kubeletPath) }} + {{- with default .Values.node.resources .Values.sidecars.nodeDriverRegistrar.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.nodeDriverRegistrar.securityContext }} securityContext: - {{- toYaml .Values.node.securityContext | nindent 12 }} - image: "{{ .Values.livenessProbe.image.repository }}:{{ .Values.livenessProbe.image.tag }}" - imagePullPolicy: {{ .Values.livenessProbe.image.pullPolicy }} + {{- toYaml . | nindent 12 }} + {{- end }} + - name: liveness-probe + image: "{{ .Values.sidecars.livenessProbe.image.repository }}:{{ .Values.sidecars.livenessProbe.image.tag }}" + imagePullPolicy: {{ .Values.sidecars.livenessProbe.image.pullPolicy }} args: - - "-v={{ .Values.logVerbosityLevel }}" - - "--csi-address=$(ADDRESS)" - {{- if .Values.livenessProbe.extraArgs }} - {{- with .Values.livenessProbe.extraArgs }} - {{- tpl . $ | trim | nindent 12 }} + - --csi-address=/csi/csi.sock + {{- range .Values.sidecars.livenessProbe.extraArgs }} + - {{ . }} {{- end }} - {{- end }} - env: - - name: ADDRESS - value: /csi/csi.sock + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - - mountPath: /csi/ - name: socket-dir - resources: {{ toYaml .Values.livenessProbe.resources | nindent 12 }} - {{ end }} + - name: plugin-dir + mountPath: /csi + {{- with default .Values.node.resources .Values.sidecars.livenessProbe.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.livenessProbe.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} volumes: - - name: socket-dir + - name: kubelet-dir + hostPath: + path: {{ .Values.node.kubeletPath }} + type: Directory + - name: plugin-dir hostPath: - path: {{ .Values.node.kubeletDir }}/plugins/csi.cloudstack.apache.org/ + path: {{ printf "%s/plugins/csi.cloudstack.apache.org/" (trimSuffix "/" .Values.node.kubeletPath) }} type: DirectoryOrCreate - name: registration-dir hostPath: - path: {{ .Values.node.kubeletDir }}/plugins_registry/ + path: {{ printf "%s/plugins_registry/" (trimSuffix "/" .Values.node.kubeletPath) }} type: Directory - - name: kubelet-dir + - name: device-dir hostPath: - path: {{ .Values.node.kubeletDir }} + path: /dev type: Directory - - name: pods-probe-dir + - name: probe-dir + {{- if .Values.node.probeDirVolume }} + {{- toYaml .Values.node.probeDirVolume | nindent 10 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- if eq .Values.node.metadataSource "cloud-init" }} + - name: cloud-init-dir hostPath: - path: /dev + path: /run/cloud-init + type: Directory + {{- end }} + {{- if eq .Values.node.metadataSource "ignition" }} + - name: ignition-dir + hostPath: + path: /run/metadata type: Directory + {{- end }} {{- if .Values.secret.enabled }} - name: cloud-config secret: @@ -165,20 +248,10 @@ spec: path: /etc/config {{- end }} {{- with .Values.node.volumes }} - {{- toYaml . | nindent 8 }} + {{- toYaml . | nindent 8 }} {{- end }} - affinity: {{ toYaml .Values.node.affinity | nindent 8 }} - nodeSelector: {{ toYaml .Values.node.nodeSelector | nindent 8 }} - tolerations: {{ toYaml .Values.node.tolerations | nindent 8 }} {{- with .Values.node.hostAliases }} hostAliases: {{- toYaml . | nindent 8 }} {{- end }} - {{- if .Values.node.priorityClassName }} - priorityClassName: {{ .Values.node.priorityClassName }} - {{- end }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} {{- end }} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/csi-node-sa.yaml b/charts/cloudstack-csi/templates/csi-node-sa.yaml new file mode 100644 index 0000000..efe14fb --- /dev/null +++ b/charts/cloudstack-csi/templates/csi-node-sa.yaml @@ -0,0 +1,16 @@ +{{- if .Values.node.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.node.serviceAccount.name }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.node.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if .Values.node.serviceAccount.automountServiceAccountToken }} +automountServiceAccountToken: {{ .Values.node.serviceAccount.automountServiceAccountToken }} +{{- end }} +{{- end -}} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/csi-sa.yaml b/charts/cloudstack-csi/templates/csi-sa.yaml deleted file mode 100644 index 61ffc42..0000000 --- a/charts/cloudstack-csi/templates/csi-sa.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{ if .Values.serviceAccount.create }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ .Values.serviceAccount.name }} - namespace: {{ .Release.Namespace }} - annotations: - {{- with .Values.commonAnnotations }} - {{- toYaml . | nindent 4 }} - {{- end }} -{{ end }} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/podmonitor.yaml b/charts/cloudstack-csi/templates/podmonitor.yaml index 736dca2..34675b3 100644 --- a/charts/cloudstack-csi/templates/podmonitor.yaml +++ b/charts/cloudstack-csi/templates/podmonitor.yaml @@ -2,14 +2,14 @@ apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: - labels: - {{- include "csi.controller.labels" . | nindent 4 }} - name: {{ include "csi.name" . }}-controller + name: {{ include "cloudstack-csi-driver.name" . }}-controller namespace: {{ .Release.Namespace }} + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} spec: podMetricsEndpoints: - interval: 30s @@ -18,5 +18,5 @@ spec: jobLabel: component selector: matchLabels: - {{- include "csi.controller.matchLabels" . | nindent 6 }} + {{- include "cloudstack-csi-driver.selectorLabels" . | nindent 6 }} {{- end }} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/secret.yaml b/charts/cloudstack-csi/templates/secret.yaml index fc0f565..c9e91a6 100644 --- a/charts/cloudstack-csi/templates/secret.yaml +++ b/charts/cloudstack-csi/templates/secret.yaml @@ -4,11 +4,13 @@ kind: Secret metadata: name: {{ .Values.secret.name | default "cloud.conf" }} namespace: {{ .Release.Namespace }} + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} type: Opaque data: - cloud.conf: {{ include "cloudConfig" . | b64enc }} + cloud-config: {{ include "cloudConfig" . | b64enc }} {{- end }} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/syncer-job.yaml b/charts/cloudstack-csi/templates/syncer-job.yaml index 0009b95..be155d4 100644 --- a/charts/cloudstack-csi/templates/syncer-job.yaml +++ b/charts/cloudstack-csi/templates/syncer-job.yaml @@ -4,38 +4,47 @@ apiVersion: batch/v1 kind: Job metadata: - name: {{ include "csi.name" . }}-sc-syncer + name: {{ include "cloudstack-csi-driver.name" . }}-sc-syncer namespace: {{ .Release.Namespace }} + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} spec: backoffLimit: {{ .Values.syncer.backoffLimit }} template: spec: - securityContext: {{- toYaml .Values.syncer.securityContext | nindent 12 }} - serviceAccountName: {{ .Values.syncer.rbac.serviceAccount.name }} + securityContext: {{- toYaml .Values.syncer.podSecurityContext | nindent 8 }} + serviceAccountName: {{ .Values.syncer.serviceAccount.name }} containers: - - image: "{{ .Values.syncer.image.repository }}:{{ .Values.syncer.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.syncer.image.imagePullPolicy }} - name: {{ include "csi.name" . }}-sc-syncer - args: - - "--cloudstackconfig=$(CLOUD_CONFIG)" - - "--kubeconfig=-" - - "--volumeExpansion=true" - {{- if .Values.syncer.extraArgs }} - {{- with .Values.syncer.extraArgs }} - {{- tpl . $ | trim | nindent 12 }} + - image: "{{ .Values.syncer.image.repository }}:{{ .Values.syncer.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.syncer.image.imagePullPolicy }} + name: {{ include "cloudstack-csi-driver.name" . }}-sc-syncer + args: + - "--cloudstackconfig=$(CLOUD_CONFIG)" + - "--kubeconfig=-" + - "--volumeExpansion=true" + {{- if .Values.syncer.extraArgs }} + {{- with .Values.syncer.extraArgs }} + {{- tpl . $ | trim | nindent 12 }} + {{- end }} + {{- end }} + env: + - name: CLOUD_CONFIG + value: /etc/config/cloud-config + volumeMounts: + - name: cloud-config + mountPath: /etc/config/ + {{- with .Values.syncer.resources }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} + {{- with .Values.syncer.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} {{- end }} - env: - - name: CLOUD_CONFIG - value: /etc/config/cloud.conf - volumeMounts: - - name: cloud-config - mountPath: /etc/config/ - resources: {{ toYaml .Values.controller.resizer.resources | nindent 12 }} volumes: {{- if .Values.secret.enabled }} - name: cloud-config @@ -46,7 +55,13 @@ spec: hostPath: path: /etc/cloudstack-csi-driver {{- end }} - affinity: {{ toYaml .Values.syncer.affinity | nindent 8 }} - nodeSelector: {{ toYaml .Values.syncer.nodeSelector | nindent 8 }} + {{- with .Values.syncer.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.syncer.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} restartPolicy: {{ .Values.syncer.restartPolicy }} {{- end }} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/syncer-rbac.yaml b/charts/cloudstack-csi/templates/syncer-rbac.yaml index a9b534d..87f22bd 100644 --- a/charts/cloudstack-csi/templates/syncer-rbac.yaml +++ b/charts/cloudstack-csi/templates/syncer-rbac.yaml @@ -1,24 +1,31 @@ {{ if .Values.syncer.enabled }} -{{ if .Values.syncer.rbac.create }} +{{ if .Values.syncer.serviceAccount.create }} --- apiVersion: v1 kind: ServiceAccount metadata: - name: {{ .Values.syncer.rbac.serviceAccount.name }} + name: {{ .Values.syncer.serviceAccount.name }} namespace: {{ .Release.Namespace }} + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.syncer.serviceAccount.annotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} +{{- if .Values.syncer.serviceAccount.automountServiceAccountToken }} +automountServiceAccountToken: {{ .Values.syncer.serviceAccount.automountServiceAccountToken }} +{{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: csi-sc-syncer-role + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} rules: - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] @@ -28,13 +35,15 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: csi-sc-syncer-binding + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} subjects: - kind: ServiceAccount - name: {{ .Values.syncer.rbac.serviceAccount.name }} + name: {{ .Values.syncer.serviceAccount.name }} namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole diff --git a/charts/cloudstack-csi/values.yaml b/charts/cloudstack-csi/values.yaml index f4b25d8..9ca492c 100644 --- a/charts/cloudstack-csi/values.yaml +++ b/charts/cloudstack-csi/values.yaml @@ -1,8 +1,8 @@ -extraLabels: {} - nameOverride: "cloudstack-csi" fullnameOverride: "" -timeout: 300s + +# Additional labels to add to all resources +extraLabels: {} ## Annotations to apply to all resources commonAnnotations: {} @@ -11,76 +11,227 @@ commonAnnotations: {} # "helm.sh/hook-weight": "-100" # "helm.sh/hook-delete-policy": before-hook-creation -# Create Prometheus Operator PodMonitor. Requires http server above. +# Create Prometheus Operator PodMonitor. # See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.PodMonitor podMonitor: enabled: false -## CloudStack CSI Controller Defaults -controller: - enabled: true +# -- Custom labels to add into metadata +customLabels: {} + +sidecars: + provisioner: + env: [] + image: + pullPolicy: IfNotPresent + repository: registry.k8s.io/sig-storage/csi-provisioner + tag: "v5.0.1" + logLevel: 2 + # Feature gates to enable + featureGates: "Topology=true" + # Extra arguments passed to external-provisioner. + extraArgs: [] + resources: {} + # Tune leader lease election for csi-provisioner. + # Leader election is on by default. + leaderElection: + enabled: true + # Optional values to tune lease behavior. + # The arguments provided must be in an acceptable time.ParseDuration format. + # Ref: https://pkg.go.dev/flag#Duration + # leaseDuration: "15s" + # renewDeadline: "10s" + # retryPeriod: "5s" + securityContext: + seccompProfile: + type: RuntimeDefault + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false attacher: + env: [] image: - repository: registry.k8s.io/sig-storage/csi-attacher - tag: v4.6.1 pullPolicy: IfNotPresent + repository: registry.k8s.io/sig-storage/csi-attacher + tag: "v4.6.1" + # Tune leader lease election for csi-attacher. + # Leader election is on by default. + leaderElection: + enabled: true + # Optional values to tune lease behavior. + # The arguments provided must be in an acceptable time.ParseDuration format. + # Ref: https://pkg.go.dev/flag#Duration + # leaseDuration: "15s" + # renewDeadline: "10s" + # retryPeriod: "5s" + logLevel: 2 + # Extra arguments passed to external-attacher. + extraArgs: [] resources: {} - extraArgs: {} - provisioner: - topology: "true" + securityContext: + seccompProfile: + type: RuntimeDefault + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + livenessProbe: image: - repository: registry.k8s.io/sig-storage/csi-provisioner - tag: v5.0.1 pullPolicy: IfNotPresent + repository: registry.k8s.io/sig-storage/livenessprobe + tag: "v2.12.0" + # Extra arguments passed to livenessprobe. + extraArgs: [] resources: {} - extraArgs: {} + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false resizer: + env: [] image: - repository: registry.k8s.io/sig-storage/csi-resizer - tag: v1.11.1 pullPolicy: IfNotPresent + repository: registry.k8s.io/sig-storage/csi-resizer + tag: "v1.11.1" + # Tune leader lease election for csi-resizer. + # Leader election is on by default. + leaderElection: + enabled: true + # Optional values to tune lease behavior. + # The arguments provided must be in an acceptable time.ParseDuration format. + # Ref: https://pkg.go.dev/flag#Duration + # leaseDuration: "15s" + # renewDeadline: "10s" + # retryPeriod: "5s" + logLevel: 2 + # Extra arguments passed to external-resizer. + extraArgs: [] resources: {} - extraArgs: {} - csiDriverController: + securityContext: + seccompProfile: + type: RuntimeDefault + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + nodeDriverRegistrar: + env: [] image: - repository: ghcr.io/leaseweb/cloudstack-csi-driver pullPolicy: IfNotPresent - # tag: 0.4.1 # defaults to .Chart.AppVersion - volumeMounts: {} + repository: registry.k8s.io/sig-storage/csi-node-driver-registrar + tag: "v2.10.1" + logLevel: 2 + # Extra arguments passed to node-driver-registrar. + extraArgs: [] resources: {} - extraArgs: {} - + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + periodSeconds: 90 + timeoutSeconds: 15 + +## CloudStack CSI Controller Defaults +controller: + enabled: true + image: + repository: ghcr.io/leaseweb/cloudstack-csi-driver + # Overrides the image tag whose default is v{{ .Chart.AppVersion }} + tag: "" + pullPolicy: IfNotPresent ## Set number of replicas (If replicaCount set to 1 by default leader-elect will be false) - replicaCount: 1 - strategy: + replicaCount: 2 + # Extra arguments passed to cloudstack-csi-driver controller. + extraArgs: [] + loggingFormat: text + logLevel: 2 + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/name" + operator: In + values: + - cloudstack-csi-controller + topologyKey: "kubernetes.io/hostname" + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + # The default filesystem type of the volume to provision when fstype is unspecified in the StorageClass. + # If the default is not set and fstype is unset in the StorageClass, then no fstype will be set + defaultFsType: ext4 + env: [] + # Use envFrom to reference ConfigMaps and Secrets across all containers in the deployment + envFrom: [] + # If set, add pv/pvc metadata to plugin create requests as parameters. + extraCreateMetadata: true + nodeSelector: {} + deploymentAnnotations: {} + podAnnotations: {} + podLabels: {} + priorityClassName: system-cluster-critical + revisionHistoryLimit: 10 + updateStrategy: # RollingUpdate strategy replaces old pods with new ones gradually, # without incurring downtime. type: RollingUpdate rollingUpdate: # maxUnavailable is the maximum number of pods that can be # unavailable during the update process. - maxUnavailable: 0 + maxUnavailable: 1 # maxSurge is the maximum number of pods that can be # created over the desired number of pods. - maxSurge: 1 - priorityClassName: "" + maxSurge: 0 + resources: + requests: + cpu: 10m + memory: 40Mi + limits: + memory: 256Mi + serviceAccount: + # A service account will be created for you if set to true. Set to false if you want to use your own. + create: true + name: cloudstack-csi-controller + annotations: {} + automountServiceAccountToken: true + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + # securityContext on the controller pod podSecurityContext: runAsNonRoot: true runAsUser: 65532 runAsGroup: 65532 fsGroup: 65532 fsGroupChangePolicy: OnRootMismatch - securityContext: {} - # capabilities: - # drop: - # - ALL - # seccompProfile: - # type: RuntimeDefault - # readOnlyRootFilesystem: true - volumes: {} - affinity: {} - nodeSelector: {} - tolerations: [] + # securityContext on the controller container (see sidecars for securityContext on sidecar containers) + securityContext: + seccompProfile: + type: RuntimeDefault + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + # Add additional volume mounts on the controller with controller.volumes and controller.volumeMounts + volumes: [] + # Add additional volumes to be mounted onto the controller: + # - name: custom-dir + # hostPath: + # path: /path/to/dir + # type: Directory + volumeMounts: [] + # And add mount paths for those additional volumes: + # - name: custom-dir + # mountPath: /mount/path + # # Allow for specifying internal IP addresses for multiple hostnames hostAliases: {} # - ip: "10.0.0.1" @@ -90,97 +241,128 @@ controller: ## CloudStack CSI Node Defaults node: enabled: true - nodeDriverRegistrar: - image: - repository: registry.k8s.io/sig-storage/csi-node-driver-registrar - tag: v2.10.1 - pullPolicy: IfNotPresent - resources: {} - extraArgs: {} - csiDriver: - image: - repository: ghcr.io/leaseweb/cloudstack-csi-driver - pullPolicy: IfNotPresent - # tag: 0.4.1 # defaults to .Chart.AppVersion - resources: {} - extraArgs: {} - volumeMounts: {} - volumes: {} + image: + repository: ghcr.io/leaseweb/cloudstack-csi-driver + # Overrides the image tag whose default is v{{ .Chart.AppVersion }} + tag: "" + pullPolicy: IfNotPresent + env: [] + envFrom: [] + kubeletPath: /var/lib/kubelet + loggingFormat: text + logLevel: 2 priorityClassName: "" - podSecurityContext: {} - - securityContext: {} - # capabilities: - # drop: - # - ALL - # seccompProfile: - # type: RuntimeDefault - + # Extra arguments passed to cloudstack-csi-driver controller. + extraArgs: [] affinity: {} - nodeSelector: {} - + daemonSetAnnotations: {} + podAnnotations: {} + podLabels: {} + terminationGracePeriodSeconds: 30 tolerations: - - operator: Exists - - kubeletDir: /var/lib/kubelet - + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + resources: + requests: + cpu: 10m + memory: 40Mi + limits: + memory: 256Mi + revisionHistoryLimit: 10 + probeDirVolume: + emptyDir: {} + serviceAccount: + create: true + name: cloudstack-csi-node + annotations: {} + automountServiceAccountToken: true + # Metadata source to try to find instance ID. + # Possible values 'cloud-init' & 'ignition' + metadataSource: ignition + # The maximum number of volumes that can be attached to a node + volumeAttachLimit: + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: "10%" + # securityContext on the node pod + podSecurityContext: + # The node pod must be run as root to bind to the registration/driver sockets + runAsNonRoot: false + runAsUser: 0 + runAsGroup: 0 + fsGroup: 0 + # securityContext on the node container (see sidecars for securityContext on sidecar containers) + # Privileged containers always run as `Unconfined`, which means that they are not restricted by a seccomp profile. + securityContext: + # readOnlyRootFilesystem: true + privileged: true + # Add additional volume mounts on the node pods with node.volumes and node.volumeMounts + volumes: [] + # Add additional volumes to be mounted onto the node pods: + # - name: custom-dir + # hostPath: + # path: /path/to/dir + # type: Directory + volumeMounts: [] + # And add mount paths for those additional volumes: + # - name: custom-dir + # mountPath: /mount/path + # # Allow for specifying internal IP addresses for multiple hostnames hostAliases: # - ip: "10.0.0.1" # hostnames: # - "keystone.hostname.com" -## CloudStack Storage Class Synecr Job Defaults +## CloudStack Storage Class Syncer Job Defaults syncer: enabled: true # Job image image: repository: "ghcr.io/leaseweb/cloudstack-csi-sc-syncer" - # tag: "0.4.1" # defaults to .Chart.AppVersion + # Overrides the image tag. Default is {{ .Chart.AppVersion }} + tag: "" imagePullPolicy: IfNotPresent + # Extra arguments passed to cloudstack-csi-sc-syncer. extraArgs: {} + affinity: {} + nodeSelector: {} + podAnnotations: {} + podLabels: {} + resources: {} + serviceAccount: + create: true + name: cloudstack-csi-sc-syncer + annotations: {} + automountServiceAccountToken: true # Job configurations backoffLimit: 4 restartPolicy: Never - rbac: - create: true - serviceAccount: - name: cloudstack-csi-sc-syncer - - # Define resources - # resources: - # limits: - # cpu: 2 - # memory: 2000Mi - # requests: - # cpu: 500m - # memory: 500Mi - - securityContext: {} - # capabilities: - # drop: - # - ALL - # seccompProfile: - # type: RuntimeDefault - - affinity: {} - - nodeSelector: {} - -# Log verbosity level. -# See https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md -# for description of individual verbosity levels. -logVerbosityLevel: 2 - -# Log format. Available options are "text" and "json" -logFormat: text + # securityContext on the syncer job + # securityContext on the controller pod + podSecurityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + # securityContext on the controller container (see sidecars for securityContext on sidecar containers) + securityContext: + seccompProfile: + type: RuntimeDefault + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false -# the secret should contain the cloudstack credentials -# there are several options to inject the credentials: +# The secret should contain the CloudStack credentials +# There are several options to inject the credentials: # 1) from kubernetes secret that doesn't exist: set "enabled" and "create" to true, this will create a secret from the values written to "data" down below # 2) from kubernetes secret that already exists: set "enabled" to true and "create" to false # 3) from host system path /etc/cloud/cloud.conf: set "enabled" to false and "hostMount" to true @@ -188,7 +370,7 @@ logFormat: text secret: enabled: true create: false - name: cloud-config + name: cloudstack-secret cloudConfigData: global: api-url: "" @@ -199,28 +381,5 @@ secret: imagePullSecrets: [] # - name: my-imagepull-secret -serviceAccount: - create: true - name: "cloudstack-csi-controller" - -rbac: +rbac: create: true - -livenessProbe: - enable: true - image: - repository: registry.k8s.io/sig-storage/livenessprobe - tag: v2.12.0 - pullPolicy: IfNotPresent - resources: {} - extraArgs: {} - livenessProbeSpec: - httpGet: - path: /healthz - port: healthz - failureThreshold: 5 - initialDelaySeconds: 60 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 15 - diff --git a/deploy/k8s/csidriver.yaml b/deploy/k8s/csidriver.yaml index dbdeb46..f5443e7 100644 --- a/deploy/k8s/csidriver.yaml +++ b/deploy/k8s/csidriver.yaml @@ -3,6 +3,8 @@ kind: CSIDriver metadata: name: csi.cloudstack.apache.org spec: + attachRequired: true + podInfoOnMount: false # Supports only persistent volumes. volumeLifecycleModes: - Persistent