From c9c7caef7a291ba73286de57e431c67bf2fade0c Mon Sep 17 00:00:00 2001 From: Hans Rakers Date: Thu, 4 Jul 2024 10:54:10 +0200 Subject: [PATCH 1/5] refactor(chart): Complete refactor of chart * Seperated sidecar settings * Improved security w/ seccomp profiles, read only fs * Adjusted node-driver-registrar liveness probe * Improve security of node daemonset by using a service account & cluster role * Reduce leases RBAC to a role instead of a clusterrole permission * Disable podInfoOnMount in CSIDriver (not needed) * Allow config of metadata source (cloud-init/ignition) * Allow config of volume attach limit --- charts/cloudstack-csi/templates/_helpers.tpl | 67 +-- .../templates/csi-clusterrole.yaml | 39 +- ...=> csi-controller-clusterrolebinding.yaml} | 26 +- .../templates/csi-controller-deploy.yaml | 349 ++++++++++----- .../templates/csi-controller-role-leases.yaml | 20 + .../csi-controller-rolebinding-leases.yaml | 24 + .../templates/csi-controller-sa.yaml | 16 + .../cloudstack-csi/templates/csi-driver.yaml | 12 +- .../templates/csi-node-clusterrole.yaml | 25 ++ .../csi-node-clusterrolebinding.yaml | 23 + .../cloudstack-csi/templates/csi-node-ds.yaml | 281 +++++++----- .../cloudstack-csi/templates/csi-node-sa.yaml | 16 + charts/cloudstack-csi/templates/csi-sa.yaml | 11 - .../cloudstack-csi/templates/podmonitor.yaml | 12 +- charts/cloudstack-csi/templates/secret.yaml | 8 +- .../cloudstack-csi/templates/syncer-job.yaml | 63 ++- .../cloudstack-csi/templates/syncer-rbac.yaml | 27 +- charts/cloudstack-csi/values.yaml | 423 ++++++++++++------ deploy/k8s/csidriver.yaml | 2 + 19 files changed, 942 insertions(+), 502 deletions(-) rename charts/cloudstack-csi/templates/{csi-clusterrolebinding.yaml => csi-controller-clusterrolebinding.yaml} (66%) create mode 100644 charts/cloudstack-csi/templates/csi-controller-role-leases.yaml create mode 100644 charts/cloudstack-csi/templates/csi-controller-rolebinding-leases.yaml create mode 100644 charts/cloudstack-csi/templates/csi-controller-sa.yaml create mode 100644 charts/cloudstack-csi/templates/csi-node-clusterrole.yaml create mode 100644 charts/cloudstack-csi/templates/csi-node-clusterrolebinding.yaml create mode 100644 charts/cloudstack-csi/templates/csi-node-sa.yaml delete mode 100644 charts/cloudstack-csi/templates/csi-sa.yaml diff --git a/charts/cloudstack-csi/templates/_helpers.tpl b/charts/cloudstack-csi/templates/_helpers.tpl index 8d5dd4d..0985bf6 100644 --- a/charts/cloudstack-csi/templates/_helpers.tpl +++ b/charts/cloudstack-csi/templates/_helpers.tpl @@ -2,7 +2,7 @@ {{/* Expand the name of the chart. */}} -{{- define "csi.name" -}} +{{- define "cloudstack-csi-driver.name" -}} {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} {{- end -}} @@ -11,7 +11,7 @@ Create a default fully qualified app name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). If release name contains chart name it will be used as a full name. */}} -{{- define "csi.fullname" -}} +{{- define "cloudstack-csi-driver.fullname" -}} {{- if .Values.fullnameOverride -}} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} {{- else -}} @@ -27,73 +27,36 @@ If release name contains chart name it will be used as a full name. {{/* Create chart name and version as used by the chart label. */}} -{{- define "csi.chart" -}} +{{- define "cloudstack-csi-driver.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} {{/* Common labels */}} -{{- define "csi.labels" -}} -app.kubernetes.io/name: {{ include "csi.name" . }} -helm.sh/chart: {{ include "csi.chart" . }} -app.kubernetes.io/instance: {{ .Release.Name }} +{{- define "cloudstack-csi-driver.labels" -}} +{{ include "cloudstack-csi-driver.selectorLabels" . }} +helm.sh/chart: {{ include "cloudstack-csi-driver.chart" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} +app.kubernetes.io/component: csi-driver app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end -}} - - -{{/* -Create the name of the service account to use -*/}} -{{- define "csi.serviceAccountName" -}} -{{- if .Values.serviceAccount.create -}} - {{ default (include "csi.fullname" .) .Values.serviceAccount.name }} -{{- else -}} - {{ default "default" .Values.serviceAccount.name }} -{{- end -}} +{{- if .Values.customLabels }} +{{ toYaml .Values.extraLabels }} +{{- end }} {{- end -}} {{/* -Create unified labels for csi components +Common selector labels */}} -{{- define "csi.common.matchLabels" -}} -app: {{ template "csi.name" . }} -release: {{ .Release.Name }} -{{- end -}} - -{{- define "csi.common.metaLabels" -}} -chart: {{ template "csi.chart" . }} -heritage: {{ .Release.Service }} -{{- if .Values.extraLabels }} -{{ toYaml .Values.extraLabels -}} -{{- end }} -{{- end -}} - -{{- define "csi.controller.matchLabels" -}} -component: controller -{{ include "csi.common.matchLabels" . }} -{{- end -}} - -{{- define "csi.controller.labels" -}} -{{ include "csi.controller.matchLabels" . }} -{{ include "csi.common.metaLabels" . }} -{{- end -}} - -{{- define "csi.node.matchLabels" -}} -component: node -{{ include "csi.common.matchLabels" . }} -{{- end -}} - -{{- define "csi.node.labels" -}} -{{ include "csi.node.matchLabels" . }} -{{ include "csi.common.metaLabels" . }} +{{- define "cloudstack-csi-driver.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cloudstack-csi-driver.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} {{- end -}} {{/* -Create cloud-config makro. +Create cloud-config macro. */}} {{- define "cloudConfig" -}} [Global] diff --git a/charts/cloudstack-csi/templates/csi-clusterrole.yaml b/charts/cloudstack-csi/templates/csi-clusterrole.yaml index 2c956a7..f82003d 100644 --- a/charts/cloudstack-csi/templates/csi-clusterrole.yaml +++ b/charts/cloudstack-csi/templates/csi-clusterrole.yaml @@ -1,18 +1,17 @@ {{ if .Values.controller.enabled }} {{ if .Values.rbac.create }} -# This YAML file contains Cluster Role objects, -# which are necessary to run cloudstack-csi-controller -{{ $enableLeaderElection := gt (int .Values.controller.replicaCount) 1 }} --- # external Attacher kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-attacher-role + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} rules: - apiGroups: [""] resources: ["persistentvolumes"] @@ -26,22 +25,18 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments/status"] verbs: ["patch"] - {{- if $enableLeaderElection}} - # leader election - - apiGroups: [coordination.k8s.io] - resources: [leases] - verbs: ["get", "watch", "list", "delete", "update", "create"] - {{- end}} --- # external Provisioner kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-provisioner-role + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} rules: - apiGroups: [""] resources: ["persistentvolumes"] @@ -70,18 +65,18 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["get", "list", "watch"] - {{- if $enableLeaderElection}} - # leader election - - apiGroups: [coordination.k8s.io] - resources: [leases] - verbs: ["get", "watch", "list", "delete", "update", "create"] - {{- end}} --- # external Resizer kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-resizer-role + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} rules: # The following rule should be uncommented for plugins that require secrets # for provisioning. @@ -103,11 +98,5 @@ rules: - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create", "update", "patch"] - {{- if $enableLeaderElection}} - # leader election - - apiGroups: [coordination.k8s.io] - resources: [leases] - verbs: ["get", "watch", "list", "delete", "update", "create"] - {{- end}} {{- end}} {{- end}} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/csi-clusterrolebinding.yaml b/charts/cloudstack-csi/templates/csi-controller-clusterrolebinding.yaml similarity index 66% rename from charts/cloudstack-csi/templates/csi-clusterrolebinding.yaml rename to charts/cloudstack-csi/templates/csi-controller-clusterrolebinding.yaml index 4420bc8..5b54315 100644 --- a/charts/cloudstack-csi/templates/csi-clusterrolebinding.yaml +++ b/charts/cloudstack-csi/templates/csi-controller-clusterrolebinding.yaml @@ -1,19 +1,19 @@ {{ if .Values.controller.enabled }} {{ if .Values.rbac.create }} -# This YAML file contains Cluster Role Binding objects, -# which are necessary to run cloudstack-csi-controller --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-attacher-binding + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} subjects: - kind: ServiceAccount - name: {{ .Values.serviceAccount.name }} + name: {{ .Values.controller.serviceAccount.name }} namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole @@ -24,13 +24,15 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-provisioner-binding + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} subjects: - kind: ServiceAccount - name: {{ .Values.serviceAccount.name }} + name: {{ .Values.controller.serviceAccount.name }} namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole @@ -41,13 +43,15 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: csi-resizer-binding + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} subjects: - kind: ServiceAccount - name: {{ .Values.serviceAccount.name }} + name: {{ .Values.controller.serviceAccount.name }} namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole diff --git a/charts/cloudstack-csi/templates/csi-controller-deploy.yaml b/charts/cloudstack-csi/templates/csi-controller-deploy.yaml index 5d751ad..5e0dc98 100644 --- a/charts/cloudstack-csi/templates/csi-controller-deploy.yaml +++ b/charts/cloudstack-csi/templates/csi-controller-deploy.yaml @@ -1,192 +1,307 @@ {{ if .Values.controller.enabled }} -{{ $enableLeaderElection := gt (int .Values.controller.replicaCount) 1 }} - apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "csi.name" . }}-controller + name: {{ include "cloudstack-csi-driver.name" . }}-controller namespace: {{ .Release.Namespace }} labels: - {{- include "csi.controller.labels" . | nindent 4 }} + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.controller.deploymentAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} spec: replicas: {{ .Values.controller.replicaCount }} + {{- if or (kindIs "float64" .Values.controller.revisionHistoryLimit) (kindIs "int64" .Values.controller.revisionHistoryLimit) }} + revisionHistoryLimit: {{ .Values.controller.revisionHistoryLimit }} + {{- end }} + {{- with .Values.controller.updateStrategy }} strategy: - type: {{ .Values.controller.strategy.type }} -{{- if eq .Values.controller.strategy.type "RollingUpdate" }} - rollingUpdate: - maxUnavailable: {{ .Values.controller.strategy.rollingUpdate.maxUnavailable }} - maxSurge: {{ .Values.controller.strategy.rollingUpdate.maxSurge }} -{{- end }} + {{- toYaml . | nindent 4 }} + {{- end }} selector: matchLabels: - {{- include "csi.controller.matchLabels" . | nindent 6 }} + app: {{ include "cloudstack-csi-driver.name" . }}-controller + {{- include "cloudstack-csi-driver.selectorLabels" . | nindent 6 }} template: metadata: labels: - {{- include "csi.controller.labels" . | nindent 8 }} + app: {{ include "cloudstack-csi-driver.name" . }}-controller + {{- include "cloudstack-csi-driver.labels" . | nindent 8 }} + {{- if .Values.controller.podLabels }} + {{- toYaml .Values.controller.podLabels | nindent 8 }} + {{- end }} + {{- if .Values.controller.podAnnotations }} annotations: - {{- with .Values.commonAnnotations }} + {{- tpl ( .Values.controller.podAnnotations | toYaml ) . | nindent 8 }} + {{- end }} + spec: + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.controller.nodeSelector }} {{- toYaml . | nindent 8 }} {{- end }} - spec: - serviceAccount: cloudstack-csi-controller + serviceAccount: {{ .Values.controller.serviceAccount.name }} + priorityClassName: {{ .Values.controller.priorityClassName }} + {{- with default .Values.controller.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + tolerations: + {{- with .Values.controller.tolerations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.controller.podSecurityContext }} securityContext: - {{- toYaml .Values.controller.podSecurityContext | nindent 8 }} + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: cloudstack-csi-controller - securityContext: - {{- toYaml .Values.controller.securityContext | nindent 12 }} - image: "{{ .Values.controller.csiDriverController.image.repository }}:{{ .Values.controller.csiDriverController.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.controller.csiDriverController.image.pullPolicy }} + image: "{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.controller.image.pullPolicy }} args: - "controller" - "--endpoint=$(CSI_ENDPOINT)" - "--cloudstack-config=$(CLOUD_CONFIG)" - - "--logging-format={{ .Values.logFormat }}" - - "--v={{ .Values.logVerbosityLevel }}" - {{- if .Values.controller.csiDriverController.extraArgs }} - {{- with .Values.controller.csiDriverController.extraArgs }} - {{- tpl . $ | trim | nindent 12 }} - {{- end }} + - "--logging-format={{ .Values.controller.loggingFormat }}" + - "--v={{ .Values.controller.logLevel }}" + {{- range .Values.controller.extraArgs }} + - {{ . }} {{- end }} env: - name: CSI_ENDPOINT value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - name: CLOUD_CONFIG - value: /etc/config/cloud.conf + value: /etc/config/cloud-config + {{- with .Values.controller.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - name: cloud-config mountPath: /etc/config/ - {{- with .Values.controller.csiDriverController.volumeMounts }} - {{- toYaml . | nindent 12 }} + {{- with .Values.controller.volumeMounts }} + {{- toYaml . | nindent 12 }} {{- end }} ports: - containerPort: 9808 name: healthz protocol: TCP - {{ if .Values.livenessProbe.enable }} - livenessProbe: {{- toYaml .Values.livenessProbe.livenessProbeSpec | nindent 12 }} + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- with .Values.controller.resources }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} - - name: external-provisioner + {{- with .Values.controller.securityContext }} securityContext: - {{- toYaml .Values.controller.securityContext | nindent 12 }} - image: "{{ .Values.controller.provisioner.image.repository }}:{{ .Values.controller.provisioner.image.tag }}" - imagePullPolicy: {{ .Values.controller.provisioner.image.pullPolicy }} + {{- toYaml . | nindent 12 }} + {{- end }} + - name: external-provisioner + image: "{{ .Values.sidecars.provisioner.image.repository }}:{{ .Values.sidecars.provisioner.image.tag }}" + imagePullPolicy: {{ .Values.sidecars.provisioner.image.pullPolicy }} args: - - "--v={{ .Values.logVerbosityLevel }}" - - "--csi-address=$(ADDRESS)" - - "--timeout={{ .Values.timeout }}" - - "--feature-gates=Topology={{ .Values.controller.provisioner.topology }}" - - "--timeout=300s" - - "--kube-api-qps=100" - - "--kube-api-burst=100" - {{- if $enableLeaderElection }} - - "--leader-election=true" - - "--leader-election-lease-duration=120s" - - "--leader-election-renew-deadline=60s" - - "--leader-election-retry-period=30s" - {{- end}} - - "--default-fstype=ext4" - - "--strict-topology" - {{- if .Values.controller.provisioner.extraArgs }} - {{- with .Values.controller.provisioner.extraArgs }} - {{- tpl . $ | trim | nindent 12 }} + {{- if not (regexMatch "(-timeout)" (join " " .Values.sidecars.provisioner.extraArgs)) }} + - --timeout=60s + {{- end }} + - --csi-address=$(ADDRESS) + - --v={{ .Values.sidecars.provisioner.logLevel }} + {{- if .Values.sidecars.provisioner.featureGates }} + - --feature-gates={{ .Values.sidecars.provisioner.featureGates }} + {{- end }} + {{- if .Values.controller.extraCreateMetadata }} + - --extra-create-metadata + {{- end }} + - --leader-election={{ .Values.sidecars.provisioner.leaderElection.enabled | required "leader election state for csi-provisioner is required, must be set to true || false." }} + {{- if .Values.sidecars.provisioner.leaderElection.enabled }} + {{- if .Values.sidecars.provisioner.leaderElection.leaseDuration }} + - --leader-election-lease-duration={{ .Values.sidecars.provisioner.leaderElection.leaseDuration }} + {{- end }} + {{- if .Values.sidecars.provisioner.leaderElection.renewDeadline}} + - --leader-election-renew-deadline={{ .Values.sidecars.provisioner.leaderElection.renewDeadline }} + {{- end }} + {{- if .Values.sidecars.provisioner.leaderElection.retryPeriod }} + - --leader-election-retry-period={{ .Values.sidecars.provisioner.leaderElection.retryPeriod }} + {{- end }} + {{- end }} + - --default-fstype={{ .Values.controller.defaultFsType }} + {{- if not (regexMatch "(-kube-api-qps)|(-kube-api-burst)|(-worker-threads)" (join " " .Values.sidecars.provisioner.extraArgs)) }} + - --kube-api-qps=20 + - --kube-api-burst=100 + - --worker-threads=100 + {{- end }} + {{- if not (regexMatch "(-retry-interval-max)" (join " " .Values.sidecars.provisioner.extraArgs)) }} + - --retry-interval-max=30m {{- end }} + - --strict-topology + {{- range .Values.sidecars.provisioner.extraArgs }} + - {{ . }} {{- end }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock + {{- with .Values.sidecars.provisioner.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: {{ toYaml .Values.controller.provisioner.resources | nindent 12 }} - - name: external-attacher + {{- with default .Values.controller.resources .Values.sidecars.provisioner.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.provisioner.securityContext }} securityContext: - {{- toYaml .Values.controller.securityContext | nindent 12 }} - image: "{{ .Values.controller.attacher.image.repository }}:{{ .Values.controller.attacher.image.tag }}" - imagePullPolicy: {{ .Values.controller.attacher.image.pullPolicy }} + {{- toYaml . | nindent 12 }} + {{- end }} + - name: external-attacher + image: "{{ .Values.sidecars.attacher.image.repository }}:{{ .Values.sidecars.attacher.image.tag }}" + imagePullPolicy: {{ .Values.sidecars.attacher.image.pullPolicy }} args: - - "--v={{ .Values.logVerbosityLevel }}" - - "--csi-address=$(ADDRESS)" - - "--timeout={{ .Values.timeout }}" - - "--default-fstype=ext4" - {{- if $enableLeaderElection }} - - "--leader-election=true" - - "--leader-election-lease-duration=120s" - - "--leader-election-renew-deadline=60s" - - "--leader-election-retry-period=30s" - {{- end}} - - "--kube-api-qps=100" - - "--kube-api-burst=100" - {{- if .Values.controller.attacher.extraArgs }} - {{- with .Values.controller.attacher.extraArgs }} - {{- tpl . $ | trim | nindent 12 }} + {{- if not (regexMatch "(-timeout)" (join " " .Values.sidecars.attacher.extraArgs)) }} + - --timeout=60s + {{- end }} + - --csi-address=$(ADDRESS) + - --v={{ .Values.sidecars.attacher.logLevel }} + - --leader-election={{ .Values.sidecars.attacher.leaderElection.enabled | required "leader election state for csi-attacher is required, must be set to true || false." }} + {{- if .Values.sidecars.attacher.leaderElection.enabled }} + {{- if .Values.sidecars.attacher.leaderElection.leaseDuration }} + - --leader-election-lease-duration={{ .Values.sidecars.attacher.leaderElection.leaseDuration }} {{- end }} + {{- if .Values.sidecars.attacher.leaderElection.renewDeadline}} + - --leader-election-renew-deadline={{ .Values.sidecars.attacher.leaderElection.renewDeadline }} + {{- end }} + {{- if .Values.sidecars.attacher.leaderElection.retryPeriod }} + - --leader-election-retry-period={{ .Values.sidecars.attacher.leaderElection.retryPeriod }} + {{- end }} + {{- end }} + {{- if not (regexMatch "(-kube-api-qps)|(-kube-api-burst)|(-worker-threads)" (join " " .Values.sidecars.attacher.extraArgs)) }} + - --kube-api-qps=20 + - --kube-api-burst=100 + - --worker-threads=100 + {{- end }} + {{- if not (regexMatch "(-retry-interval-max)" (join " " .Values.sidecars.attacher.extraArgs)) }} + - --retry-interval-max=5m + {{- end }} + {{- range .Values.sidecars.attacher.extraArgs }} + - {{ . }} {{- end }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock + {{- with .Values.sidecars.attacher.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: {{ toYaml .Values.controller.attacher.resources | nindent 12 }} - - name: external-resizer + {{- with default .Values.controller.resources .Values.sidecars.attacher.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.attacher.securityContext }} securityContext: - {{- toYaml .Values.controller.securityContext | nindent 12 }} - image: "{{ .Values.controller.resizer.image.repository }}:{{ .Values.controller.resizer.image.tag }}" - imagePullPolicy: {{ .Values.controller.resizer.image.pullPolicy }} + {{- toYaml . | nindent 12 }} + {{- end }} + - name: external-resizer + image: "{{ .Values.sidecars.resizer.image.repository }}:{{ .Values.sidecars.resizer.image.tag }}" + imagePullPolicy: {{ .Values.sidecars.resizer.image.pullPolicy }} args: - - "--v={{ .Values.logVerbosityLevel }}" - - "--csi-address=$(ADDRESS)" - - "--timeout={{ .Values.timeout }}" - {{- if $enableLeaderElection }} - - "--leader-election=true" - - "--leader-election-lease-duration=120s" - - "--leader-election-renew-deadline=60s" - - "--leader-election-retry-period=30s" - {{- end}} - - "--kube-api-qps=100" - - "--kube-api-burst=100" - {{- if .Values.controller.resizer.extraArgs }} - {{- with .Values.controller.resizer.extraArgs }} - {{- tpl . $ | trim | nindent 12 }} + {{- if not (regexMatch "(-timeout)" (join " " .Values.sidecars.resizer.extraArgs)) }} + - --timeout=60s + {{- end }} + - --csi-address=$(ADDRESS) + - --v={{ .Values.sidecars.resizer.logLevel }} + - --handle-volume-inuse-error=false + {{- with .Values.sidecars.resizer.leaderElection }} + - --leader-election={{ .enabled | default true }} + {{- if .leaseDuration }} + - --leader-election-lease-duration={{ .leaseDuration }} + {{- end }} + {{- if .renewDeadline }} + - --leader-election-renew-deadline={{ .renewDeadline }} {{- end }} + {{- if .retryPeriod }} + - --leader-election-retry-period={{ .retryPeriod }} + {{- end }} + {{- end }} + {{- if not (regexMatch "(-kube-api-qps)|(-kube-api-burst)|(-workers)" (join " " .Values.sidecars.resizer.extraArgs)) }} + - --kube-api-qps=20 + - --kube-api-burst=100 + - --workers=100 + {{- end }} + {{- if not (regexMatch "(-retry-interval-max)" (join " " .Values.sidecars.resizer.extraArgs)) }} + - --retry-interval-max=30m + {{- end }} + {{- range .Values.sidecars.resizer.extraArgs }} + - {{ . }} {{- end }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock + {{- with .Values.sidecars.resizer.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ - resources: {{ toYaml .Values.controller.resizer.resources | nindent 12 }} - {{ if .Values.livenessProbe.enable }} - - name: liveness-probe + {{- with default .Values.controller.resources .Values.sidecars.resizer.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.resizer.securityContext }} securityContext: - {{- toYaml .Values.node.securityContext | nindent 12 }} - image: "{{ .Values.livenessProbe.image.repository }}:{{ .Values.livenessProbe.image.tag }}" - imagePullPolicy: {{ .Values.livenessProbe.image.pullPolicy }} + {{- toYaml . | nindent 12 }} + {{- end }} + - name: liveness-probe + image: "{{ .Values.sidecars.livenessProbe.image.repository }}:{{ .Values.sidecars.livenessProbe.image.tag }}" + imagePullPolicy: {{ .Values.sidecars.livenessProbe.image.pullPolicy }} args: - - "--v={{ .Values.logVerbosityLevel }}" - - "--csi-address=$(ADDRESS)" - {{- if .Values.livenessProbe.extraArgs }} - {{- with .Values.livenessProbe.extraArgs }} - {{- tpl . $ | trim | nindent 12 }} - {{- end }} + - --csi-address=/csi/csi.sock + {{- range .Values.sidecars.livenessProbe.extraArgs }} + - {{ . }} {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} env: - name: ADDRESS value: /var/lib/csi/sockets/pluginproxy/csi.sock volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ - name: socket-dir - resources: {{ toYaml .Values.livenessProbe.resources | nindent 12 }} - {{- end }} + - name: socket-dir + mountPath: /csi + {{- with default .Values.controller.resources .Values.sidecars.livenessProbe.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.livenessProbe.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} volumes: - name: socket-dir emptyDir: {} @@ -202,18 +317,14 @@ spec: {{- with .Values.controller.volumes }} {{- toYaml . | nindent 8 }} {{- end }} - affinity: {{ toYaml .Values.controller.affinity | nindent 8 }} - nodeSelector: {{ toYaml .Values.controller.nodeSelector | nindent 8 }} - tolerations: {{ toYaml .Values.controller.tolerations | nindent 8 }} {{- with .Values.controller.hostAliases }} hostAliases: {{- toYaml . | nindent 8 }} {{- end }} - {{- if .Values.controller.priorityClassName }} - priorityClassName: {{ .Values.controller.priorityClassName }} - {{- end }} - {{- with .Values.imagePullSecrets }} + {{- if .Values.imagePullSecrets }} imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} {{- end }} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/csi-controller-role-leases.yaml b/charts/cloudstack-csi/templates/csi-controller-role-leases.yaml new file mode 100644 index 0000000..2684d4f --- /dev/null +++ b/charts/cloudstack-csi/templates/csi-controller-role-leases.yaml @@ -0,0 +1,20 @@ +{{ if .Values.controller.enabled }} +{{ if .Values.rbac.create }} +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + namespace: {{ .Release.Namespace }} + name: csi-leases-role + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +{{- end}} +{{- end}} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/csi-controller-rolebinding-leases.yaml b/charts/cloudstack-csi/templates/csi-controller-rolebinding-leases.yaml new file mode 100644 index 0000000..5fb443c --- /dev/null +++ b/charts/cloudstack-csi/templates/csi-controller-rolebinding-leases.yaml @@ -0,0 +1,24 @@ +{{ if .Values.controller.enabled }} +{{ if .Values.rbac.create }} +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-leases-rolebinding + namespace: {{ .Release.Namespace }} + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + name: {{ .Values.controller.serviceAccount.name }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: csi-leases-role + apiGroup: rbac.authorization.k8s.io +{{- end}} +{{- end}} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/csi-controller-sa.yaml b/charts/cloudstack-csi/templates/csi-controller-sa.yaml new file mode 100644 index 0000000..c819c3d --- /dev/null +++ b/charts/cloudstack-csi/templates/csi-controller-sa.yaml @@ -0,0 +1,16 @@ +{{- if .Values.controller.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.controller.serviceAccount.name }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.controller.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if .Values.controller.serviceAccount.automountServiceAccountToken }} +automountServiceAccountToken: {{ .Values.controller.serviceAccount.automountServiceAccountToken }} +{{- end }} +{{- end -}} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/csi-driver.yaml b/charts/cloudstack-csi/templates/csi-driver.yaml index 323eaa5..0e35864 100644 --- a/charts/cloudstack-csi/templates/csi-driver.yaml +++ b/charts/cloudstack-csi/templates/csi-driver.yaml @@ -1,16 +1,16 @@ -{{- if or (.Values.controller.enabled) (.Values.node.enabled) }} apiVersion: storage.k8s.io/v1 kind: CSIDriver metadata: name: csi.cloudstack.apache.org + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} spec: attachRequired: true - podInfoOnMount: true + podInfoOnMount: false # Supports only persistent volumes. volumeLifecycleModes: - - Persistent -{{- end }} + - Persistent diff --git a/charts/cloudstack-csi/templates/csi-node-clusterrole.yaml b/charts/cloudstack-csi/templates/csi-node-clusterrole.yaml new file mode 100644 index 0000000..82c46b1 --- /dev/null +++ b/charts/cloudstack-csi/templates/csi-node-clusterrole.yaml @@ -0,0 +1,25 @@ +{{ if .Values.node.enabled }} +{{ if .Values.rbac.create }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cloudstack-csi-node-role + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get"] +{{- end}} +{{- end}} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/csi-node-clusterrolebinding.yaml b/charts/cloudstack-csi/templates/csi-node-clusterrolebinding.yaml new file mode 100644 index 0000000..0debd55 --- /dev/null +++ b/charts/cloudstack-csi/templates/csi-node-clusterrolebinding.yaml @@ -0,0 +1,23 @@ +{{ if .Values.node.enabled }} +{{ if .Values.rbac.create }} +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cloudstack-csi-node-binding + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +subjects: + - kind: ServiceAccount + name: {{ .Values.node.serviceAccount.name }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: cloudstack-csi-node-role + apiGroup: rbac.authorization.k8s.io +{{- end}} +{{- end}} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/csi-node-ds.yaml b/charts/cloudstack-csi/templates/csi-node-ds.yaml index 125e3ea..fda2410 100644 --- a/charts/cloudstack-csi/templates/csi-node-ds.yaml +++ b/charts/cloudstack-csi/templates/csi-node-ds.yaml @@ -3,158 +3,241 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: {{ include "csi.name" . }}-node + name: {{ include "cloudstack-csi-driver.name" . }}-node namespace: {{ .Release.Namespace }} labels: - {{- include "csi.node.labels" . | nindent 4 }} + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.node.daemonSetAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} spec: + {{- if or (kindIs "float64" .Values.node.revisionHistoryLimit) (kindIs "int64" .Values.node.revisionHistoryLimit) }} + revisionHistoryLimit: {{ .Values.node.revisionHistoryLimit }} + {{- end }} selector: matchLabels: - {{- include "csi.node.matchLabels" . | nindent 6 }} + app: {{ include "cloudstack-csi-driver.name" . }}-node + {{- include "cloudstack-csi-driver.selectorLabels" . | nindent 6 }} updateStrategy: - type: RollingUpdate + {{- toYaml .Values.node.updateStrategy | nindent 4 }} template: metadata: labels: - {{- include "csi.node.labels" . | nindent 8 }} + app: {{ include "cloudstack-csi-driver.name" . }}-node + {{- include "cloudstack-csi-driver.selectorLabels" . | nindent 8 }} + {{- if .Values.node.podLabels }} + {{- toYaml .Values.node.podLabels | nindent 8 }} + {{- end }} + {{- with .Values.node.podAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 8 }} - {{- end }} + {{- end }} spec: + {{- with .Values.node.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.node.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ .Values.node.serviceAccount.name }} + terminationGracePeriodSeconds: {{ .Values.node.terminationGracePeriodSeconds }} + priorityClassName: {{ .Values.node.priorityClassName | default "system-node-critical" }} + tolerations: + {{- with .Values.node.tolerations }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.node.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: cloudstack-csi-node - image: "{{ .Values.node.csiDriver.image.repository }}:{{ .Values.node.csiDriver.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.node.csiDriver.image.pullPolicy }} + image: "{{ .Values.node.image.repository }}:{{ .Values.node.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.node.image.pullPolicy }} args: - - "node" - - "--endpoint=$(CSI_ENDPOINT)" - - "--cloudstack-config=$(CLOUD_CONFIG)" - - "--logging-format={{ .Values.logFormat }}" - - "--node-name=$(NODE_NAME)" - - "--v={{ .Values.logVerbosityLevel }}" - {{- if .Values.node.csiDriver.extraArgs }} - {{- with .Values.node.csiDriver.extraArgs }} - {{- tpl . $ | trim | nindent 12 }} + - node + - --endpoint=$(CSI_ENDPOINT) + - --cloudstack-config=$(CLOUD_CONFIG) + {{- with .Values.node.volumeAttachLimit }} + - --volume-attach-limit={{ . }} {{- end }} + - --node-name=$(NODE_NAME) + {{- with .Values.node.loggingFormat }} + - --logging-format={{ . }} + {{- end }} + - --v={{ .Values.node.logLevel }} + {{- range .Values.node.extraArgs }} + - {{ . }} {{- end }} env: - name: CSI_ENDPOINT value: unix:///csi/csi.sock - name: CLOUD_CONFIG - value: /etc/config/cloud.conf + value: /etc/config/cloud-config - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName - {{ if .Values.livenessProbe.enable }} - livenessProbe: {{- toYaml .Values.livenessProbe.livenessProbeSpec | nindent 12 }} - {{ end }} - ports: - - containerPort: 9808 - name: healthz - protocol: TCP - securityContext: - privileged: true - capabilities: - add: ["SYS_ADMIN"] - allowPrivilegeEscalation: true + {{- with .Values.node.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - - name: socket-dir - mountPath: /csi - name: kubelet-dir - mountPath: {{ .Values.node.kubeletDir }} - # needed so that any mounts setup inside this container are - # propagated back to the host machine. + mountPath: {{ .Values.node.kubeletPath }} mountPropagation: "Bidirectional" - - name: pods-probe-dir + - name: plugin-dir + mountPath: /csi + - name: device-dir mountPath: /dev - mountPropagation: "HostToContainer" - name: cloud-config mountPath: /etc/config - {{- with .Values.node.csiDriver.volumeMounts }} + {{- if eq .Values.node.metadataSource "cloud-init" }} + - name: cloud-init-dir + mountPath: /run/cloud-init + {{- end }} + {{- if eq .Values.node.metadataSource "ignition" }} + - name: ignition-dir + mountPath: /run/metadata + {{- end }} + {{- with .Values.node.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: healthz + containerPort: 9808 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + timeoutSeconds: 3 + periodSeconds: 10 + failureThreshold: 5 + {{- with .Values.node.resources }} + resources: {{- toYaml . | nindent 12 }} {{- end }} - resources: {{ toYaml .Values.node.csiDriver.resources | nindent 12 }} - - name: node-driver-registrar + {{- with .Values.node.securityContext }} securityContext: - {{- toYaml .Values.node.securityContext | nindent 12 }} - image: "{{ .Values.node.nodeDriverRegistrar.image.repository }}:{{ .Values.node.nodeDriverRegistrar.image.tag }}" - imagePullPolicy: {{ .Values.node.nodeDriverRegistrar.image.pullPolicy }} + {{- toYaml . | nindent 12 }} + {{- end }} + - name: node-driver-registrar + image: "{{ .Values.sidecars.nodeDriverRegistrar.image.repository }}:{{ .Values.sidecars.nodeDriverRegistrar.image.tag }}" + imagePullPolicy: {{ .Values.sidecars.nodeDriverRegistrar.image.pullPolicy }} args: - - "-v={{ .Values.logVerbosityLevel }}" - - "--csi-address=$(ADDRESS)" - - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" - - "--health-port=9809" - {{- if .Values.node.nodeDriverRegistrar.extraArgs }} - {{- with .Values.node.nodeDriverRegistrar.extraArgs }} - {{- tpl . $ | trim | nindent 12 }} - {{- end }} + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --health-port=9809 + - --v={{ .Values.sidecars.nodeDriverRegistrar.logLevel }} + {{- range .Values.sidecars.nodeDriverRegistrar.extraArgs }} + - {{ . }} {{- end }} - ports: - - containerPort: 9809 - name: healthz - {{ if .Values.livenessProbe.enable }} - livenessProbe: {{- toYaml .Values.livenessProbe.livenessProbeSpec | nindent 12 }} - {{- end }} env: - name: ADDRESS value: /csi/csi.sock - name: DRIVER_REG_SOCK_PATH - value: {{ .Values.node.kubeletDir }}/plugins/csi.cloudstack.apache.org/csi.sock - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName + value: {{ printf "%s/plugins/csi.cloudstack.apache.org/csi.sock" (trimSuffix "/" .Values.node.kubeletPath) }} + {{- with .Values.sidecars.nodeDriverRegistrar.env }} + {{- . | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} + ports: + - containerPort: 9809 + name: healthz + {{- with .Values.sidecars.nodeDriverRegistrar.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} volumeMounts: - - mountPath: /csi - name: socket-dir + - name: plugin-dir + mountPath: /csi - name: registration-dir mountPath: /registration - resources: {{ toYaml .Values.node.nodeDriverRegistrar.resources | nindent 12 }} - {{ if .Values.livenessProbe.enable }} - - name: liveness-probe + - name: probe-dir + mountPath: {{ printf "%s/plugins/csi.cloudstack.apache.org/" (trimSuffix "/" .Values.node.kubeletPath) }} + {{- with default .Values.node.resources .Values.sidecars.nodeDriverRegistrar.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.nodeDriverRegistrar.securityContext }} securityContext: - {{- toYaml .Values.node.securityContext | nindent 12 }} - image: "{{ .Values.livenessProbe.image.repository }}:{{ .Values.livenessProbe.image.tag }}" - imagePullPolicy: {{ .Values.livenessProbe.image.pullPolicy }} + {{- toYaml . | nindent 12 }} + {{- end }} + - name: liveness-probe + image: "{{ .Values.sidecars.livenessProbe.image.repository }}:{{ .Values.sidecars.livenessProbe.image.tag }}" + imagePullPolicy: {{ .Values.sidecars.livenessProbe.image.pullPolicy }} args: - - "-v={{ .Values.logVerbosityLevel }}" - - "--csi-address=$(ADDRESS)" - {{- if .Values.livenessProbe.extraArgs }} - {{- with .Values.livenessProbe.extraArgs }} - {{- tpl . $ | trim | nindent 12 }} + - --csi-address=/csi/csi.sock + {{- range .Values.sidecars.livenessProbe.extraArgs }} + - {{ . }} {{- end }} - {{- end }} - env: - - name: ADDRESS - value: /csi/csi.sock + {{- with .Values.controller.envFrom }} + envFrom: + {{- . | toYaml | nindent 12 }} + {{- end }} volumeMounts: - - mountPath: /csi/ - name: socket-dir - resources: {{ toYaml .Values.livenessProbe.resources | nindent 12 }} - {{ end }} + - name: plugin-dir + mountPath: /csi + {{- with default .Values.node.resources .Values.sidecars.livenessProbe.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.sidecars.livenessProbe.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} volumes: - - name: socket-dir + - name: kubelet-dir + hostPath: + path: {{ .Values.node.kubeletPath }} + type: Directory + - name: plugin-dir hostPath: - path: {{ .Values.node.kubeletDir }}/plugins/csi.cloudstack.apache.org/ + path: {{ printf "%s/plugins/csi.cloudstack.apache.org/" (trimSuffix "/" .Values.node.kubeletPath) }} type: DirectoryOrCreate - name: registration-dir hostPath: - path: {{ .Values.node.kubeletDir }}/plugins_registry/ + path: {{ printf "%s/plugins_registry/" (trimSuffix "/" .Values.node.kubeletPath) }} type: Directory - - name: kubelet-dir + - name: device-dir hostPath: - path: {{ .Values.node.kubeletDir }} + path: /dev type: Directory - - name: pods-probe-dir + - name: probe-dir + {{- if .Values.node.probeDirVolume }} + {{- toYaml .Values.node.probeDirVolume | nindent 10 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- if eq .Values.node.metadataSource "cloud-init" }} + - name: cloud-init-dir hostPath: - path: /dev + path: /run/cloud-init + type: Directory + {{- end }} + {{- if eq .Values.node.metadataSource "ignition" }} + - name: ignition-dir + hostPath: + path: /run/metadata type: Directory + {{- end }} {{- if .Values.secret.enabled }} - name: cloud-config secret: @@ -165,20 +248,10 @@ spec: path: /etc/config {{- end }} {{- with .Values.node.volumes }} - {{- toYaml . | nindent 8 }} + {{- toYaml . | nindent 8 }} {{- end }} - affinity: {{ toYaml .Values.node.affinity | nindent 8 }} - nodeSelector: {{ toYaml .Values.node.nodeSelector | nindent 8 }} - tolerations: {{ toYaml .Values.node.tolerations | nindent 8 }} {{- with .Values.node.hostAliases }} hostAliases: {{- toYaml . | nindent 8 }} {{- end }} - {{- if .Values.node.priorityClassName }} - priorityClassName: {{ .Values.node.priorityClassName }} - {{- end }} - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} {{- end }} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/csi-node-sa.yaml b/charts/cloudstack-csi/templates/csi-node-sa.yaml new file mode 100644 index 0000000..efe14fb --- /dev/null +++ b/charts/cloudstack-csi/templates/csi-node-sa.yaml @@ -0,0 +1,16 @@ +{{- if .Values.node.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.node.serviceAccount.name }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.node.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- if .Values.node.serviceAccount.automountServiceAccountToken }} +automountServiceAccountToken: {{ .Values.node.serviceAccount.automountServiceAccountToken }} +{{- end }} +{{- end -}} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/csi-sa.yaml b/charts/cloudstack-csi/templates/csi-sa.yaml deleted file mode 100644 index 61ffc42..0000000 --- a/charts/cloudstack-csi/templates/csi-sa.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{ if .Values.serviceAccount.create }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ .Values.serviceAccount.name }} - namespace: {{ .Release.Namespace }} - annotations: - {{- with .Values.commonAnnotations }} - {{- toYaml . | nindent 4 }} - {{- end }} -{{ end }} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/podmonitor.yaml b/charts/cloudstack-csi/templates/podmonitor.yaml index 736dca2..34675b3 100644 --- a/charts/cloudstack-csi/templates/podmonitor.yaml +++ b/charts/cloudstack-csi/templates/podmonitor.yaml @@ -2,14 +2,14 @@ apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: - labels: - {{- include "csi.controller.labels" . | nindent 4 }} - name: {{ include "csi.name" . }}-controller + name: {{ include "cloudstack-csi-driver.name" . }}-controller namespace: {{ .Release.Namespace }} + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} spec: podMetricsEndpoints: - interval: 30s @@ -18,5 +18,5 @@ spec: jobLabel: component selector: matchLabels: - {{- include "csi.controller.matchLabels" . | nindent 6 }} + {{- include "cloudstack-csi-driver.selectorLabels" . | nindent 6 }} {{- end }} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/secret.yaml b/charts/cloudstack-csi/templates/secret.yaml index fc0f565..c9e91a6 100644 --- a/charts/cloudstack-csi/templates/secret.yaml +++ b/charts/cloudstack-csi/templates/secret.yaml @@ -4,11 +4,13 @@ kind: Secret metadata: name: {{ .Values.secret.name | default "cloud.conf" }} namespace: {{ .Release.Namespace }} + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} type: Opaque data: - cloud.conf: {{ include "cloudConfig" . | b64enc }} + cloud-config: {{ include "cloudConfig" . | b64enc }} {{- end }} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/syncer-job.yaml b/charts/cloudstack-csi/templates/syncer-job.yaml index 0009b95..be155d4 100644 --- a/charts/cloudstack-csi/templates/syncer-job.yaml +++ b/charts/cloudstack-csi/templates/syncer-job.yaml @@ -4,38 +4,47 @@ apiVersion: batch/v1 kind: Job metadata: - name: {{ include "csi.name" . }}-sc-syncer + name: {{ include "cloudstack-csi-driver.name" . }}-sc-syncer namespace: {{ .Release.Namespace }} + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} spec: backoffLimit: {{ .Values.syncer.backoffLimit }} template: spec: - securityContext: {{- toYaml .Values.syncer.securityContext | nindent 12 }} - serviceAccountName: {{ .Values.syncer.rbac.serviceAccount.name }} + securityContext: {{- toYaml .Values.syncer.podSecurityContext | nindent 8 }} + serviceAccountName: {{ .Values.syncer.serviceAccount.name }} containers: - - image: "{{ .Values.syncer.image.repository }}:{{ .Values.syncer.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.syncer.image.imagePullPolicy }} - name: {{ include "csi.name" . }}-sc-syncer - args: - - "--cloudstackconfig=$(CLOUD_CONFIG)" - - "--kubeconfig=-" - - "--volumeExpansion=true" - {{- if .Values.syncer.extraArgs }} - {{- with .Values.syncer.extraArgs }} - {{- tpl . $ | trim | nindent 12 }} + - image: "{{ .Values.syncer.image.repository }}:{{ .Values.syncer.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.syncer.image.imagePullPolicy }} + name: {{ include "cloudstack-csi-driver.name" . }}-sc-syncer + args: + - "--cloudstackconfig=$(CLOUD_CONFIG)" + - "--kubeconfig=-" + - "--volumeExpansion=true" + {{- if .Values.syncer.extraArgs }} + {{- with .Values.syncer.extraArgs }} + {{- tpl . $ | trim | nindent 12 }} + {{- end }} + {{- end }} + env: + - name: CLOUD_CONFIG + value: /etc/config/cloud-config + volumeMounts: + - name: cloud-config + mountPath: /etc/config/ + {{- with .Values.syncer.resources }} + resources: + {{- toYaml . | nindent 12 }} {{- end }} + {{- with .Values.syncer.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} {{- end }} - env: - - name: CLOUD_CONFIG - value: /etc/config/cloud.conf - volumeMounts: - - name: cloud-config - mountPath: /etc/config/ - resources: {{ toYaml .Values.controller.resizer.resources | nindent 12 }} volumes: {{- if .Values.secret.enabled }} - name: cloud-config @@ -46,7 +55,13 @@ spec: hostPath: path: /etc/cloudstack-csi-driver {{- end }} - affinity: {{ toYaml .Values.syncer.affinity | nindent 8 }} - nodeSelector: {{ toYaml .Values.syncer.nodeSelector | nindent 8 }} + {{- with .Values.syncer.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.syncer.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} restartPolicy: {{ .Values.syncer.restartPolicy }} {{- end }} \ No newline at end of file diff --git a/charts/cloudstack-csi/templates/syncer-rbac.yaml b/charts/cloudstack-csi/templates/syncer-rbac.yaml index a9b534d..87f22bd 100644 --- a/charts/cloudstack-csi/templates/syncer-rbac.yaml +++ b/charts/cloudstack-csi/templates/syncer-rbac.yaml @@ -1,24 +1,31 @@ {{ if .Values.syncer.enabled }} -{{ if .Values.syncer.rbac.create }} +{{ if .Values.syncer.serviceAccount.create }} --- apiVersion: v1 kind: ServiceAccount metadata: - name: {{ .Values.syncer.rbac.serviceAccount.name }} + name: {{ .Values.syncer.serviceAccount.name }} namespace: {{ .Release.Namespace }} + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.syncer.serviceAccount.annotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} +{{- if .Values.syncer.serviceAccount.automountServiceAccountToken }} +automountServiceAccountToken: {{ .Values.syncer.serviceAccount.automountServiceAccountToken }} +{{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: csi-sc-syncer-role + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} rules: - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] @@ -28,13 +35,15 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: csi-sc-syncer-binding + labels: + {{- include "cloudstack-csi-driver.labels" . | nindent 4 }} + {{- with .Values.commonAnnotations }} annotations: - {{- with .Values.commonAnnotations }} {{- toYaml . | nindent 4 }} - {{- end }} + {{- end }} subjects: - kind: ServiceAccount - name: {{ .Values.syncer.rbac.serviceAccount.name }} + name: {{ .Values.syncer.serviceAccount.name }} namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole diff --git a/charts/cloudstack-csi/values.yaml b/charts/cloudstack-csi/values.yaml index f4b25d8..9ca492c 100644 --- a/charts/cloudstack-csi/values.yaml +++ b/charts/cloudstack-csi/values.yaml @@ -1,8 +1,8 @@ -extraLabels: {} - nameOverride: "cloudstack-csi" fullnameOverride: "" -timeout: 300s + +# Additional labels to add to all resources +extraLabels: {} ## Annotations to apply to all resources commonAnnotations: {} @@ -11,76 +11,227 @@ commonAnnotations: {} # "helm.sh/hook-weight": "-100" # "helm.sh/hook-delete-policy": before-hook-creation -# Create Prometheus Operator PodMonitor. Requires http server above. +# Create Prometheus Operator PodMonitor. # See https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.PodMonitor podMonitor: enabled: false -## CloudStack CSI Controller Defaults -controller: - enabled: true +# -- Custom labels to add into metadata +customLabels: {} + +sidecars: + provisioner: + env: [] + image: + pullPolicy: IfNotPresent + repository: registry.k8s.io/sig-storage/csi-provisioner + tag: "v5.0.1" + logLevel: 2 + # Feature gates to enable + featureGates: "Topology=true" + # Extra arguments passed to external-provisioner. + extraArgs: [] + resources: {} + # Tune leader lease election for csi-provisioner. + # Leader election is on by default. + leaderElection: + enabled: true + # Optional values to tune lease behavior. + # The arguments provided must be in an acceptable time.ParseDuration format. + # Ref: https://pkg.go.dev/flag#Duration + # leaseDuration: "15s" + # renewDeadline: "10s" + # retryPeriod: "5s" + securityContext: + seccompProfile: + type: RuntimeDefault + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false attacher: + env: [] image: - repository: registry.k8s.io/sig-storage/csi-attacher - tag: v4.6.1 pullPolicy: IfNotPresent + repository: registry.k8s.io/sig-storage/csi-attacher + tag: "v4.6.1" + # Tune leader lease election for csi-attacher. + # Leader election is on by default. + leaderElection: + enabled: true + # Optional values to tune lease behavior. + # The arguments provided must be in an acceptable time.ParseDuration format. + # Ref: https://pkg.go.dev/flag#Duration + # leaseDuration: "15s" + # renewDeadline: "10s" + # retryPeriod: "5s" + logLevel: 2 + # Extra arguments passed to external-attacher. + extraArgs: [] resources: {} - extraArgs: {} - provisioner: - topology: "true" + securityContext: + seccompProfile: + type: RuntimeDefault + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + livenessProbe: image: - repository: registry.k8s.io/sig-storage/csi-provisioner - tag: v5.0.1 pullPolicy: IfNotPresent + repository: registry.k8s.io/sig-storage/livenessprobe + tag: "v2.12.0" + # Extra arguments passed to livenessprobe. + extraArgs: [] resources: {} - extraArgs: {} + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false resizer: + env: [] image: - repository: registry.k8s.io/sig-storage/csi-resizer - tag: v1.11.1 pullPolicy: IfNotPresent + repository: registry.k8s.io/sig-storage/csi-resizer + tag: "v1.11.1" + # Tune leader lease election for csi-resizer. + # Leader election is on by default. + leaderElection: + enabled: true + # Optional values to tune lease behavior. + # The arguments provided must be in an acceptable time.ParseDuration format. + # Ref: https://pkg.go.dev/flag#Duration + # leaseDuration: "15s" + # renewDeadline: "10s" + # retryPeriod: "5s" + logLevel: 2 + # Extra arguments passed to external-resizer. + extraArgs: [] resources: {} - extraArgs: {} - csiDriverController: + securityContext: + seccompProfile: + type: RuntimeDefault + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + nodeDriverRegistrar: + env: [] image: - repository: ghcr.io/leaseweb/cloudstack-csi-driver pullPolicy: IfNotPresent - # tag: 0.4.1 # defaults to .Chart.AppVersion - volumeMounts: {} + repository: registry.k8s.io/sig-storage/csi-node-driver-registrar + tag: "v2.10.1" + logLevel: 2 + # Extra arguments passed to node-driver-registrar. + extraArgs: [] resources: {} - extraArgs: {} - + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + periodSeconds: 90 + timeoutSeconds: 15 + +## CloudStack CSI Controller Defaults +controller: + enabled: true + image: + repository: ghcr.io/leaseweb/cloudstack-csi-driver + # Overrides the image tag whose default is v{{ .Chart.AppVersion }} + tag: "" + pullPolicy: IfNotPresent ## Set number of replicas (If replicaCount set to 1 by default leader-elect will be false) - replicaCount: 1 - strategy: + replicaCount: 2 + # Extra arguments passed to cloudstack-csi-driver controller. + extraArgs: [] + loggingFormat: text + logLevel: 2 + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: "app.kubernetes.io/name" + operator: In + values: + - cloudstack-csi-controller + topologyKey: "kubernetes.io/hostname" + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + # The default filesystem type of the volume to provision when fstype is unspecified in the StorageClass. + # If the default is not set and fstype is unset in the StorageClass, then no fstype will be set + defaultFsType: ext4 + env: [] + # Use envFrom to reference ConfigMaps and Secrets across all containers in the deployment + envFrom: [] + # If set, add pv/pvc metadata to plugin create requests as parameters. + extraCreateMetadata: true + nodeSelector: {} + deploymentAnnotations: {} + podAnnotations: {} + podLabels: {} + priorityClassName: system-cluster-critical + revisionHistoryLimit: 10 + updateStrategy: # RollingUpdate strategy replaces old pods with new ones gradually, # without incurring downtime. type: RollingUpdate rollingUpdate: # maxUnavailable is the maximum number of pods that can be # unavailable during the update process. - maxUnavailable: 0 + maxUnavailable: 1 # maxSurge is the maximum number of pods that can be # created over the desired number of pods. - maxSurge: 1 - priorityClassName: "" + maxSurge: 0 + resources: + requests: + cpu: 10m + memory: 40Mi + limits: + memory: 256Mi + serviceAccount: + # A service account will be created for you if set to true. Set to false if you want to use your own. + create: true + name: cloudstack-csi-controller + annotations: {} + automountServiceAccountToken: true + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + # securityContext on the controller pod podSecurityContext: runAsNonRoot: true runAsUser: 65532 runAsGroup: 65532 fsGroup: 65532 fsGroupChangePolicy: OnRootMismatch - securityContext: {} - # capabilities: - # drop: - # - ALL - # seccompProfile: - # type: RuntimeDefault - # readOnlyRootFilesystem: true - volumes: {} - affinity: {} - nodeSelector: {} - tolerations: [] + # securityContext on the controller container (see sidecars for securityContext on sidecar containers) + securityContext: + seccompProfile: + type: RuntimeDefault + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + # Add additional volume mounts on the controller with controller.volumes and controller.volumeMounts + volumes: [] + # Add additional volumes to be mounted onto the controller: + # - name: custom-dir + # hostPath: + # path: /path/to/dir + # type: Directory + volumeMounts: [] + # And add mount paths for those additional volumes: + # - name: custom-dir + # mountPath: /mount/path + # # Allow for specifying internal IP addresses for multiple hostnames hostAliases: {} # - ip: "10.0.0.1" @@ -90,97 +241,128 @@ controller: ## CloudStack CSI Node Defaults node: enabled: true - nodeDriverRegistrar: - image: - repository: registry.k8s.io/sig-storage/csi-node-driver-registrar - tag: v2.10.1 - pullPolicy: IfNotPresent - resources: {} - extraArgs: {} - csiDriver: - image: - repository: ghcr.io/leaseweb/cloudstack-csi-driver - pullPolicy: IfNotPresent - # tag: 0.4.1 # defaults to .Chart.AppVersion - resources: {} - extraArgs: {} - volumeMounts: {} - volumes: {} + image: + repository: ghcr.io/leaseweb/cloudstack-csi-driver + # Overrides the image tag whose default is v{{ .Chart.AppVersion }} + tag: "" + pullPolicy: IfNotPresent + env: [] + envFrom: [] + kubeletPath: /var/lib/kubelet + loggingFormat: text + logLevel: 2 priorityClassName: "" - podSecurityContext: {} - - securityContext: {} - # capabilities: - # drop: - # - ALL - # seccompProfile: - # type: RuntimeDefault - + # Extra arguments passed to cloudstack-csi-driver controller. + extraArgs: [] affinity: {} - nodeSelector: {} - + daemonSetAnnotations: {} + podAnnotations: {} + podLabels: {} + terminationGracePeriodSeconds: 30 tolerations: - - operator: Exists - - kubeletDir: /var/lib/kubelet - + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + resources: + requests: + cpu: 10m + memory: 40Mi + limits: + memory: 256Mi + revisionHistoryLimit: 10 + probeDirVolume: + emptyDir: {} + serviceAccount: + create: true + name: cloudstack-csi-node + annotations: {} + automountServiceAccountToken: true + # Metadata source to try to find instance ID. + # Possible values 'cloud-init' & 'ignition' + metadataSource: ignition + # The maximum number of volumes that can be attached to a node + volumeAttachLimit: + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: "10%" + # securityContext on the node pod + podSecurityContext: + # The node pod must be run as root to bind to the registration/driver sockets + runAsNonRoot: false + runAsUser: 0 + runAsGroup: 0 + fsGroup: 0 + # securityContext on the node container (see sidecars for securityContext on sidecar containers) + # Privileged containers always run as `Unconfined`, which means that they are not restricted by a seccomp profile. + securityContext: + # readOnlyRootFilesystem: true + privileged: true + # Add additional volume mounts on the node pods with node.volumes and node.volumeMounts + volumes: [] + # Add additional volumes to be mounted onto the node pods: + # - name: custom-dir + # hostPath: + # path: /path/to/dir + # type: Directory + volumeMounts: [] + # And add mount paths for those additional volumes: + # - name: custom-dir + # mountPath: /mount/path + # # Allow for specifying internal IP addresses for multiple hostnames hostAliases: # - ip: "10.0.0.1" # hostnames: # - "keystone.hostname.com" -## CloudStack Storage Class Synecr Job Defaults +## CloudStack Storage Class Syncer Job Defaults syncer: enabled: true # Job image image: repository: "ghcr.io/leaseweb/cloudstack-csi-sc-syncer" - # tag: "0.4.1" # defaults to .Chart.AppVersion + # Overrides the image tag. Default is {{ .Chart.AppVersion }} + tag: "" imagePullPolicy: IfNotPresent + # Extra arguments passed to cloudstack-csi-sc-syncer. extraArgs: {} + affinity: {} + nodeSelector: {} + podAnnotations: {} + podLabels: {} + resources: {} + serviceAccount: + create: true + name: cloudstack-csi-sc-syncer + annotations: {} + automountServiceAccountToken: true # Job configurations backoffLimit: 4 restartPolicy: Never - rbac: - create: true - serviceAccount: - name: cloudstack-csi-sc-syncer - - # Define resources - # resources: - # limits: - # cpu: 2 - # memory: 2000Mi - # requests: - # cpu: 500m - # memory: 500Mi - - securityContext: {} - # capabilities: - # drop: - # - ALL - # seccompProfile: - # type: RuntimeDefault - - affinity: {} - - nodeSelector: {} - -# Log verbosity level. -# See https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md -# for description of individual verbosity levels. -logVerbosityLevel: 2 - -# Log format. Available options are "text" and "json" -logFormat: text + # securityContext on the syncer job + # securityContext on the controller pod + podSecurityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + # securityContext on the controller container (see sidecars for securityContext on sidecar containers) + securityContext: + seccompProfile: + type: RuntimeDefault + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false -# the secret should contain the cloudstack credentials -# there are several options to inject the credentials: +# The secret should contain the CloudStack credentials +# There are several options to inject the credentials: # 1) from kubernetes secret that doesn't exist: set "enabled" and "create" to true, this will create a secret from the values written to "data" down below # 2) from kubernetes secret that already exists: set "enabled" to true and "create" to false # 3) from host system path /etc/cloud/cloud.conf: set "enabled" to false and "hostMount" to true @@ -188,7 +370,7 @@ logFormat: text secret: enabled: true create: false - name: cloud-config + name: cloudstack-secret cloudConfigData: global: api-url: "" @@ -199,28 +381,5 @@ secret: imagePullSecrets: [] # - name: my-imagepull-secret -serviceAccount: - create: true - name: "cloudstack-csi-controller" - -rbac: +rbac: create: true - -livenessProbe: - enable: true - image: - repository: registry.k8s.io/sig-storage/livenessprobe - tag: v2.12.0 - pullPolicy: IfNotPresent - resources: {} - extraArgs: {} - livenessProbeSpec: - httpGet: - path: /healthz - port: healthz - failureThreshold: 5 - initialDelaySeconds: 60 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 15 - diff --git a/deploy/k8s/csidriver.yaml b/deploy/k8s/csidriver.yaml index dbdeb46..f5443e7 100644 --- a/deploy/k8s/csidriver.yaml +++ b/deploy/k8s/csidriver.yaml @@ -3,6 +3,8 @@ kind: CSIDriver metadata: name: csi.cloudstack.apache.org spec: + attachRequired: true + podInfoOnMount: false # Supports only persistent volumes. volumeLifecycleModes: - Persistent From 4ea00b29e4c4c9dea89ab19351d938260596f6c4 Mon Sep 17 00:00:00 2001 From: Hans Rakers Date: Thu, 4 Jul 2024 10:55:59 +0200 Subject: [PATCH 2/5] chore: fix typo --- pkg/syncer/run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/syncer/run.go b/pkg/syncer/run.go index 99d5497..b81affb 100644 --- a/pkg/syncer/run.go +++ b/pkg/syncer/run.go @@ -86,7 +86,7 @@ func (s syncer) Run(ctx context.Context) error { if s.volumeExpansion { allowVolumeExpansion := s.volumeExpansion - log.Printf("Setting allowVolumeExpasion to %t...", allowVolumeExpansion) + log.Printf("Setting allowVolumeExpansion to %t", allowVolumeExpansion) } if len(errs) == 0 { From 459d49a1a86e0210899d479573314681ce9df366 Mon Sep 17 00:00:00 2001 From: Hans Rakers Date: Thu, 4 Jul 2024 10:56:58 +0200 Subject: [PATCH 3/5] chore: Add/adjust some logging --- pkg/cloud/metadata.go | 8 ++++---- pkg/cloud/node.go | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/pkg/cloud/metadata.go b/pkg/cloud/metadata.go index 8848527..17a0c80 100644 --- a/pkg/cloud/metadata.go +++ b/pkg/cloud/metadata.go @@ -25,7 +25,7 @@ func (c *client) metadataInstanceID(ctx context.Context) string { // Try a NODE_ID environment variable if envNodeID := os.Getenv("NODE_ID"); envNodeID != "" { - logger.V(4).Info("Found CloudStack VM ID from envvar NODE_ID", "nodeID", envNodeID) + logger.Info("Found CloudStack VM ID from envvar NODE_ID", "nodeID", envNodeID) return envNodeID } @@ -38,7 +38,7 @@ func (c *client) metadataInstanceID(ctx context.Context) string { if err != nil { logger.Error(err, "Cannot read cloud-init instance data") } else if ciData.V1.InstanceID != "" { - logger.V(4).Info("Found CloudStack VM ID from cloud-init", "nodeID", ciData.V1.InstanceID) + logger.Info("Found CloudStack VM ID from cloud-init", "nodeID", ciData.V1.InstanceID) return ciData.V1.InstanceID } @@ -57,7 +57,7 @@ func (c *client) metadataInstanceID(ctx context.Context) string { if err != nil { logger.Error(err, "Cannot read ignition metadata") } else if instanceID != "" { - logger.V(4).Info("Found CloudStack VM ID from ignition", "nodeID", instanceID) + logger.Info("Found CloudStack VM ID from ignition", "nodeID", instanceID) return instanceID } @@ -68,7 +68,7 @@ func (c *client) metadataInstanceID(ctx context.Context) string { logger.Error(err, "Cannot read file "+ignitionMetadataFilePath) } - logger.V(4).Info("CloudStack VM ID not found in meta-data") + logger.Info("CloudStack VM ID not found in meta-data") return "" } diff --git a/pkg/cloud/node.go b/pkg/cloud/node.go index 29c11bf..c3eab50 100644 --- a/pkg/cloud/node.go +++ b/pkg/cloud/node.go @@ -1,17 +1,25 @@ package cloud -import "context" +import ( + "context" + + "k8s.io/klog/v2" +) func (c *client) GetNodeInfo(ctx context.Context, vmName string) (*VM, error) { - // First, try to read the instance ID from meta-data (cloud-init) + logger := klog.FromContext(ctx) + + // First, try to read the instance ID from meta-data. if id := c.metadataInstanceID(ctx); id != "" { // Instance ID found using metadata + logger.V(4).Info("Looking up node info using VM ID found in metadata", "nodeID", id) // Use CloudStack API to get VM info return c.GetVMByID(ctx, id) } - // VM ID was not found using metadata. - // Use VM name instead + // VM ID was not found using metadata, fall back to using VM name instead. + logger.V(4).Info("Looking up node info using VM name", "nodeName", vmName) + return c.getVMByName(ctx, vmName) } From 1f99da8a716b5528dcb4c7fb6e256fa74c16247e Mon Sep 17 00:00:00 2001 From: Hans Rakers Date: Thu, 4 Jul 2024 11:05:28 +0200 Subject: [PATCH 4/5] chore: Major bump chart version 2.0.0 --- charts/cloudstack-csi/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/cloudstack-csi/Chart.yaml b/charts/cloudstack-csi/Chart.yaml index f72787c..a0bbb22 100644 --- a/charts/cloudstack-csi/Chart.yaml +++ b/charts/cloudstack-csi/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v2 name: cloudstack-csi -description: A Helm chart for Kubernetes +description: A Helm chart for CloudStack CSI driver type: application -version: 1.2.0 +version: 2.0.0 appVersion: 0.6.0 sources: - https://github.com/Leaseweb/cloudstack-csi-driver From 6e7eda85b80d870fe0128f2939af40a74338f387 Mon Sep 17 00:00:00 2001 From: Hans Rakers Date: Fri, 5 Jul 2024 11:17:48 +0200 Subject: [PATCH 5/5] fix: Add tolerations to syncer job --- .../cloudstack-csi/templates/syncer-job.yaml | 20 +++++++++++-------- charts/cloudstack-csi/values.yaml | 8 +++++++- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/charts/cloudstack-csi/templates/syncer-job.yaml b/charts/cloudstack-csi/templates/syncer-job.yaml index be155d4..19cfa92 100644 --- a/charts/cloudstack-csi/templates/syncer-job.yaml +++ b/charts/cloudstack-csi/templates/syncer-job.yaml @@ -18,6 +18,18 @@ spec: spec: securityContext: {{- toYaml .Values.syncer.podSecurityContext | nindent 8 }} serviceAccountName: {{ .Values.syncer.serviceAccount.name }} + tolerations: + { { - with .Values.node.tolerations } } + { { - toYaml . | nindent 8 } } + { { - end } } + {{- with .Values.syncer.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + nodeSelector: + kubernetes.io/os: linux + {{- with .Values.syncer.nodeSelector }} + {{- toYaml . | nindent 8 }} + {{- end }} containers: - image: "{{ .Values.syncer.image.repository }}:{{ .Values.syncer.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.syncer.image.imagePullPolicy }} @@ -55,13 +67,5 @@ spec: hostPath: path: /etc/cloudstack-csi-driver {{- end }} - {{- with .Values.syncer.affinity }} - affinity: {{- toYaml . | nindent 8 }} - {{- end }} - nodeSelector: - kubernetes.io/os: linux - {{- with .Values.syncer.nodeSelector }} - {{- toYaml . | nindent 8 }} - {{- end }} restartPolicy: {{ .Values.syncer.restartPolicy }} {{- end }} \ No newline at end of file diff --git a/charts/cloudstack-csi/values.yaml b/charts/cloudstack-csi/values.yaml index 9ca492c..c733046 100644 --- a/charts/cloudstack-csi/values.yaml +++ b/charts/cloudstack-csi/values.yaml @@ -139,7 +139,6 @@ controller: # Overrides the image tag whose default is v{{ .Chart.AppVersion }} tag: "" pullPolicy: IfNotPresent - ## Set number of replicas (If replicaCount set to 1 by default leader-elect will be false) replicaCount: 2 # Extra arguments passed to cloudstack-csi-driver controller. extraArgs: [] @@ -335,6 +334,13 @@ syncer: nodeSelector: {} podAnnotations: {} podLabels: {} + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node.kubernetes.io/not-ready + operator: Exists + effect: NoSchedule resources: {} serviceAccount: create: true