Add CSI driver for VMware Cloud Director (#2092)

* Add CSI driver for VMware Cloud Director Signed-off-by: Waleed Malik <[email protected]> * vcd: improve handling of storage profile and vApp name Signed-off-by: Waleed Malik <[email protected]> * Update generated code Signed-off-by: Waleed Malik <[email protected]> * Refactored code Signed-off-by: Waleed Malik <[email protected]> * vcloud-csi-configmap shouldn't be immutable Signed-off-by: Waleed Malik <[email protected]>
kubermatic · Jun 8, 2022 · 5b5e9d7 · 5b5e9d7
1 parent bf69ab5
commit 5b5e9d7
Show file tree

Hide file tree

Showing 16 changed files with 480 additions and 46 deletions.
diff --git a/addons/csi-vmware-cloud-director/csi-controller.yaml b/addons/csi-vmware-cloud-director/csi-controller.yaml
@@ -0,0 +1,186 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-vcd-controller-sa
+  namespace: kube-system
+
+---
+# external attacher
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-attacher-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments/status"]
+    verbs: ["patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-attacher-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-vcd-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-attacher-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+# external Provisioner
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-provisioner-role
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "delete", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["get", "list"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-provisioner-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-vcd-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-provisioner-role
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: csi-vcd-controllerplugin
+  namespace: kube-system
+spec:
+  serviceName: csi-vcd-controller-service
+  replicas: 1
+  selector:
+    matchLabels:
+      app: csi-vcd-controllerplugin
+  template:
+    metadata:
+      labels:
+        app: csi-vcd-controllerplugin
+    spec:
+      serviceAccount: csi-vcd-controller-sa
+      dnsPolicy: Default
+      containers:
+        - name: csi-attacher
+          image: k8s.gcr.io/sig-storage/csi-attacher:v3.2.1
+          imagePullPolicy: IfNotPresent
+          args:
+            - --csi-address=$(ADDRESS)
+            - --timeout=180s
+            - --v=5
+          env:
+            - name: ADDRESS
+              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: csi-provisioner
+          image: k8s.gcr.io/sig-storage/csi-provisioner:v2.2.2
+          imagePullPolicy: IfNotPresent
+          args:
+            - --csi-address=$(ADDRESS)
+            - --timeout=300s
+            - --v=5
+          env:
+            - name: ADDRESS
+              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: vcd-csi-plugin
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: {{ .InternalImages.Get "VMwareCloudDirectorCSI" }}
+          imagePullPolicy: IfNotPresent
+          command:
+            - /opt/vcloud/bin/cloud-director-named-disk-csi-driver
+            - --cloud-config=/etc/kubernetes/vcloud/vcloud-csi-config.yaml
+            - --endpoint=$(CSI_ENDPOINT)
+            - --v=5
+          env:
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CSI_ENDPOINT
+              value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+            - name: pods-probe-dir
+              mountPath: /dev
+              mountPropagation: HostToContainer
+            - name: pv-dir
+              mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi/pv
+              mountPropagation: "Bidirectional"
+            - name: vcloud-csi-config-volume
+              mountPath: /etc/kubernetes/vcloud
+            - name: vcloud-basic-auth-volume
+              mountPath: /etc/kubernetes/vcloud/basic-auth
+      volumes:
+        - name: socket-dir
+          emptyDir: {}
+        - name: pods-probe-dir
+          hostPath:
+            path: /dev
+            type: Directory
+        - name: pv-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/kubernetes.io/csi/pv
+            type: DirectoryOrCreate
+        - name: vcloud-csi-config-volume
+          configMap:
+            name: vcloud-csi-configmap
+        - name: vcloud-basic-auth-volume
+          secret:
+            secretName: vcloud-basic-auth
+---
diff --git a/addons/csi-vmware-cloud-director/csi-driver.yaml b/addons/csi-vmware-cloud-director/csi-driver.yaml
@@ -0,0 +1,9 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  name: named-disk.csi.cloud-director.vmware.com
+spec:
+  attachRequired: true
+  podInfoOnMount: false
+---
diff --git a/addons/csi-vmware-cloud-director/csi-node.yaml b/addons/csi-vmware-cloud-director/csi-node.yaml
@@ -0,0 +1,150 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-vcd-node-sa
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-nodeplugin-role
+rules:
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-nodeplugin-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-vcd-node-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: csi-nodeplugin-role
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: csi-vcd-nodeplugin
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: csi-vcd-nodeplugin
+  template:
+    metadata:
+      labels:
+        app: csi-vcd-nodeplugin
+    spec:
+      serviceAccount: csi-vcd-node-sa
+      hostNetwork: true
+      dnsPolicy: Default
+      containers:
+        - name: node-driver-registrar
+          image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0
+          imagePullPolicy: "IfNotPresent"
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+            - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                  [
+                    "/bin/sh",
+                    "-c",
+                    "rm -rf /registration/named-disk.csi.cloud-director.vmware.com /registration/named-disk.csi.cloud-director.vmware.com-reg.sock",
+                  ]
+          env:
+            - name: ADDRESS
+              value: /csi/csi.sock
+            - name: DRIVER_REG_SOCK_PATH
+              value: /var/lib/kubelet/csi-plugins/named-disk.csi.cloud-director.vmware.com/csi.sock
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
+        - name: vcd-csi-plugin
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: {{ .InternalImages.Get "VMwareCloudDirectorCSI" }}
+          imagePullPolicy: IfNotPresent
+          command:
+            - /opt/vcloud/bin/cloud-director-named-disk-csi-driver
+            - --nodeid=$(NODE_ID)
+            - --endpoint=$(CSI_ENDPOINT)
+            - --cloud-config=/etc/kubernetes/vcloud/vcloud-csi-config.yaml
+            - --v=5
+          env:
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CSI_ENDPOINT
+              value: unix://csi/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: kubelet-dir
+              mountPath: /var/lib/kubelet/pods
+              mountPropagation: "Bidirectional"
+            - name: pods-cloud-data
+              mountPath: /var/lib/cloud/data
+              readOnly: true
+            - name: pods-probe-dir
+              mountPath: /dev
+              mountPropagation: "HostToContainer"
+            - name: pv-dir
+              mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi/pv
+              mountPropagation: "Bidirectional"
+            - name: vcloud-csi-config-volume
+              mountPath: /etc/kubernetes/vcloud
+            - name: vcloud-basic-auth-volume
+              mountPath: /etc/kubernetes/vcloud/basic-auth
+      volumes:
+        - name: socket-dir
+          hostPath:
+            path: /var/lib/kubelet/csi-plugins/named-disk.csi.cloud-director.vmware.com
+            type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry/
+            type: DirectoryOrCreate
+        - name: kubelet-dir
+          hostPath:
+            path: /var/lib/kubelet/pods
+            type: DirectoryOrCreate
+        - name: pods-cloud-data
+          hostPath:
+            path: /var/lib/cloud/data
+            type: DirectoryOrCreate
+        - name: pods-probe-dir
+          hostPath:
+            path: /dev
+            type: Directory
+        - name: pv-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/kubernetes.io/csi/pv
+            type: DirectoryOrCreate
+        - name: vcloud-csi-config-volume
+          configMap:
+            name: vcloud-csi-configmap
+        - name: vcloud-basic-auth-volume
+          secret:
+            secretName: vcloud-basic-auth
+---
diff --git a/addons/csi-vmware-cloud-director/vcloud-basic-auth.yaml b/addons/csi-vmware-cloud-director/vcloud-basic-auth.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: vcloud-basic-auth
+  namespace: kube-system
+data:
+  username: {{ required "Please provide VCD_USER" .Credentials.VCD_USER | b64enc }}
+  password: {{ required "Please provide VCD_PASSWORD" .Credentials.VCD_PASSWORD | b64enc }}
diff --git a/addons/csi-vmware-cloud-director/vcloud-csi-config.yaml b/addons/csi-vmware-cloud-director/vcloud-csi-config.yaml
@@ -0,0 +1,16 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vcloud-csi-configmap
+  namespace: kube-system
+data:
+  # host shouldn't have the `/api` suffix.
+  vcloud-csi-config.yaml: |+
+    vcd:
+      host: {{ required "Please provide VCD_URL" (trimSuffix "/api" .Credentials.VCD_URL) }}
+      org: {{ required "Please provide VCD_ORG" .Credentials.VCD_ORG }}
+      vdc: {{ required "Please provide VCD_VDC" .Credentials.VCD_VDC }}
+      vAppName: {{ .Config.CloudProvider.VMwareCloudDirector.VApp }}
+    clusterid: {{ .Config.Name }}
+---
diff --git a/addons/default-storage-class/storage-class.yaml b/addons/default-storage-class/storage-class.yaml
@@ -206,3 +206,17 @@ metadata:
 driver: dobs.csi.digitalocean.com
 deletionPolicy: Delete
 {{ end }}
+
+{{ if eq .Config.CloudProvider.CloudProviderName "vmwareCloudDirector" }}
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+  name: vcd-disk-dev
+provisioner: named-disk.csi.cloud-director.vmware.com
+reclaimPolicy: Delete
+parameters:
+  storageProfile: {{ default .Config.CloudProvider.VMwareCloudDirector.StorageProfile .Params.storageProfile | quote }}
+  filesystem: {{ default "ext4" .Params.filesystem | quote }}
+{{ end }}