csi-wrapper: add example for azuredisk-csi-driver

This example follows the same instructions as the azurefile-csi-driver example, but uses the azuredisk-csi-driver instead to provision storage through Azure disks. Signed-off-by: Daniel Weiße <[email protected]>
confidential-containers · Nov 5, 2024 · 489c96e · 489c96e
1 parent 5b10093
commit 489c96e
Show file tree

Hide file tree

Showing 10 changed files with 638 additions and 100 deletions.
diff --git a/src/csi-wrapper/examples/azure/README.md b/src/csi-wrapper/examples/azure/README.md
diff --git a/src/csi-wrapper/examples/azure/disk/azure-disk-csi-wrapper-podvm.yaml b/src/csi-wrapper/examples/azure/disk/azure-disk-csi-wrapper-podvm.yaml
@@ -0,0 +1,52 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: csi-azuredisk-podvm-sa
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: azure-disk-csi-wrapper-podvm
+rules:
+  - apiGroups: ['confidentialcontainers.org']
+    resources: ['peerpodvolumes']
+    verbs: ['get', 'list', 'watch', 'create', 'delete', 'update', 'patch']
+  - apiGroups: ['confidentialcontainers.org']
+    resources: ['peerpodvolumes/status']
+    verbs: ['update']
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: azure-disk-csi-wrapper-podvm-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-azuredisk-podvm-sa
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: azure-disk-csi-wrapper-podvm
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: azure-disk-csi-wrapper-podvm
+  namespace: default
+rules:
+  - apiGroups: ['']
+    resources: ['secrets']
+    verbs: ['get', 'list', 'watch']
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: azure-disk-csi-wrapper-podvm-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-azuredisk-podvm-sa
+    namespace: default
+roleRef:
+  kind: Role
+  name: azure-disk-csi-wrapper-podvm
+  apiGroup: rbac.authorization.k8s.io
diff --git a/src/csi-wrapper/examples/azure/disk/azure-disk-csi-wrapper-runner.yaml b/src/csi-wrapper/examples/azure/disk/azure-disk-csi-wrapper-runner.yaml
@@ -0,0 +1,76 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: azure-disk-csi-wrapper-runner
+rules:
+  - apiGroups: ['']
+    resources: ['secrets']
+    verbs: ['get', 'list', 'watch']
+  - apiGroups: ['']
+    resources: ['persistentvolumes']
+    verbs: ['get', 'list', 'watch', 'create', 'delete']
+  - apiGroups: ['']
+    resources: ['persistentvolumeclaims']
+    verbs: ['get', 'list', 'watch', 'update']
+  - apiGroups: ['storage.k8s.io']
+    resources: ['storageclasses']
+    verbs: ['get', 'list', 'watch']
+  - apiGroups: ['']
+    resources: ['events']
+    verbs: ['list', 'watch', 'create', 'update', 'patch']
+  - apiGroups: ['snapshot.storage.k8s.io']
+    resources: ['volumesnapshots']
+    verbs: ['get', 'list']
+  - apiGroups: ['snapshot.storage.k8s.io']
+    resources: ['volumesnapshotcontents']
+    verbs: ['get', 'list']
+  - apiGroups: ['storage.k8s.io']
+    resources: ['csinodes']
+    verbs: ['get', 'list', 'watch']
+  - apiGroups: ['']
+    resources: ['nodes']
+    verbs: ['get', 'list', 'watch']
+  - apiGroups: ['storage.k8s.io']
+    resources: ['volumeattachments']
+    verbs: ['get', 'list', 'watch']
+  - apiGroups: ['']
+    resources: ['configmaps']
+    verbs: ['get', 'list']
+  - apiGroups: ['confidentialcontainers.org']
+    resources: ['peerpodvolumes']
+    verbs: ['get', 'list', 'watch', 'create', 'delete', 'update', 'patch']
+  - apiGroups: ['confidentialcontainers.org']
+    resources: ['peerpodvolumes/status']
+    verbs: ['update']
+  - apiGroups: ['']
+    resources: ['pods']
+    verbs: ['get', 'list']
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: azure-disk-csi-wrapper-controller-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-azuredisk-controller-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: azure-disk-csi-wrapper-runner
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: azure-disk-csi-wrapper-node-binding
+subjects:
+  - kind: ServiceAccount
+    name: csi-azuredisk-node-sa
+    namespace: kube-system
+roleRef:
+  kind: ClusterRole
+  name: azure-disk-csi-wrapper-runner
+  apiGroup: rbac.authorization.k8s.io
diff --git a/src/csi-wrapper/examples/azure/disk/azure-disk-storageclass-for-peerpod.yaml b/src/csi-wrapper/examples/azure/disk/azure-disk-storageclass-for-peerpod.yaml
@@ -0,0 +1,11 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: azure-disk-storage
+provisioner: disk.csi.azure.com
+parameters:
+  skuName: StandardSSD_LRS # available values: StandardSSD_LRS, StandardSSD_ZRS, Premium_LRS, Premium_ZRS, etc.
+  peerpod: 'true'
+reclaimPolicy: Delete
+allowVolumeExpansion: true
+volumeBindingMode: Immediate
diff --git a/src/csi-wrapper/examples/azure/disk/cloud-config.yaml b/src/csi-wrapper/examples/azure/disk/cloud-config.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cloud-config
+  namespace: kube-system
+type: Opaque
+data:
+  cloud-config: "@@CLOUD_CONFIG_BASE64@@"
diff --git a/src/csi-wrapper/examples/azure/disk/dynamic-pvc.yaml b/src/csi-wrapper/examples/azure/disk/dynamic-pvc.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pvc-azuredisk
+spec:
+  storageClassName: azure-disk-storage
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
diff --git a/src/csi-wrapper/examples/azure/disk/nginx-kata-with-my-pvc-and-csi-wrapper.yaml b/src/csi-wrapper/examples/azure/disk/nginx-kata-with-my-pvc-and-csi-wrapper.yaml
@@ -0,0 +1,84 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-pv-disk
+  labels:
+    app: nginx
+  namespace: default
+spec:
+  runtimeClassName: kata-remote
+  containers:
+    - name: azure-disk-podvm-node-driver
+      env:
+        - name: KUBE_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+      command: ['/bin/sh']
+      args:
+        - -c
+        - |
+          mount -t devtmpfs none /dev && \
+          exec /azurediskplugin --v=4 --disable-update-cache
+      image: mcr.microsoft.com/oss/kubernetes-csi/azuredisk-csi:v1.31.0
+      imagePullPolicy: Always
+      securityContext:
+        privileged: true
+        runAsNonRoot: false
+        runAsUser: 0
+      ports:
+        - containerPort: 9808
+          name: healthz
+          protocol: TCP
+      volumeMounts:
+        - name: kubelet-data-dir
+          mountPath: /var/lib/kubelet
+          mountPropagation: Bidirectional
+        - mountPath: /tmp
+          name: plugin-dir
+    - name: csi-podvm-wrapper
+      env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAME_SPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: POD_UID
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.uid
+        - name: POD_NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+      args:
+        - -v=2
+        - --endpoint=/tmp/csi-podvm-wrapper.sock
+        - --target-endpoint=/tmp/csi.sock
+        - --namespace=kube-system
+      image: quay.io/confidential-containers/csi-podvm-wrapper:latest
+      imagePullPolicy: Always
+      volumeMounts:
+        - mountPath: /tmp
+          name: plugin-dir
+    - name: nginx
+      image: nginx:stable
+      imagePullPolicy: Always
+      volumeMounts:
+        - name: my-volume
+          mountPath: /mount-path
+          mountPropagation: HostToContainer
+  serviceAccountName: csi-azuredisk-podvm-sa
+  volumes:
+    - name: kubelet-data-dir
+      hostPath:
+        path: /var/lib/kubelet
+        type: Directory
+    - emptyDir: {}
+      name: plugin-dir
+    - name: my-volume
+      persistentVolumeClaim:
+        claimName: pvc-azuredisk
diff --git a/src/csi-wrapper/examples/azure/disk/patch-controller.yaml b/src/csi-wrapper/examples/azure/disk/patch-controller.yaml
@@ -0,0 +1,48 @@
+spec:
+  replicas: 1
+  template:
+    spec:
+      containers:
+        - name: csi-controller-wrapper
+          env:
+            - name: POD_NAME_SPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          args:
+            - -v=2
+            - --endpoint=/csi/csi-controller-wrapper.sock
+            - --target-endpoint=/csi/csi.sock
+            - --namespace=$(POD_NAME_SPACE)
+          image: quay.io/confidential-containers/csi-controller-wrapper:latest
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+
+        - name: azuredisk
+          args:
+            - "--v=5"
+            - "--endpoint=$(CSI_ENDPOINT)"
+            - "--metrics-address=0.0.0.0:29604"
+            - "--user-agent-suffix=OSS-kubectl"
+            - "--disable-avset-nodes=false"
+            - "--allow-empty-cloud-config=false"
+            - "--cloud-config-secret-name=cloud-config"
+            - "--cloud-config-secret-namespace=kube-system"
+
+        - name: csi-attacher
+          args:
+            - -v=2
+            - --csi-address=/csi/csi-controller-wrapper.sock
+            - --timeout=900s
+
+        - name: csi-provisioner
+          env:
+            - name: ADDRESS
+              value: /csi/csi-controller-wrapper.sock
+
+        - name: csi-resizer
+          env:
+            - name: ADDRESS
+              value: /csi/csi-controller-wrapper.sock
diff --git a/src/csi-wrapper/examples/azure/disk/patch-node.yaml b/src/csi-wrapper/examples/azure/disk/patch-node.yaml
@@ -0,0 +1,51 @@
+spec:
+  template:
+    spec:
+      containers:
+        - name: csi-node-wrapper
+          args:
+            - --v=2
+            - --endpoint=/csi/csi-node-wrapper.sock
+            - --target-endpoint=/csi/csi.sock
+            - --namespace=$(POD_NAME_SPACE)
+          env:
+            - name: POD_NAME_SPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          image: quay.io/confidential-containers/csi-node-wrapper:latest
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - mountPath: /csi
+              name: socket-dir
+            - mountPath: /run/peerpod/
+              name: podvminfo-dir
+            - mountPath: /run/kata-containers/shared/direct-volumes
+              name: kata-direct-volumes-dir
+            - mountPath: /var/lib/kubelet
+              name: kubelet-data-dir
+
+        - name: node-driver-registrar
+          env:
+            - name: ADDRESS
+              value: /csi/csi-node-wrapper.sock
+            - name: DRIVER_REG_SOCK_PATH
+              value: /var/lib/kubelet/plugins/disk.csi.azure.com/csi-node-wrapper.sock
+
+      volumes:
+        - name: podvminfo-dir
+          hostPath:
+            path: /run/peerpod/
+            type: Directory
+        - name: kata-direct-volumes-dir
+          hostPath:
+            path: /run/kata-containers/shared/direct-volumes
+            type: DirectoryOrCreate
+        - name: kubelet-data-dir
+          hostPath:
+            path: /var/lib/kubelet
+            type: Directory
diff --git a/src/csi-wrapper/examples/azure/disk/static-pvc.yaml b/src/csi-wrapper/examples/azure/disk/static-pvc.yaml
@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pvc-azuredisk
+spec:
+  storageClassName: azure-disk-storage
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi
+  volumeName: pv-azuredisk
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: pv-azuredisk
+spec:
+  capacity:
+    storage: 10Gi
+  accessModes:
+    - ReadWriteOnce
+  persistentVolumeReclaimPolicy: Retain
+  storageClassName: azure-disk-storage
+  csi:
+    driver: disk.csi.azure.com
+    volumeHandle: '@@AZURE_DISK_ID@@'
+    volumeAttributes:
+      fsType: ext4
+      peerpod: 'true' # Indicator for csi-wrapper that this is a volume should be used for a peerpod volume
+---