feat: rootless dind (#521)

codefresh-io · Nov 18, 2024 · f4ffb45 · f4ffb45
1 parent 47eb2a8
commit f4ffb45
Show file tree

Hide file tree

Showing 16 changed files with 228 additions and 70 deletions.
diff --git a/.gitignore b/.gitignore
@@ -42,10 +42,10 @@ venona/venona
 **/*.tgz
 **/charts/**/charts
 **/dry-run.yaml
-**/values-dev.yaml
+**/values-dev**.yaml
 
 # coverage
 **/cover
 
 # debug
-**/debug
+**/.debug
diff --git a/charts/cf-runtime/.ci/values-rootless.yaml b/charts/cf-runtime/.ci/values-rootless.yaml
@@ -0,0 +1,32 @@
+volumeProvisioner:
+  env:
+    IS_ROOTLESS: true
+  dind-lv-monitor:
+    image:
+      tag: 1.30.0-rootless
+      digest: sha256:712e549e6e843b04684647f17e0973f8047e0d60e6e8b38a693ea64dc75b0479
+    containerSecurityContext:
+      runAsUser: 1000
+    podSecurityContext:
+      fsGroup: 1000
+      fsGroupChangePolicy: "OnRootMismatch"
+    volumePermissions:
+      enabled: true
+
+runtime:
+  dind:
+    image:
+      tag: 26.1.4-1.28.10-rootless
+      digest: sha256:59dfc004eb22a8f09c8a3d585271a055af9df4591ab815bca418c24a2077f5c8
+    userVolumeMounts:
+      dind:
+        name: dind
+        mountPath: /home/rootless/.local/share/docker
+    containerSecurityContext:
+      privileged: true
+      runAsUser: 1000
+    podSecurityContext:
+      fsGroup: 1000
+      fsGroupChangePolicy: "OnRootMismatch"
+    volumePermissions:
+      enabled: true
diff --git a/charts/cf-runtime/Chart.yaml b/charts/cf-runtime/Chart.yaml
@@ -1,7 +1,7 @@
 apiVersion: v2
 description: A Helm chart for Codefresh Runner
 name: cf-runtime
-version: 7.0.1
+version: 7.1.0
 keywords:
   - codefresh
   - runner
@@ -17,8 +17,14 @@ annotations:
   artifacthub.io/containsSecurityUpdates: "false"
   # Supported kinds: `added`, `changed`, `deprecated`, `removed`, `fixed`, `security`:
   artifacthub.io/changes: |
-    - kind: security
-      description: "updating k8s-agent"
+    - kind: changed
+      description: "(rootless runtime) Update dind-volume-provisioner and dind-volume-utils images"
+    - kind: added
+      description: "(rootless runtime) Add values-rootless.yaml example"
+    - kind: changed
+      description: "(dind) Update runtime environment spec to allow specifying podSecurityContext and containerSecurityContext for dind"
+    - kind: fixed
+      description: "Update helm unittests"
 dependencies:
   - name: cf-common
     repository: oci://quay.io/codefresh/charts

diff --git a/charts/cf-runtime/README.md b/charts/cf-runtime/README.md
@@ -1,6 +1,6 @@
 ## Codefresh Runner
 
-![Version: 7.0.1](https://img.shields.io/badge/Version-7.0.1-informational?style=flat-square)
+![Version: 7.1.0](https://img.shields.io/badge/Version-7.1.0-informational?style=flat-square)
 
 Helm chart for deploying [Codefresh Runner](https://codefresh.io/docs/docs/installation/codefresh-runner/) to Kubernetes.
 
@@ -715,14 +715,48 @@ volumeProvisioner:
 ### Rootless DinD
 
 DinD pod runs a `priviliged` container with **rootfull** docker.
-To run the docker daemon as non-root user (**rootless** mode), change dind image tag:
 
-`values.yaml`
+To run the docker daemon as non-root user (**rootless** mode), refer to `values-rootless.yaml`:
+
 ```yaml
+volumeProvisioner:
+  env:
+    IS_ROOTLESS: true
+  # -- Only if local volumes are used as backend storage (ignored for ebs/ebs-csi disks)
+  dind-lv-monitor:
+    image:
+      tag: 1.30.0-rootless
+      digest: sha256:712e549e6e843b04684647f17e0973f8047e0d60e6e8b38a693ea64dc75b0479
+    containerSecurityContext:
+      runAsUser: 1000
+    podSecurityContext:
+      fsGroup: 1000
+      # Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#configure-volume-permission-and-ownership-change-policy-for-pods
+      fsGroupChangePolicy: "OnRootMismatch"
+    # -- Enable initContainer to run chmod for /var/lib/codefresh/dind-volumes on host nodes
+    volumePermissions:
+      enabled: false
+
 runtime:
   dind:
     image:
-      tag: rootless
+      tag: 26.1.4-1.28.10-rootless
+      digest: sha256:59dfc004eb22a8f09c8a3d585271a055af9df4591ab815bca418c24a2077f5c8
+    userVolumeMounts:
+      dind:
+        name: dind
+        mountPath: /home/rootless/.local/share/docker
+    containerSecurityContext:
+      privileged: true
+      runAsUser: 1000
+    podSecurityContext:
+      fsGroup: 1000
+      # Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#configure-volume-permission-and-ownership-change-policy-for-pods
+      fsGroupChangePolicy: "OnRootMismatch"
+    # -- Enable initContainer to run chmod for /home/rootless in DinD pod
+    # !!! Will slow down dind pod startup
+    volumePermissions:
+      enabled: true
 ```
 
 ### ARM
@@ -1143,13 +1177,15 @@ Go to [https://<YOUR_ONPREM_DOMAIN_HERE>/admin/runtime-environments/system](http
 | runtime.accounts | list | `[]` | (for On-Premise only) Assign accounts to runtime (list of account ids) |
 | runtime.agent | bool | `true` | (for On-Premise only) Enable agent |
 | runtime.description | string | `""` | Runtime description |
-| runtime.dind | object | `{"affinity":{},"env":{"DOCKER_ENABLE_DEPRECATED_PULL_SCHEMA_1_IMAGE":true},"image":{"digest":"sha256:ccaf26ab24db0e00760beba79ce1810a12aef5be296f538ceab416af9ec481f7","pullPolicy":"IfNotPresent","registry":"quay.io","repository":"codefresh/dind","tag":"26.1.4-1.28.7"},"nodeSelector":{},"podAnnotations":{},"podLabels":{},"pvcs":{"dind":{"annotations":{},"name":"dind","reuseVolumeSelector":"codefresh-app,io.codefresh.accountName","reuseVolumeSortOrder":"pipeline_id","storageClassName":"{{ include \"dind-volume-provisioner.storageClassName\" . }}","volumeSize":"16Gi"}},"resources":{"limits":{"cpu":"400m","memory":"800Mi"},"requests":null},"schedulerName":"","serviceAccount":"codefresh-engine","terminationGracePeriodSeconds":30,"tolerations":[],"userAccess":true,"userVolumeMounts":{},"userVolumes":{}}` | Parameters for DinD (docker-in-docker) pod (aka "runtime" pod). |
+| runtime.dind | object | `{"affinity":{},"containerSecurityContext":{},"env":{"DOCKER_ENABLE_DEPRECATED_PULL_SCHEMA_1_IMAGE":true},"image":{"digest":"sha256:ccaf26ab24db0e00760beba79ce1810a12aef5be296f538ceab416af9ec481f7","pullPolicy":"IfNotPresent","registry":"quay.io","repository":"codefresh/dind","tag":"26.1.4-1.28.7"},"nodeSelector":{},"podAnnotations":{},"podLabels":{},"podSecurityContext":{},"pvcs":{"dind":{"annotations":{},"name":"dind","reuseVolumeSelector":"codefresh-app,io.codefresh.accountName","reuseVolumeSortOrder":"pipeline_id","storageClassName":"{{ include \"dind-volume-provisioner.storageClassName\" . }}","volumeSize":"16Gi"}},"resources":{"limits":{"cpu":"400m","memory":"800Mi"},"requests":null},"schedulerName":"","serviceAccount":"codefresh-engine","terminationGracePeriodSeconds":30,"tolerations":[],"userAccess":true,"userVolumeMounts":{},"userVolumes":{},"volumePermissions":{"enabled":false,"image":{"digest":"sha256:2995c82e8e723d9a5c8585cb8e901d1c50e3c2759031027d3bff577449435157","registry":"docker.io","repository":"alpine","tag":3.18},"resources":{},"securityContext":{"runAsUser":0}}}` | Parameters for DinD (docker-in-docker) pod (aka "runtime" pod). |
 | runtime.dind.affinity | object | `{}` | Set affinity |
+| runtime.dind.containerSecurityContext | object | `{}` | Set container security context. |
 | runtime.dind.env | object | `{"DOCKER_ENABLE_DEPRECATED_PULL_SCHEMA_1_IMAGE":true}` | Set additional env vars. |
 | runtime.dind.image | object | `{"digest":"sha256:ccaf26ab24db0e00760beba79ce1810a12aef5be296f538ceab416af9ec481f7","pullPolicy":"IfNotPresent","registry":"quay.io","repository":"codefresh/dind","tag":"26.1.4-1.28.7"}` | Set dind image. |
 | runtime.dind.nodeSelector | object | `{}` | Set node selector. |
 | runtime.dind.podAnnotations | object | `{}` | Set pod annotations. |
 | runtime.dind.podLabels | object | `{}` | Set pod labels. |
+| runtime.dind.podSecurityContext | object | `{}` | Set security context for the pod. |
 | runtime.dind.pvcs | object | `{"dind":{"annotations":{},"name":"dind","reuseVolumeSelector":"codefresh-app,io.codefresh.accountName","reuseVolumeSortOrder":"pipeline_id","storageClassName":"{{ include \"dind-volume-provisioner.storageClassName\" . }}","volumeSize":"16Gi"}}` | PV claim spec parametes. |
 | runtime.dind.pvcs.dind | object | `{"annotations":{},"name":"dind","reuseVolumeSelector":"codefresh-app,io.codefresh.accountName","reuseVolumeSortOrder":"pipeline_id","storageClassName":"{{ include \"dind-volume-provisioner.storageClassName\" . }}","volumeSize":"16Gi"}` | Default dind PVC parameters |
 | runtime.dind.pvcs.dind.annotations | object | `{}` | PV annotations. |
@@ -1234,7 +1270,7 @@ Go to [https://<YOUR_ONPREM_DOMAIN_HERE>/admin/runtime-environments/system](http
 | volumeProvisioner.dind-lv-monitor | object | See below | `dind-lv-monitor` DaemonSet parameters (local volumes cleaner) |
 | volumeProvisioner.enabled | bool | `true` | Enable volume-provisioner |
 | volumeProvisioner.env | object | `{}` | Add additional env vars |
-| volumeProvisioner.image | object | `{"digest":"sha256:c036ad717391debdf43f8da337b81b5df0e79de274d2d9af1425c675b0296dda","registry":"quay.io","repository":"codefresh/dind-volume-provisioner","tag":"1.35.0"}` | Set image |
+| volumeProvisioner.image | object | `{"digest":"sha256:ede6f663c912a08b7d335b5ec5518ccc266b27c431d0854d22971005992adc5d","registry":"quay.io","repository":"codefresh/dind-volume-provisioner","tag":"1.35.2"}` | Set image |
 | volumeProvisioner.nodeSelector | object | `{}` | Set node selector |
 | volumeProvisioner.podAnnotations | object | `{}` | Set pod annotations |
 | volumeProvisioner.podSecurityContext | object | See below | Set security context for the pod |

diff --git a/charts/cf-runtime/README.md.gotmpl b/charts/cf-runtime/README.md.gotmpl
@@ -717,14 +717,48 @@ volumeProvisioner:
 ### Rootless DinD
 
 DinD pod runs a `priviliged` container with **rootfull** docker.
-To run the docker daemon as non-root user (**rootless** mode), change dind image tag:
 
-`values.yaml`
+To run the docker daemon as non-root user (**rootless** mode), refer to `values-rootless.yaml`:
+
 ```yaml
+volumeProvisioner:
+  env:
+    IS_ROOTLESS: true
+  # -- Only if local volumes are used as backend storage (ignored for ebs/ebs-csi disks)
+  dind-lv-monitor:
+    image:
+      tag: 1.30.0-rootless
+      digest: sha256:712e549e6e843b04684647f17e0973f8047e0d60e6e8b38a693ea64dc75b0479
+    containerSecurityContext:
+      runAsUser: 1000
+    podSecurityContext:
+      fsGroup: 1000
+      # Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#configure-volume-permission-and-ownership-change-policy-for-pods
+      fsGroupChangePolicy: "OnRootMismatch"
+    # -- Enable initContainer to run chmod for /var/lib/codefresh/dind-volumes on host nodes
+    volumePermissions:
+      enabled: false
+
 runtime:
   dind:
     image:
-      tag: rootless
+      tag: 26.1.4-1.28.10-rootless
+      digest: sha256:59dfc004eb22a8f09c8a3d585271a055af9df4591ab815bca418c24a2077f5c8
+    userVolumeMounts:
+      dind:
+        name: dind
+        mountPath: /home/rootless/.local/share/docker
+    containerSecurityContext:
+      privileged: true
+      runAsUser: 1000
+    podSecurityContext:
+      fsGroup: 1000
+      # Ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#configure-volume-permission-and-ownership-change-policy-for-pods
+      fsGroupChangePolicy: "OnRootMismatch"
+    # -- Enable initContainer to run chmod for /home/rootless in DinD pod
+    # !!! Will slow down dind pod startup
+    volumePermissions:
+      enabled: true
 ```
 
 ### ARM

diff --git a/charts/cf-runtime/templates/_components/volume-provisioner/_daemonset.yaml b/charts/cf-runtime/templates/_components/volume-provisioner/_daemonset.yaml
@@ -37,7 +37,7 @@ spec:
         args:
          - -ec
          - |
-           chown -R {{ .Values.podSecurityContext.runAsUser }}:{{ .Values.podSecurityContext.fsGroup }} {{ $localVolumeParentDir }}
+           chown -R {{ .Values.containerSecurityContext.runAsUser }}:{{ .Values.podSecurityContext.fsGroup }} {{ $localVolumeParentDir }}
         volumeMounts:
         - mountPath: {{ $localVolumeParentDir }}
           name: dind-volume-dir
@@ -95,4 +95,4 @@ spec:
         {{- toYaml . | nindent 6 }}
       {{- end }}
 {{- end }}
-{{- end -}}
+{{- end -}}
diff --git a/charts/cf-runtime/templates/runtime/runtime-env-spec-tmpl.yaml b/charts/cf-runtime/templates/runtime/runtime-env-spec-tmpl.yaml
@@ -183,6 +183,34 @@ dockerDaemonScheduler:
       secret:
         secretName: codefresh-certs-server
   {{- end }}
+  {{- with $dindContext.podSecurityContext }}
+  podSecurityContext: {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with $dindContext.containerSecurityContext }}
+  containerSecurityContext: {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- if $dindContext.volumePermissions.enabled }}
+  initContainers:
+  - name: volume-permissions
+    image: {{ include (printf "%s.image.name" $cfCommonTplSemver ) (dict "image" $dindContext.volumePermissions.image "context" .) }}
+    imagePullPolicy: {{ $dindContext.volumePermissions.image.pullPolicy | default "Always" }}
+    command:
+      - /bin/sh
+    args:
+      - -ec
+      - |
+        chown -R {{ $dindContext.containerSecurityContext.runAsUser }}:{{ $dindContext.podSecurityContext.fsGroup }} /home/rootless/.local/share/docker
+    volumeMounts:
+    - mountPath: /home/rootless/.local/share/docker
+      name: dind
+    {{- if eq ( toString ( $dindContext.volumePermissions.securityContext.runAsUser )) "auto" }}
+    securityContext: {{- omit $dindContext.volumePermissions.securityContext "runAsUser" | toYaml | nindent 6 }}
+    {{- else }}
+    securityContext: {{- $dindContext.volumePermissions.securityContext | toYaml | nindent 6 }}
+    {{- end }}
+    resources:
+      {{- toYaml $dindContext.volumePermissions.resources | nindent 6 }}
+  {{- end }}
 extends: {{- toYaml .Values.runtime.runtimeExtends | nindent 2 }}
   {{- if .Values.runtime.description }}
 description: {{ .Values.runtime.description }}

diff --git a/charts/cf-runtime/tests/private-registry/private_registry_test.yaml b/charts/cf-runtime/tests/private-registry/private_registry_test.yaml
@@ -4,21 +4,15 @@ values:
   - ../values.yaml
   - ../values-private-registry.yaml
 templates:
-  - templates/hooks/post-install/cm-update-runtime.yaml
-  - templates/runner/deployment.yaml
-  - templates/volume-provisioner/deployment.yaml
-  - templates/volume-provisioner/daemonset.yaml
-  - templates/volume-provisioner/cronjob.yaml
-  - templates/monitor/deployment.yaml
-  - templates/app-proxy/deployment.yaml
+  - templates/**.yaml
 release:
   name: cf-runtime
   namespace: codefresh
   revision: 1
   upgrade: true
-chart:
-  version: 1.0.0
-  appVersion: 1.0.0
+# chart:
+#   version: 1.0.0
+#   appVersion: 1.0.0
 tests:
   - it: Test private registry in runtime spec
     template: templates/hooks/post-install/cm-update-runtime.yaml

diff --git a/charts/cf-runtime/tests/runner/runner_test.yaml b/charts/cf-runtime/tests/runner/runner_test.yaml
@@ -3,16 +3,13 @@ suite: runner test
 values:
   - ../values.yaml
 templates:
-  - templates/runner/deployment.yaml
-  - templates/runner/rbac.yaml
-  - templates/runner/secret.yaml
+  - templates/**.yaml
 release:
   name: cf-runtime
   namespace: codefresh
   revision: 1
   upgrade: true
 chart:
-  version: 1.0.0
   appVersion: 1.0.0
 tests:
   - it: Test runner default metadata
@@ -24,14 +21,12 @@ tests:
           of: Deployment
       - isNull:
           path: metadata.annotations
-      - equal:
+      - isSubset:
           path: metadata.labels
-          value:
+          content:
             app.kubernetes.io/instance: cf-runtime
             app.kubernetes.io/managed-by: Helm
             app.kubernetes.io/name: cf-runtime
-            app.kubernetes.io/version: 1.0.0
-            helm.sh/chart: cf-runtime-1.0.0
             codefresh.io/application: runner
       - equal:
           path: metadata.name

diff --git a/charts/cf-runtime/tests/runtime/runtime_onprem_test.yaml b/charts/cf-runtime/tests/runtime/runtime_onprem_test.yaml
@@ -1,16 +1,13 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/quintush/helm-unittest/master/schema/helm-testsuite.json
 suite: runtime onprem test
 templates:
-  - templates/hooks/post-install/job-update-runtime.yaml
-  - templates/hooks/post-install/cm-update-runtime.yaml
-  - templates/runtime/secret.yaml
+  - templates/**.yaml
 release:
   name: cf-runtime
   namespace: codefresh
   revision: 1
   upgrade: true
 chart:
-  version: 1.0.0
   appVersion: 1.0.0
 tests:
   - it: Test default runtime spec metadata

diff --git a/charts/cf-runtime/tests/runtime/runtime_test.yaml b/charts/cf-runtime/tests/runtime/runtime_test.yaml
@@ -3,16 +3,13 @@ suite: runtime test
 values:
   - ../values.yaml
 templates:
-  - templates/hooks/post-install/job-update-runtime.yaml
-  - templates/hooks/post-install/cm-update-runtime.yaml
-  - templates/runtime/secret.yaml
+  - templates/**.yaml
 release:
   name: cf-runtime
   namespace: codefresh
   revision: 1
   upgrade: true
 chart:
-  version: 1.0.0
   appVersion: 1.0.0
 tests:
   - it: Test default runtime spec metadata

diff --git a/charts/cf-runtime/tests/volume-provisioner/cronjob_test.yaml b/charts/cf-runtime/tests/volume-provisioner/cronjob_test.yaml
@@ -3,15 +3,13 @@ suite: dind-volume-cleanup test
 values:
   - ../values.yaml
 templates:
-  - templates/volume-provisioner/cronjob.yaml
-  - templates/volume-provisioner/storageclass.yaml
+  - templates/**.yaml
 release:
   name: cf-runtime
   namespace: codefresh
   revision: 1
   upgrade: true
 chart:
-  version: 1.0.0
   appVersion: 1.0.0
 tests:
   - it: Test dind-volume-cleanup default metadata
@@ -25,14 +23,12 @@ tests:
           of: CronJob
       - isNull:
           path: metadata.annotations
-      - equal:
+      - isSubset:
           path: metadata.labels
-          value:
+          content:
             app.kubernetes.io/instance: cf-runtime
             app.kubernetes.io/managed-by: Helm
             app.kubernetes.io/name: cf-runtime
-            app.kubernetes.io/version: 1.0.0
-            helm.sh/chart: cf-runtime-1.0.0
             codefresh.io/application: pv-cleanup
       - equal:
           path: metadata.name