Use the nydus-snapshotter by default

config/samples/ccruntime/base/ccruntime.yaml

@@ -55,6 +55,12 @@ spec:
           name: confidential-containers-artifacts
         - mountPath: /etc/systemd/system/
           name: etc-systemd-system
+        - mountPath: /etc/containerd/
+          name: containerd-conf
+        - mountPath: /usr/local/bin/
+          name: local-bin
+        - mountPath: /var/lib/containerd-nydus/
+          name: containerd-nydus
       volumes:
         - hostPath:
             path: /opt/confidential-containers/
@@ -64,29 +70,52 @@ spec:
             path: /etc/systemd/system/
             type: ""
           name: etc-systemd-system
+        - hostPath:
+            path: /etc/containerd/
+            type: ""
+          name: containerd-conf
+        - hostPath:
+            path: /usr/local/bin/
+            type: ""
+          name: local-bin
+        - hostPath:
+            path: /var/lib/containerd-nydus/
+            type: ""
+          name: containerd-nydus
       environmentVariables:
         # If set to true, this will install the CoCo fork of the containerd,
         # the one allowing images to be pulled inside the guest and has patches
         # for handling GPU / VFIO, on the node
         # default: true
         - name: "INSTALL_COCO_CONTAINERD"
-          value: "true"
+          value: "false"
         # If set to true, this will install the v1.7.0 release of containerd on the node.
         # default: false
         - name: "INSTALL_OFFICIAL_CONTAINERD"
-          value: "false"
+          value: "true"
         # If set to true, this will install the CoCo fork of the containerd,
         # the one that has patches for handling GPU / VFIO, on the node
         # default: false
         - name: "INSTALL_VFIO_GPU_CONTAINERD"
           value: "false"
+        # If set to true, this will install nydus-snapshotter and nydus-image
+        # on the node
+        # default: false
+        - name: "INSTALL_NYDUS_SNAPSHOTTER"
+          value: "true"
     preInstall:
       image: quay.io/confidential-containers/reqs-payload
       volumeMounts:
         - mountPath: /opt/confidential-containers/
           name: confidential-containers-artifacts
         - mountPath: /etc/systemd/system/
           name: etc-systemd-system
+        - mountPath: /etc/containerd/
+          name: containerd-conf
+        - mountPath: /usr/local/bin/
+          name: local-bin
+        - mountPath: /var/lib/containerd-nydus/
+          name: containerd-nydus
       volumes:
         - hostPath:
             path: /opt/confidential-containers/
@@ -96,21 +125,38 @@ spec:
             path: /etc/systemd/system/
             type: ""
           name: etc-systemd-system
+        - hostPath:
+            path: /etc/containerd/
+            type: ""
+          name: containerd-conf
+        - hostPath:
+            path: /usr/local/bin/
+            type: ""
+          name: local-bin
+        - hostPath:
+            path: /var/lib/containerd-nydus/
+            type: ""
+          name: containerd-nydus
       environmentVariables:
         # If set to true, this will install the CoCo fork of the containerd,
         # the one allowing images to be pulled inside the guest and has patches
         # for handling GPU / VFIO, on the node
         - name: "INSTALL_COCO_CONTAINERD"
-          value: "true"
+          value: "false"
         # If set to true, this will install the v1.7.0 release of containerd on the node.
         # default: false
         - name: "INSTALL_OFFICIAL_CONTAINERD"
-          value: "false"
+          value: "true"
         # If set to true, this will install the CoCo fork of the containerd,
         # the one that has patches for handling GPU / VFIO, on the node
         # default: false
         - name: "INSTALL_VFIO_GPU_CONTAINERD"
           value: "false"
+        # If set to true, this will install nydus-snapshotter and nydus-image
+        # on the node
+        # default: false
+        - name: "INSTALL_NYDUS_SNAPSHOTTER"
+          value: "true"
     environmentVariables:
       - name: NODE_NAME
         valueFrom:
@@ -119,3 +165,5 @@ spec:
             fieldPath: spec.nodeName
       - name: "CONFIGURE_CC"
         value: "yes"
+      - name: "DEBUG"
+        value: "false"

config/samples/ccruntime/default/kustomization.yaml

@@ -19,17 +19,17 @@ patches:
       path: /spec/config/runtimeClasses
       value:
       - name: "kata-clh"
-        snapshotter: "overlayfs"
+        snapshotter: "nydus"
       - name: "kata-clh-tdx"
-        snapshotter: "overlayfs"
+        snapshotter: "nydus"
       - name: "kata-qemu"
-        snapshotter: "overlayfs"
+        snapshotter: "nydus"
       - name: "kata-qemu-tdx"
-        snapshotter: "overlayfs"
+        snapshotter: "nydus"
       - name: "kata-qemu-sev"
-        snapshotter: "overlayfs"
+        snapshotter: "nydus"
       - name: "kata-qemu-snp"
-        snapshotter: "overlayfs"
+        snapshotter: "nydus"
     - op: add
       path: /spec/config/defaultRuntimeClassName
       value: "kata-qemu"

config/samples/ccruntime/peer-pods/kustomization.yaml

@@ -19,7 +19,7 @@ patches:
       path: /spec/config/runtimeClasses
       value:
       - name: "kata-remote"
-        snapshotter: "overlayfs"
+        snapshotter: "nydus"
     - op: add
       path: /spec/config/debug
       value: false

config/samples/ccruntime/s390x/kustomization.yaml

@@ -17,9 +17,9 @@ patches:
       path: /spec/config/runtimeClasses
       value:
       - name: "kata-qemu"
-        snapshotter: "overlayfs"
+        snapshotter: "nydus"
       - name: "kata-qemu-se"
-        snapshotter: "overlayfs"
+        snapshotter: "nydus"
     - op: add
       path: /spec/config/defaultRuntimeClassName
       value: "kata-qemu"

config/samples/ccruntime/ssh-demo/kustomization.yaml

@@ -18,11 +18,11 @@ patches:
       path: /spec/config/runtimeClasses
       value:
       - name: "kata"
-        snapshotter: "overlayfs"
+        snapshotter: "nydus"
       - name: "kata-clh"
-        snapshotter: "overlayfs"
+        snapshotter: "nydus"
       - name: "kata-qemu"
-        snapshotter: "overlayfs"
+        snapshotter: "nydus"
   target:
     kind: CcRuntime
 

config/samples/enclave-cc/base/ccruntime-enclave-cc.yaml

@@ -53,15 +53,33 @@ spec:
           name: confidential-containers-artifacts
         - mountPath: /etc/systemd/system/
           name: etc-systemd-system
+        - mountPath: /etc/containerd/
+          name: containerd-conf
+        - mountPath: /usr/local/bin/
+          name: local-bin
+        - mountPath: /var/lib/containerd-nydus/
+          name: containerd-nydus
       volumes:
         - hostPath:
             path: /opt/confidential-containers/
             type: DirectoryOrCreate
           name: confidential-containers-artifacts
+        - hostPath:
+            path: /etc/containerd/
+            type: ""
+          name: containerd-conf
         - hostPath:
             path: /etc/systemd/system/
             type: ""
           name: etc-systemd-system
+        - hostPath:
+            path: /usr/local/bin/
+            type: ""
+          name: local-bin
+        - hostPath:
+            path: /var/lib/containerd-nydus/
+            type: ""
+          name: containerd-nydus
       environmentVariables:
         # If set to true, this will install the CoCo fork of the containerd on the node.
         # default: true
@@ -76,13 +94,24 @@ spec:
         # default: false
         - name: "INSTALL_VFIO_GPU_CONTAINERD"
           value: "false"
+        # If set to true, this will install nydus-snapshotter and nydus-image
+        # on the node
+        # default: false
+        - name: "INSTALL_NYDUS_SNAPSHOTTER"
+          value: "false"
     preInstall:
       image: quay.io/confidential-containers/reqs-payload
       volumeMounts:
         - mountPath: /opt/confidential-containers/
           name: confidential-containers-artifacts
         - mountPath: /etc/systemd/system/
           name: etc-systemd-system
+        - mountPath: /etc/containerd/
+          name: containerd-conf
+        - mountPath: /usr/local/bin/
+          name: local-bin
+        - mountPath: /var/lib/containerd-nydus/
+          name: containerd-nydus
       volumes:
         - hostPath:
             path: /opt/confidential-containers/
@@ -92,6 +121,18 @@ spec:
             path: /etc/systemd/system/
             type: ""
           name: etc-systemd-system
+        - hostPath:
+            path: /etc/containerd/
+            type: ""
+          name: containerd-conf
+        - hostPath:
+            path: /usr/local/bin/
+            type: ""
+          name: local-bin
+        - hostPath:
+            path: /var/lib/containerd-nydus/
+            type: ""
+          name: containerd-nydus
       environmentVariables:
         # If set to true, this will install the CoCo fork of the containerd on the node.
         # default: true
@@ -106,6 +147,11 @@ spec:
         # default: false
         - name: "INSTALL_VFIO_GPU_CONTAINERD"
           value: "false"
+        # If set to true, this will install nydus-snapshotter and nydus-image
+        # on the node
+        # default: false
+        - name: "INSTALL_NYDUS_SNAPSHOTTER"
+          value: "false"
     environmentVariables:
       - name: NODE_NAME
         valueFrom:

controllers/ccruntime_controller.go

@@ -226,6 +226,20 @@ func (r *CcRuntimeReconciler) processCcRuntimeDeleteRequest() (ctrl.Result, erro
 			if r.ccRuntime.Spec.Config.PostUninstall.Image == "" {
 				controllerutil.RemoveFinalizer(r.ccRuntime, RuntimeConfigFinalizer)
 			} else if r.ccRuntime.Spec.Config.PostUninstall.Image != "" {
+				// FXIME: This should be treated in a better way, as just having the sleep
+				//        here won't do us any good in the future.
+				//
+				//        What's basically happening, and forcing us to do this, is the
+				//        fact that the Uninstall and postUninstall daemonsets are being
+				//        started at exactly the same time, leading to a race condition
+				//        when changing the containerd configuration.
+				//
+				//        When looking at the kata-containers payload code, we see that the
+				//        the label is only set after containerd is successfully reconfigured,
+				//        and looking at this function we see we shouldn't reach this part
+				//        before the label is set.  However, that's not what we're facing ...
+				time.Sleep(time.Second * 60)
+
 				result, err = handlePostUninstall(r)
 				if !result.Requeue {
 					controllerutil.RemoveFinalizer(r.ccRuntime, RuntimeConfigFinalizer)
@@ -470,7 +484,7 @@ func (r *CcRuntimeReconciler) monitorCcRuntimeInstallation() (ctrl.Result, error
 			foundRc := &nodeapi.RuntimeClass{}
 			err := r.Client.Get(context.TODO(), types.NamespacedName{Name: runtimeClass.Name}, foundRc)
 			if errors.IsNotFound(err) {
-				r.Log.Info("The runtime payload failed to create the runtime class named %s", runtimeClass.Name)
+				r.Log.Info("The runtime payload failed to create the runtime class", "runtimeClassName", runtimeClass.Name)
 				return ctrl.Result{}, err
 			}
 			runtimeClassNames = append(runtimeClassNames, runtimeClass.Name)
@@ -622,7 +636,7 @@ func (r *CcRuntimeReconciler) processDaemonset(operation DaemonOperation) *appsv
 		//        payload script supports setting one snapshotter per runtime handler.
 		//        For now, for the v0.8.0 release, we're fine assuming that all the
 		//        set snapshotters are going to be the same.
-		if snapshotter != "" {
+		if snapshotter == "" && runtimeClass.Snapshotter != "" {
 			snapshotter = runtimeClass.Snapshotter
 		}
 	}

install/pre-install-payload/Dockerfile

@@ -32,9 +32,9 @@ ARG NODE_DESTINATION=${DESTINATION}/opt/confidential-containers
 RUN \
 	mkdir -p ${NODE_DESTINATION} && \
 	apk --no-cache add curl && \
-	curl -fOL --progress-bar ${OFFICIAL_CONTAINERD_REPO}/releases/download/v${OFFICIAL_CONTAINERD_VERSION}/containerd-${OFFICIAL_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz && \
-	tar xvzpf containerd-${OFFICIAL_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz -C ${NODE_DESTINATION} && \
-	rm containerd-${OFFICIAL_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz
+	curl -fOL --progress-bar ${OFFICIAL_CONTAINERD_REPO}/releases/download/v${OFFICIAL_CONTAINERD_VERSION}/containerd-static-${OFFICIAL_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz && \
+	tar xvzpf containerd-static-${OFFICIAL_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz -C ${NODE_DESTINATION} && \
+	rm containerd-static-${OFFICIAL_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz
 
 #### Confidential Containers forked containerd for VFIO / GPU stuff
 
@@ -50,10 +50,31 @@ ARG NODE_DESTINATION=${DESTINATION}/opt/confidential-containers
 RUN \
 	mkdir -p ${NODE_DESTINATION} && \
 	apk --no-cache add curl && \
-	curl -fOL --progress-bar ${VFIO_GPU_CONTAINERD_REPO}/releases/download/v${VFIO_GPU_CONTAINERD_VERSION}/containerd-${VFIO_GPU_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz && \
-	tar xvzpf containerd-${VFIO_GPU_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz -C ${NODE_DESTINATION} && \
-	rm containerd-${VFIO_GPU_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz
+	curl -fOL --progress-bar ${VFIO_GPU_CONTAINERD_REPO}/releases/download/v${VFIO_GPU_CONTAINERD_VERSION}/containerd-static-${VFIO_GPU_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz && \
+	tar xvzpf containerd-static-${VFIO_GPU_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz -C ${NODE_DESTINATION} && \
+	rm containerd-static-${VFIO_GPU_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz
 
+#### Nydus snapshotter & nydus image
+
+FROM golang:1.19-alpine AS nydus-binary-downloader
+
+ARG ARCH
+ARG NYDUS_SNAPSHOTTER_VERSION
+ARG NYDUS_SNAPSHOTTER_REPO
+
+ARG DESTINATION=/opt/confidential-containers-pre-install-artifacts
+ARG NODE_DESTINATION=${DESTINATION}/opt/confidential-containers
+
+ENV GOARCH=${ARCH}
+
+RUN mkdir -p ${NODE_DESTINATION}/bin && \
+    apk add --no-cache curl && \
+    if [ "${ARCH}" = "amd64" ]; then ARCH=x86_64; fi && \
+    curl -fOL --progress-bar ${NYDUS_SNAPSHOTTER_REPO}/releases/download/${NYDUS_SNAPSHOTTER_VERSION}/nydus-snapshotter-${NYDUS_SNAPSHOTTER_VERSION}-${ARCH}.tgz && \
+    tar xvzpf nydus-snapshotter-${NYDUS_SNAPSHOTTER_VERSION}-${ARCH}.tgz -C / && \
+    rm nydus-snapshotter-${NYDUS_SNAPSHOTTER_VERSION}-${ARCH}.tgz && \
+    mv /nydus-snapshotter/* ${NODE_DESTINATION}/bin/ && \
+    rm -rf /nydus-snapshotter
 
 #### kubectl
 
@@ -76,14 +97,25 @@ RUN apk --no-cache add bash
 ARG DESTINATION=/opt/confidential-containers-pre-install-artifacts
 ARG NODE_DESTINATION=${DESTINATION}/opt/confidential-containers
 ARG NODE_CONTAINERD_SYSTEMD_DESTINATION=${DESTINATION}/etc/systemd/system/containerd.service.d/
+ARG NODE_NYDUS_SNAPSHOTTER_SYSTEMD_DESTINATION=${DESTINATION}/etc/systemd/system/nydus-snapshotter.service
 
 ARG CONTAINERD_SYSTEMD_ARTIFACTS=./containerd/containerd-for-cc-override.conf
+ARG NYDUS_SNAPSHOTTER_SYSTEMD_ARTIFACTS=./remote-snapshotter/nydus-snapshotter/nydus-snapshotter.service
+ARG NYDUS_SNAPSHOTTER_ARTIFACTS=./remote-snapshotter/nydus-snapshotter/config-coco-guest-pulling.toml
 
 COPY --from=coco-containerd-binary-downloader ${NODE_DESTINATION}/bin/containerd ${NODE_DESTINATION}/bin/coco-containerd
 COPY --from=official-containerd-binary-downloader ${NODE_DESTINATION}/bin/containerd ${NODE_DESTINATION}/bin/official-containerd
 COPY --from=vfio-gpu-containerd-binary-downloader ${NODE_DESTINATION}/bin/containerd ${NODE_DESTINATION}/bin/vfio-gpu-containerd
+
+COPY --from=nydus-binary-downloader ${NODE_DESTINATION}/bin/* ${NODE_DESTINATION}/bin/
+
 COPY --from=kubectl-binary-downloader /usr/bin/kubectl /usr/bin/kubectl
 COPY ${CONTAINERD_SYSTEMD_ARTIFACTS} ${NODE_CONTAINERD_SYSTEMD_DESTINATION}
+COPY ${NYDUS_SNAPSHOTTER_SYSTEMD_ARTIFACTS} ${NODE_NYDUS_SNAPSHOTTER_SYSTEMD_DESTINATION}
+COPY ${NYDUS_SNAPSHOTTER_ARTIFACTS} ${NODE_DESTINATION}/share/nydus-snapshotter/config-coco-guest-pulling.toml
 
 ARG CONTAINER_ENGINE_ARTIFACTS=./scripts
 COPY ${CONTAINER_ENGINE_ARTIFACTS}/* ${DESTINATION}/scripts/
+
+# Also copy `ctr` to our final image, so we can use it to remove nydus snapshots
+COPY --from=official-containerd-binary-downloader ${NODE_DESTINATION}/bin/ctr /usr/bin/ctr

install/pre-install-payload/Makefile

@@ -1,11 +1,13 @@
 COCO_CONTAINERD_VERSION = 1.6.8.2
-OFFICIAL_CONTAINERD_VERSION = 1.7.0
+OFFICIAL_CONTAINERD_VERSION = 1.7.7
 VFIO_GPU_CONTAINERD_VERSION = 1.7.0.0
+NYDUS_SNAPSHOTTER_VERSION = v0.13.3-multiarch
 
 BASH = bash
 
 reqs-image:
 	coco_containerd_version=$(COCO_CONTAINERD_VERSION) \
 	official_containerd_version=$(OFFICIAL_CONTAINERD_VERSION) \
 	vfio_gpu_containerd_version=$(VFIO_GPU_CONTAINERD_VERSION) \
+	nydus_snapshotter_version=${NYDUS_SNAPSHOTTER_VERSION} \
 	$(BASH) -x payload.sh