From a86b15f34e80c2a8b36c724057cf5d5781ac35da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 9 Oct 2023 16:47:57 +0200 Subject: [PATCH 01/15] config: ccruntime: Add a "DEBUG" env var MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will be used, either by the admin or by our tests, to set the correct debug level we want for the payload we're shipping, including the pre-install one. The default is "false", as usually folks won't need fully verbose logs in production. Signed-off-by: Fabiano Fidêncio --- config/samples/ccruntime/base/ccruntime.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/samples/ccruntime/base/ccruntime.yaml b/config/samples/ccruntime/base/ccruntime.yaml index d209c5c7..7a9c183a 100644 --- a/config/samples/ccruntime/base/ccruntime.yaml +++ b/config/samples/ccruntime/base/ccruntime.yaml @@ -119,3 +119,5 @@ spec: fieldPath: spec.nodeName - name: "CONFIGURE_CC" value: "yes" + - name: "DEBUG" + value: "false" From c94de0559a7b8300ec233d803a6ebff51bea43a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 20 Oct 2023 11:25:45 +0200 Subject: [PATCH 02/15] pre-install: Fix setup_env_for_arch() switch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit That was a typo that was there since d1268cfce56, when the code was moved to this repo. Signed-off-by: ChengyuZhu6 Signed-off-by: Fabiano Fidêncio --- install/pre-install-payload/payload.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/pre-install-payload/payload.sh b/install/pre-install-payload/payload.sh index 29ee1a8f..d854b05c 100755 --- a/install/pre-install-payload/payload.sh +++ b/install/pre-install-payload/payload.sh @@ -32,7 +32,7 @@ function setup_env_for_arch() { kernel_arch="s390x" golang_arch="s390x" ;; - (*) echo "$1 is not supported" > /dev/stderr && exit 1 + *) echo "$1 is not supported" >/dev/stderr && exit 1 ;; esac } From b385546c8c5dc3adb1435e96150c5735edf1d12c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 20 Oct 2023 12:32:40 +0200 Subject: [PATCH 03/15] pre-install: Don't fail the build if there's no manifest to be purged MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: ChengyuZhu6 Signed-off-by: Fabiano Fidêncio --- install/pre-install-payload/payload.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/pre-install-payload/payload.sh b/install/pre-install-payload/payload.sh index d854b05c..eb005022 100755 --- a/install/pre-install-payload/payload.sh +++ b/install/pre-install-payload/payload.sh @@ -46,7 +46,7 @@ function purge_previous_manifests() { # * ':' by '-' sanitised_manifest="$(echo ${manifest} | sed 's|/|_|g' | sed 's|:|-|g')" - rm -rf ${HOME}/.docker/manifests/${sanitised_manifest} + rm -rf ${HOME}/.docker/manifests/${sanitised_manifest} || true } function build_payload() { From 36a34f507a7b21a1f17264d13231042401964c0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 20 Oct 2023 11:30:32 +0200 Subject: [PATCH 04/15] pre-install: Fix official containerd installation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 17a924862fa07 introduced this, and was calling the wrong method. Signed-off-by: ChengyuZhu6 Signed-off-by: Fabiano Fidêncio --- install/pre-install-payload/scripts/reqs-deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/pre-install-payload/scripts/reqs-deploy.sh b/install/pre-install-payload/scripts/reqs-deploy.sh index 75d591f7..0d2f882a 100755 --- a/install/pre-install-payload/scripts/reqs-deploy.sh +++ b/install/pre-install-payload/scripts/reqs-deploy.sh @@ -59,7 +59,7 @@ function install_artifacts() { fi if [ "${INSTALL_OFFICIAL_CONTAINERD}" = "true" ]; then - install_coco_containerd_artefacts + install_official_containerd_artefacts fi if [ "${INSTALL_VFIO_GPU_CONTAINERD}" = "true" ]; then From c909be26f345715aefdf3279e1c88c6df5b8a2c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 24 Oct 2023 11:57:32 +0200 Subject: [PATCH 05/15] pre-install: Ensure we use static releases of containerd (when possible) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will reduce the risk of something breaking on the container / host due to the lack of libraries that needed to be loaded with a dynamically linked binary. Signed-off-by: Fabiano Fidêncio --- install/pre-install-payload/Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/install/pre-install-payload/Dockerfile b/install/pre-install-payload/Dockerfile index eb60fa00..0f201bac 100644 --- a/install/pre-install-payload/Dockerfile +++ b/install/pre-install-payload/Dockerfile @@ -32,9 +32,9 @@ ARG NODE_DESTINATION=${DESTINATION}/opt/confidential-containers RUN \ mkdir -p ${NODE_DESTINATION} && \ apk --no-cache add curl && \ - curl -fOL --progress-bar ${OFFICIAL_CONTAINERD_REPO}/releases/download/v${OFFICIAL_CONTAINERD_VERSION}/containerd-${OFFICIAL_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz && \ - tar xvzpf containerd-${OFFICIAL_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz -C ${NODE_DESTINATION} && \ - rm containerd-${OFFICIAL_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz + curl -fOL --progress-bar ${OFFICIAL_CONTAINERD_REPO}/releases/download/v${OFFICIAL_CONTAINERD_VERSION}/containerd-static-${OFFICIAL_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz && \ + tar xvzpf containerd-static-${OFFICIAL_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz -C ${NODE_DESTINATION} && \ + rm containerd-static-${OFFICIAL_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz #### Confidential Containers forked containerd for VFIO / GPU stuff @@ -50,9 +50,9 @@ ARG NODE_DESTINATION=${DESTINATION}/opt/confidential-containers RUN \ mkdir -p ${NODE_DESTINATION} && \ apk --no-cache add curl && \ - curl -fOL --progress-bar ${VFIO_GPU_CONTAINERD_REPO}/releases/download/v${VFIO_GPU_CONTAINERD_VERSION}/containerd-${VFIO_GPU_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz && \ - tar xvzpf containerd-${VFIO_GPU_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz -C ${NODE_DESTINATION} && \ - rm containerd-${VFIO_GPU_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz + curl -fOL --progress-bar ${VFIO_GPU_CONTAINERD_REPO}/releases/download/v${VFIO_GPU_CONTAINERD_VERSION}/containerd-static-${VFIO_GPU_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz && \ + tar xvzpf containerd-static-${VFIO_GPU_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz -C ${NODE_DESTINATION} && \ + rm containerd-static-${VFIO_GPU_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz #### kubectl From b85d78cda102e8c9b2b55ce8c496c57f9d8c600b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 24 Oct 2023 10:25:59 +0200 Subject: [PATCH 06/15] pre-install: Ensure ctr is part of the image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As we'll need to use it to properly clean up the nydus snapshots when uninstalling nydus. Signed-off-by: Fabiano Fidêncio --- install/pre-install-payload/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install/pre-install-payload/Dockerfile b/install/pre-install-payload/Dockerfile index 0f201bac..4b86e532 100644 --- a/install/pre-install-payload/Dockerfile +++ b/install/pre-install-payload/Dockerfile @@ -87,3 +87,6 @@ COPY ${CONTAINERD_SYSTEMD_ARTIFACTS} ${NODE_CONTAINERD_SYSTEMD_DESTINATION} ARG CONTAINER_ENGINE_ARTIFACTS=./scripts COPY ${CONTAINER_ENGINE_ARTIFACTS}/* ${DESTINATION}/scripts/ + +# Also copy `ctr` to our final image, so we can use it to remove nydus snapshots +COPY --from=official-containerd-binary-downloader ${NODE_DESTINATION}/bin/ctr /usr/bin/ctr From b6c8111bc7489e78f8acbfbd6c993f45be21460a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 20 Oct 2023 12:28:43 +0200 Subject: [PATCH 07/15] pre-install: Build snapshotter as part of the payload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nydus snapshotter will be used for replacing the forked containerd, allowing us to pull the image inside the guest, and this commits is adding the ability to build it. Getting it to be used will come later on on this series. Signed-off-by: ChengyuZhu6 Signed-off-by: Fabiano Fidêncio --- install/pre-install-payload/Dockerfile | 31 +++++++++++++++++++ install/pre-install-payload/Makefile | 2 ++ install/pre-install-payload/payload.sh | 4 +++ .../config-coco-guest-pulling.toml | 15 +++++++++ .../nydus-snapshotter.service | 10 ++++++ 5 files changed, 62 insertions(+) create mode 100644 install/pre-install-payload/remote-snapshotter/nydus-snapshotter/config-coco-guest-pulling.toml create mode 100644 install/pre-install-payload/remote-snapshotter/nydus-snapshotter/nydus-snapshotter.service diff --git a/install/pre-install-payload/Dockerfile b/install/pre-install-payload/Dockerfile index 4b86e532..e69284d7 100644 --- a/install/pre-install-payload/Dockerfile +++ b/install/pre-install-payload/Dockerfile @@ -54,6 +54,29 @@ RUN \ tar xvzpf containerd-static-${VFIO_GPU_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz -C ${NODE_DESTINATION} && \ rm containerd-static-${VFIO_GPU_CONTAINERD_VERSION}-linux-${ARCH}.tar.gz +#### Nydus snapshotter & nydus image + +FROM golang:1.19-alpine AS nydus-binary-downloader + +ARG ARCH +ARG NYDUS_SNAPSHOTTER_VERSION +ARG NYDUS_SNAPSHOTTER_REPO + +ARG DESTINATION=/opt/confidential-containers-pre-install-artifacts +ARG NODE_DESTINATION=${DESTINATION}/opt/confidential-containers + +ENV GOARCH=${ARCH} + +RUN mkdir -p ${NODE_DESTINATION}/bin && \ + apk add --no-cache ca-certificates build-base git curl && \ + git clone ${NYDUS_SNAPSHOTTER_REPO} -b ${NYDUS_SNAPSHOTTER_VERSION} /nydus-snapshotter && \ + sed -i -e '/{STATIC_OPTIMIZER_SERVER_BIN}/d' /nydus-snapshotter/Makefile && \ + make -C /nydus-snapshotter static-release && \ + chmod +x /nydus-snapshotter/bin/containerd-nydus-grpc && \ + chmod +x /nydus-snapshotter/bin/nydus-overlayfs && \ + mv /nydus-snapshotter/bin/containerd-nydus-grpc ${NODE_DESTINATION}/bin && \ + mv /nydus-snapshotter/bin/nydus-overlayfs ${NODE_DESTINATION}/bin && \ + rm -rf /nydus-snapshotter #### kubectl @@ -76,14 +99,22 @@ RUN apk --no-cache add bash ARG DESTINATION=/opt/confidential-containers-pre-install-artifacts ARG NODE_DESTINATION=${DESTINATION}/opt/confidential-containers ARG NODE_CONTAINERD_SYSTEMD_DESTINATION=${DESTINATION}/etc/systemd/system/containerd.service.d/ +ARG NODE_NYDUS_SNAPSHOTTER_SYSTEMD_DESTINATION=${DESTINATION}/etc/systemd/system/nydus-snapshotter.service ARG CONTAINERD_SYSTEMD_ARTIFACTS=./containerd/containerd-for-cc-override.conf +ARG NYDUS_SNAPSHOTTER_SYSTEMD_ARTIFACTS=./remote-snapshotter/nydus-snapshotter/nydus-snapshotter.service +ARG NYDUS_SNAPSHOTTER_ARTIFACTS=./remote-snapshotter/nydus-snapshotter/config-coco-guest-pulling.toml COPY --from=coco-containerd-binary-downloader ${NODE_DESTINATION}/bin/containerd ${NODE_DESTINATION}/bin/coco-containerd COPY --from=official-containerd-binary-downloader ${NODE_DESTINATION}/bin/containerd ${NODE_DESTINATION}/bin/official-containerd COPY --from=vfio-gpu-containerd-binary-downloader ${NODE_DESTINATION}/bin/containerd ${NODE_DESTINATION}/bin/vfio-gpu-containerd + +COPY --from=nydus-binary-downloader ${NODE_DESTINATION}/bin/* ${NODE_DESTINATION}/bin/ + COPY --from=kubectl-binary-downloader /usr/bin/kubectl /usr/bin/kubectl COPY ${CONTAINERD_SYSTEMD_ARTIFACTS} ${NODE_CONTAINERD_SYSTEMD_DESTINATION} +COPY ${NYDUS_SNAPSHOTTER_SYSTEMD_ARTIFACTS} ${NODE_NYDUS_SNAPSHOTTER_SYSTEMD_DESTINATION} +COPY ${NYDUS_SNAPSHOTTER_ARTIFACTS} ${NODE_DESTINATION}/share/nydus-snapshotter/config-coco-guest-pulling.toml ARG CONTAINER_ENGINE_ARTIFACTS=./scripts COPY ${CONTAINER_ENGINE_ARTIFACTS}/* ${DESTINATION}/scripts/ diff --git a/install/pre-install-payload/Makefile b/install/pre-install-payload/Makefile index 9a9d980c..c7dc0653 100644 --- a/install/pre-install-payload/Makefile +++ b/install/pre-install-payload/Makefile @@ -1,6 +1,7 @@ COCO_CONTAINERD_VERSION = 1.6.8.2 OFFICIAL_CONTAINERD_VERSION = 1.7.0 VFIO_GPU_CONTAINERD_VERSION = 1.7.0.0 +NYDUS_SNAPSHOTTER_VERSION = v0.13.3 BASH = bash @@ -8,4 +9,5 @@ reqs-image: coco_containerd_version=$(COCO_CONTAINERD_VERSION) \ official_containerd_version=$(OFFICIAL_CONTAINERD_VERSION) \ vfio_gpu_containerd_version=$(VFIO_GPU_CONTAINERD_VERSION) \ + nydus_snapshotter_version=${NYDUS_SNAPSHOTTER_VERSION} \ $(BASH) -x payload.sh diff --git a/install/pre-install-payload/payload.sh b/install/pre-install-payload/payload.sh index eb005022..a1e954a1 100755 --- a/install/pre-install-payload/payload.sh +++ b/install/pre-install-payload/payload.sh @@ -12,6 +12,8 @@ official_containerd_repo=${official_containerd_repo:-"https://github.com/contain official_containerd_version=${official_containerd_version:-"1.7.0"} vfio_gpu_containerd_repo=${vfio_gpu_containerd_repo:-"https://github.com/confidential-containers/containerd"} vfio_gpu_containerd_version=${vfio_gpu_containerd_version:-"1.7.0.0"} +nydus_snapshotter_repo=${nydus_snapshotter_repo:-"https://github.com/containerd/nydus-snapshotter"} +nydus_snapshotter_version=${nydus_snapshotter_version:-"v0.13.3"} containerd_dir="$(mktemp -d -t containerd-XXXXXXXXXX)/containerd" extra_docker_manifest_flags="${extra_docker_manifest_flags:-}" @@ -66,6 +68,8 @@ function build_payload() { --build-arg OFFICIAL_CONTAINERD_REPO="${official_containerd_repo}" \ --build-arg VFIO_GPU_CONTAINERD_VERSION="${vfio_gpu_containerd_version}" \ --build-arg VFIO_GPU_CONTAINERD_REPO="${vfio_gpu_containerd_repo}" \ + --build-arg NYDUS_SNAPSHOTTER_VERSION="${nydus_snapshotter_version}" \ + --build-arg NYDUS_SNAPSHOTTER_REPO="${nydus_snapshotter_repo}" \ -t "${registry}:${kernel_arch}-${tag}" \ --platform="${arch}" \ --load \ diff --git a/install/pre-install-payload/remote-snapshotter/nydus-snapshotter/config-coco-guest-pulling.toml b/install/pre-install-payload/remote-snapshotter/nydus-snapshotter/config-coco-guest-pulling.toml new file mode 100644 index 00000000..67a9be2d --- /dev/null +++ b/install/pre-install-payload/remote-snapshotter/nydus-snapshotter/config-coco-guest-pulling.toml @@ -0,0 +1,15 @@ +version = 1 + +# Snapshotter's own home directory where it stores and creates necessary resources +root = "/var/lib/containerd-nydus" + +# The snapshotter's GRPC server socket, containerd will connect to plugin on this socket +address = "/run/containerd-nydus/containerd-nydus-grpc.sock" + +[daemon] +# Enable proxy mode +fs_driver = "proxy" + +[snapshot] +# Insert Kata volume information to `Mount.Options` +enable_kata_volume = true \ No newline at end of file diff --git a/install/pre-install-payload/remote-snapshotter/nydus-snapshotter/nydus-snapshotter.service b/install/pre-install-payload/remote-snapshotter/nydus-snapshotter/nydus-snapshotter.service new file mode 100644 index 00000000..2226d447 --- /dev/null +++ b/install/pre-install-payload/remote-snapshotter/nydus-snapshotter/nydus-snapshotter.service @@ -0,0 +1,10 @@ +[Unit] +Description=Nydus snapshotter +After=network.target local-fs.target +Before=containerd.service + +[Service] +ExecStart=/opt/confidential-containers/bin/containerd-nydus-grpc --config /opt/confidential-containers/share/nydus-snapshotter/config-coco-guest-pulling.toml --log-to-stdout + +[Install] +RequiredBy=containerd.service From 1acf8b0303e6570a6b41ecddd3248be0bb32b0fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Tue, 24 Oct 2023 13:33:50 +0200 Subject: [PATCH 08/15] pre-install: Temporarily use fidencio's nydus-snapshotter release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a release of nydus-snapshotter that has absolutely **ZERO** code changed, but that adds a tarball for s390x. This has been done in order to work around the issue we have seen building the binaries for s390x as part of our CI, at least till nydus-snapshotter releases a s390x tarball, which is being tracked as part of https://github.com/containerd/nydus-snapshotter/issues/548 ``` The error we're facing is: 22.67 -> unzip /go/pkg/mod/cache/download/github.com/freddierice/go-losetup/@v/v0.0.0-20220711213114-2a14873012db.zip: zip: checksum error 22.69 go: downloading github.com/rs/xid v1.4.0 22.69 go: downloading github.com/containerd/stargz-snapshotter/estargz v0.14.3 22.69 go: downloading golang.org/x/net v0.10.0 22.71 go: downloading github.com/docker/distribution v2.8.2+incompatible 22.77 -> unzip /go/pkg/mod/cache/download/github.com/rs/xid/@v/v1.4.0.zip: zip: checksum error 22.83 go: downloading github.com/moby/locker v1.0.1 24.73 pkg/filesystem/stargz_adaptor.go:19:2: github.com/KarpelesLab/reflink@v1.0.1: Get "https://proxy.golang.org/github.com/%21karpeles%21lab/reflink/@v/v1.0.1.zip": local error: tls: bad record MAC ``` This is not ideal, but this is "good enough" to unblock this release. Signed-off-by: Fabiano Fidêncio --- install/pre-install-payload/Dockerfile | 14 ++++++-------- install/pre-install-payload/Makefile | 2 +- install/pre-install-payload/payload.sh | 4 ++-- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/install/pre-install-payload/Dockerfile b/install/pre-install-payload/Dockerfile index e69284d7..aa8d2ec5 100644 --- a/install/pre-install-payload/Dockerfile +++ b/install/pre-install-payload/Dockerfile @@ -68,14 +68,12 @@ ARG NODE_DESTINATION=${DESTINATION}/opt/confidential-containers ENV GOARCH=${ARCH} RUN mkdir -p ${NODE_DESTINATION}/bin && \ - apk add --no-cache ca-certificates build-base git curl && \ - git clone ${NYDUS_SNAPSHOTTER_REPO} -b ${NYDUS_SNAPSHOTTER_VERSION} /nydus-snapshotter && \ - sed -i -e '/{STATIC_OPTIMIZER_SERVER_BIN}/d' /nydus-snapshotter/Makefile && \ - make -C /nydus-snapshotter static-release && \ - chmod +x /nydus-snapshotter/bin/containerd-nydus-grpc && \ - chmod +x /nydus-snapshotter/bin/nydus-overlayfs && \ - mv /nydus-snapshotter/bin/containerd-nydus-grpc ${NODE_DESTINATION}/bin && \ - mv /nydus-snapshotter/bin/nydus-overlayfs ${NODE_DESTINATION}/bin && \ + apk add --no-cache curl && \ + if [ "${ARCH}" = "amd64" ]; then ARCH=x86_64; fi && \ + curl -fOL --progress-bar ${NYDUS_SNAPSHOTTER_REPO}/releases/download/${NYDUS_SNAPSHOTTER_VERSION}/nydus-snapshotter-${NYDUS_SNAPSHOTTER_VERSION}-${ARCH}.tgz && \ + tar xvzpf nydus-snapshotter-${NYDUS_SNAPSHOTTER_VERSION}-${ARCH}.tgz -C / && \ + rm nydus-snapshotter-${NYDUS_SNAPSHOTTER_VERSION}-${ARCH}.tgz && \ + mv /nydus-snapshotter/* ${NODE_DESTINATION}/bin/ && \ rm -rf /nydus-snapshotter #### kubectl diff --git a/install/pre-install-payload/Makefile b/install/pre-install-payload/Makefile index c7dc0653..a0aaac76 100644 --- a/install/pre-install-payload/Makefile +++ b/install/pre-install-payload/Makefile @@ -1,7 +1,7 @@ COCO_CONTAINERD_VERSION = 1.6.8.2 OFFICIAL_CONTAINERD_VERSION = 1.7.0 VFIO_GPU_CONTAINERD_VERSION = 1.7.0.0 -NYDUS_SNAPSHOTTER_VERSION = v0.13.3 +NYDUS_SNAPSHOTTER_VERSION = v0.13.3-multiarch BASH = bash diff --git a/install/pre-install-payload/payload.sh b/install/pre-install-payload/payload.sh index a1e954a1..8820c43b 100755 --- a/install/pre-install-payload/payload.sh +++ b/install/pre-install-payload/payload.sh @@ -12,8 +12,8 @@ official_containerd_repo=${official_containerd_repo:-"https://github.com/contain official_containerd_version=${official_containerd_version:-"1.7.0"} vfio_gpu_containerd_repo=${vfio_gpu_containerd_repo:-"https://github.com/confidential-containers/containerd"} vfio_gpu_containerd_version=${vfio_gpu_containerd_version:-"1.7.0.0"} -nydus_snapshotter_repo=${nydus_snapshotter_repo:-"https://github.com/containerd/nydus-snapshotter"} -nydus_snapshotter_version=${nydus_snapshotter_version:-"v0.13.3"} +nydus_snapshotter_repo=${nydus_snapshotter_repo:-"https://github.com/fidencio/nydus-snapshotter"} +nydus_snapshotter_version=${nydus_snapshotter_version:-"v0.13.3-multiarch"} containerd_dir="$(mktemp -d -t containerd-XXXXXXXXXX)/containerd" extra_docker_manifest_flags="${extra_docker_manifest_flags:-}" From 124b8fbc5e0aea5e7728b1e1adbf5f87a5f81ab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 20 Oct 2023 11:35:29 +0200 Subject: [PATCH 09/15] pre-install: config: Install nydus-snapshotter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nydus-snapshotter is going to be used in order to replace our dependency on the forked containerd. For now, we're not actually removing the forked containerd dependency, but rather giving the admin the choice to use nydus-snapshotter. Using the forked containerd will be deprecated as part of our **next** release. Signed-off-by: ChengyuZhu6 Signed-off-by: Fabiano Fidêncio --- config/samples/ccruntime/base/ccruntime.yaml | 46 +++++++++ .../enclave-cc/base/ccruntime-enclave-cc.yaml | 46 +++++++++ .../scripts/reqs-deploy.sh | 93 ++++++++++++++++++- 3 files changed, 183 insertions(+), 2 deletions(-) diff --git a/config/samples/ccruntime/base/ccruntime.yaml b/config/samples/ccruntime/base/ccruntime.yaml index 7a9c183a..dc0abcaa 100644 --- a/config/samples/ccruntime/base/ccruntime.yaml +++ b/config/samples/ccruntime/base/ccruntime.yaml @@ -55,6 +55,12 @@ spec: name: confidential-containers-artifacts - mountPath: /etc/systemd/system/ name: etc-systemd-system + - mountPath: /etc/containerd/ + name: containerd-conf + - mountPath: /usr/local/bin/ + name: local-bin + - mountPath: /var/lib/containerd-nydus/ + name: containerd-nydus volumes: - hostPath: path: /opt/confidential-containers/ @@ -64,6 +70,18 @@ spec: path: /etc/systemd/system/ type: "" name: etc-systemd-system + - hostPath: + path: /etc/containerd/ + type: "" + name: containerd-conf + - hostPath: + path: /usr/local/bin/ + type: "" + name: local-bin + - hostPath: + path: /var/lib/containerd-nydus/ + type: "" + name: containerd-nydus environmentVariables: # If set to true, this will install the CoCo fork of the containerd, # the one allowing images to be pulled inside the guest and has patches @@ -80,6 +98,11 @@ spec: # default: false - name: "INSTALL_VFIO_GPU_CONTAINERD" value: "false" + # If set to true, this will install nydus-snapshotter and nydus-image + # on the node + # default: false + - name: "INSTALL_NYDUS_SNAPSHOTTER" + value: "false" preInstall: image: quay.io/confidential-containers/reqs-payload volumeMounts: @@ -87,6 +110,12 @@ spec: name: confidential-containers-artifacts - mountPath: /etc/systemd/system/ name: etc-systemd-system + - mountPath: /etc/containerd/ + name: containerd-conf + - mountPath: /usr/local/bin/ + name: local-bin + - mountPath: /var/lib/containerd-nydus/ + name: containerd-nydus volumes: - hostPath: path: /opt/confidential-containers/ @@ -96,6 +125,18 @@ spec: path: /etc/systemd/system/ type: "" name: etc-systemd-system + - hostPath: + path: /etc/containerd/ + type: "" + name: containerd-conf + - hostPath: + path: /usr/local/bin/ + type: "" + name: local-bin + - hostPath: + path: /var/lib/containerd-nydus/ + type: "" + name: containerd-nydus environmentVariables: # If set to true, this will install the CoCo fork of the containerd, # the one allowing images to be pulled inside the guest and has patches @@ -111,6 +152,11 @@ spec: # default: false - name: "INSTALL_VFIO_GPU_CONTAINERD" value: "false" + # If set to true, this will install nydus-snapshotter and nydus-image + # on the node + # default: false + - name: "INSTALL_NYDUS_SNAPSHOTTER" + value: "false" environmentVariables: - name: NODE_NAME valueFrom: diff --git a/config/samples/enclave-cc/base/ccruntime-enclave-cc.yaml b/config/samples/enclave-cc/base/ccruntime-enclave-cc.yaml index 772df519..3b2a018d 100644 --- a/config/samples/enclave-cc/base/ccruntime-enclave-cc.yaml +++ b/config/samples/enclave-cc/base/ccruntime-enclave-cc.yaml @@ -53,15 +53,33 @@ spec: name: confidential-containers-artifacts - mountPath: /etc/systemd/system/ name: etc-systemd-system + - mountPath: /etc/containerd/ + name: containerd-conf + - mountPath: /usr/local/bin/ + name: local-bin + - mountPath: /var/lib/containerd-nydus/ + name: containerd-nydus volumes: - hostPath: path: /opt/confidential-containers/ type: DirectoryOrCreate name: confidential-containers-artifacts + - hostPath: + path: /etc/containerd/ + type: "" + name: containerd-conf - hostPath: path: /etc/systemd/system/ type: "" name: etc-systemd-system + - hostPath: + path: /usr/local/bin/ + type: "" + name: local-bin + - hostPath: + path: /var/lib/containerd-nydus/ + type: "" + name: containerd-nydus environmentVariables: # If set to true, this will install the CoCo fork of the containerd on the node. # default: true @@ -76,6 +94,11 @@ spec: # default: false - name: "INSTALL_VFIO_GPU_CONTAINERD" value: "false" + # If set to true, this will install nydus-snapshotter and nydus-image + # on the node + # default: false + - name: "INSTALL_NYDUS_SNAPSHOTTER" + value: "false" preInstall: image: quay.io/confidential-containers/reqs-payload volumeMounts: @@ -83,6 +106,12 @@ spec: name: confidential-containers-artifacts - mountPath: /etc/systemd/system/ name: etc-systemd-system + - mountPath: /etc/containerd/ + name: containerd-conf + - mountPath: /usr/local/bin/ + name: local-bin + - mountPath: /var/lib/containerd-nydus/ + name: containerd-nydus volumes: - hostPath: path: /opt/confidential-containers/ @@ -92,6 +121,18 @@ spec: path: /etc/systemd/system/ type: "" name: etc-systemd-system + - hostPath: + path: /etc/containerd/ + type: "" + name: containerd-conf + - hostPath: + path: /usr/local/bin/ + type: "" + name: local-bin + - hostPath: + path: /var/lib/containerd-nydus/ + type: "" + name: containerd-nydus environmentVariables: # If set to true, this will install the CoCo fork of the containerd on the node. # default: true @@ -106,6 +147,11 @@ spec: # default: false - name: "INSTALL_VFIO_GPU_CONTAINERD" value: "false" + # If set to true, this will install nydus-snapshotter and nydus-image + # on the node + # default: false + - name: "INSTALL_NYDUS_SNAPSHOTTER" + value: "false" environmentVariables: - name: NODE_NAME valueFrom: diff --git a/install/pre-install-payload/scripts/reqs-deploy.sh b/install/pre-install-payload/scripts/reqs-deploy.sh index 0d2f882a..76aa3541 100755 --- a/install/pre-install-payload/scripts/reqs-deploy.sh +++ b/install/pre-install-payload/scripts/reqs-deploy.sh @@ -4,6 +4,9 @@ set -o errexit set -o pipefail set -o nounset +containerd_config="/etc/containerd/config.toml" +artifacts_dir="/opt/confidential-containers-pre-install-artifacts" + die() { msg="$*" echo "ERROR: $msg" >&2 @@ -34,11 +37,9 @@ function install_containerd_artefacts() { echo "Copying ${flavour} containerd-for-cc artifacts onto host" - local artifacts_dir="/opt/confidential-containers-pre-install-artifacts" install -D -m 755 ${artifacts_dir}/opt/confidential-containers/bin/${flavour}-containerd /opt/confidential-containers/bin/containerd install -D -m 644 ${artifacts_dir}/etc/systemd/system/containerd.service.d/containerd-for-cc-override.conf /etc/systemd/system/containerd.service.d/containerd-for-cc-override.conf - } function install_coco_containerd_artefacts() { @@ -53,6 +54,24 @@ function install_vfio_gpu_containerd_artefacts() { install_containerd_artefacts "vfio-gpu" } +function install_nydus_snapshotter_artefacts() { + echo "Copying nydus-snapshotter artifacts onto host" + + install -D -m 755 ${artifacts_dir}/opt/confidential-containers/bin/containerd-nydus-grpc /opt/confidential-containers/bin/containerd-nydus-grpc + install -D -m 755 ${artifacts_dir}/opt/confidential-containers/bin/nydus-overlayfs /opt/confidential-containers/bin/nydus-overlayfs + ln -sf /opt/confidential-containers/bin/nydus-overlayfs /usr/local/bin/nydus-overlayfs + + install -D -m 644 ${artifacts_dir}/opt/confidential-containers/share/nydus-snapshotter/config-coco-guest-pulling.toml /opt/confidential-containers/share/nydus-snapshotter/config-coco-guest-pulling.toml + install -D -m 644 ${artifacts_dir}/etc/systemd/system/nydus-snapshotter.service /etc/systemd/system/nydus-snapshotter.service + + host_systemctl daemon-reload + host_systemctl enable nydus-snapshotter.service + + configure_nydus_snapshotter_for_containerd + + restart_systemd_service +} + function install_artifacts() { if [ "${INSTALL_COCO_CONTAINERD}" = "true" ]; then install_coco_containerd_artefacts @@ -65,6 +84,10 @@ function install_artifacts() { if [ "${INSTALL_VFIO_GPU_CONTAINERD}" = "true" ]; then install_vfio_gpu_containerd_artefacts fi + + if [ "${INSTALL_NYDUS_SNAPSHOTTER}" = "true" ]; then + install_nydus_snapshotter_artefacts + fi } function uninstall_containerd_artefacts() { @@ -87,7 +110,31 @@ function uninstall_containerd_artefacts() { fi } +function uninstall_nydus_snapshotter_artefacts() { + if host_systemctl list-units | grep -q nydus-snapshotter; then + remove_nydus_snapshotter_from_containerd + host_systemctl disable --now nydus-snapshotter.service + rm -rf /etc/systemd/system/nydus-snapshotter.service + + restart_systemd_service + fi + + echo "Removing nydus-snapshotter artifacts from host" + for i in `ctr -n k8s.io snapshot --snapshotter nydus list | grep -v KEY | cut -d' ' -f1`; do + ctr -n k8s.io snapshot --snapshotter nydus rm $i || true + done + rm -f /opt/confidential-containers/bin/containerd-nydus-grpc + rm -f /opt/confidential-containers/bin/nydus-overlayfs + rm -f /usr/local/bin/nydus-overlayfs + rm -f /opt/confidential-containers/share/remote-snapshotter/config-coco-guest-pulling.toml + rm -rf /var/lib/containerd-nydus/* +} + function uninstall_artifacts() { + if [ "${INSTALL_NYDUS_SNAPSHOTTER}" = "true" ]; then + uninstall_nydus_snapshotter_artefacts + fi + if [ "${INSTALL_COCO_CONTAINERD}" = "true" ] || [ "${INSTALL_OFFICIAL_CONTAINERD}" = "true" ] || [ "${INSTALL_VFIO_GPU_CONTAINERD}" = "true" ]; then uninstall_containerd_artefacts fi @@ -99,6 +146,47 @@ function restart_systemd_service() { host_systemctl restart "${container_engine}" } +function configure_nydus_snapshotter_for_containerd() { + echo "configure nydus snapshotter for containerd" + + if [ ! -f "$containerd_config" ]; then + die "failed to find containerd config" + fi + + containerd_imports_path="/etc/containerd/config.toml.d" + + echo "Create ${containerd_imports_path}" + mkdir -p "${containerd_imports_path}" + + echo "Drop-in the nydus configuration" + cat << EOF | tee "${containerd_imports_path}"/nydus-snapshotter.toml +[proxy_plugins] + [proxy_plugins.nydus] + type = "snapshot" + address = "/run/containerd-nydus/containerd-nydus-grpc.sock" +EOF + if grep -q "^imports = " "$containerd_config"; then + sed -i -e "s|^imports = \[\(.*\)\]|imports = [\"${containerd_imports_path}/nydus-snapshotter.toml\", \1]|g" ${containerd_config} + sed -i -e "s|, ]|]|g" ${containerd_config} + else + sed -i -e "1s|^|imports = [\"${containerd_imports_path}/nydus-snapshotter.toml\"]\n|" ${containerd_config} + fi + + sed -i -e "s|disable_snapshot_annotations = true|disable_snapshot_annotations = false|" ${containerd_config} +} + +function remove_nydus_snapshotter_from_containerd() { + echo "Remove nydus snapshotter from containerd" + + containerd_imports_path="/etc/containerd/config.toml.d" + + rm -f "${containerd_imports_path}/nydus-snapshotter.toml" + sed -i -e "s|\"${containerd_imports_path}/nydus-snapshotter.toml\"||g" ${containerd_config} + sed -i -e "s|, ]|]|g" ${containerd_config} + + sed -i -e "s|disable_snapshot_annotations = false|disable_snapshot_annotations = true|" ${containerd_config} +} + label_node() { case "${1}" in install) @@ -120,6 +208,7 @@ function main() { echo "INSTALL_COCO_CONTAINERD: ${INSTALL_COCO_CONTAINERD}" echo "INSTALL_OFFICIAL_CONTAINERD: ${INSTALL_OFFICIAL_CONTAINERD}" echo "INSTALL_VFIO_GPU_CONTAINERD: ${INSTALL_VFIO_GPU_CONTAINERD}" + echo "INSTALL_NYDUS_SNAPSHOTTER: ${INSTALL_NYDUS_SNAPSHOTTER}" # script requires that user is root local euid=$(id -u) From 411ca5af8314930b56a395c18b996f7c7ac7b269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Fri, 20 Oct 2023 12:30:45 +0200 Subject: [PATCH 10/15] config: Switch to using the nydus-snapshotter by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is done for the kata-containers based payloads. Enclave CC will still keep using the forked containerd Signed-off-by: ChengyuZhu6 Signed-off-by: Fabiano Fidêncio --- config/samples/ccruntime/base/ccruntime.yaml | 12 ++++++------ config/samples/ccruntime/default/kustomization.yaml | 12 ++++++------ .../samples/ccruntime/peer-pods/kustomization.yaml | 2 +- config/samples/ccruntime/s390x/kustomization.yaml | 4 ++-- config/samples/ccruntime/ssh-demo/kustomization.yaml | 6 +++--- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/config/samples/ccruntime/base/ccruntime.yaml b/config/samples/ccruntime/base/ccruntime.yaml index dc0abcaa..deec149a 100644 --- a/config/samples/ccruntime/base/ccruntime.yaml +++ b/config/samples/ccruntime/base/ccruntime.yaml @@ -88,11 +88,11 @@ spec: # for handling GPU / VFIO, on the node # default: true - name: "INSTALL_COCO_CONTAINERD" - value: "true" + value: "false" # If set to true, this will install the v1.7.0 release of containerd on the node. # default: false - name: "INSTALL_OFFICIAL_CONTAINERD" - value: "false" + value: "true" # If set to true, this will install the CoCo fork of the containerd, # the one that has patches for handling GPU / VFIO, on the node # default: false @@ -102,7 +102,7 @@ spec: # on the node # default: false - name: "INSTALL_NYDUS_SNAPSHOTTER" - value: "false" + value: "true" preInstall: image: quay.io/confidential-containers/reqs-payload volumeMounts: @@ -142,11 +142,11 @@ spec: # the one allowing images to be pulled inside the guest and has patches # for handling GPU / VFIO, on the node - name: "INSTALL_COCO_CONTAINERD" - value: "true" + value: "false" # If set to true, this will install the v1.7.0 release of containerd on the node. # default: false - name: "INSTALL_OFFICIAL_CONTAINERD" - value: "false" + value: "true" # If set to true, this will install the CoCo fork of the containerd, # the one that has patches for handling GPU / VFIO, on the node # default: false @@ -156,7 +156,7 @@ spec: # on the node # default: false - name: "INSTALL_NYDUS_SNAPSHOTTER" - value: "false" + value: "true" environmentVariables: - name: NODE_NAME valueFrom: diff --git a/config/samples/ccruntime/default/kustomization.yaml b/config/samples/ccruntime/default/kustomization.yaml index 7c4e3661..ac60fd8e 100644 --- a/config/samples/ccruntime/default/kustomization.yaml +++ b/config/samples/ccruntime/default/kustomization.yaml @@ -19,17 +19,17 @@ patches: path: /spec/config/runtimeClasses value: - name: "kata-clh" - snapshotter: "overlayfs" + snapshotter: "nydus" - name: "kata-clh-tdx" - snapshotter: "overlayfs" + snapshotter: "nydus" - name: "kata-qemu" - snapshotter: "overlayfs" + snapshotter: "nydus" - name: "kata-qemu-tdx" - snapshotter: "overlayfs" + snapshotter: "nydus" - name: "kata-qemu-sev" - snapshotter: "overlayfs" + snapshotter: "nydus" - name: "kata-qemu-snp" - snapshotter: "overlayfs" + snapshotter: "nydus" - op: add path: /spec/config/defaultRuntimeClassName value: "kata-qemu" diff --git a/config/samples/ccruntime/peer-pods/kustomization.yaml b/config/samples/ccruntime/peer-pods/kustomization.yaml index bcdbf2fc..707d84eb 100644 --- a/config/samples/ccruntime/peer-pods/kustomization.yaml +++ b/config/samples/ccruntime/peer-pods/kustomization.yaml @@ -19,7 +19,7 @@ patches: path: /spec/config/runtimeClasses value: - name: "kata-remote" - snapshotter: "overlayfs" + snapshotter: "nydus" - op: add path: /spec/config/debug value: false diff --git a/config/samples/ccruntime/s390x/kustomization.yaml b/config/samples/ccruntime/s390x/kustomization.yaml index b5763a90..e1016d78 100644 --- a/config/samples/ccruntime/s390x/kustomization.yaml +++ b/config/samples/ccruntime/s390x/kustomization.yaml @@ -17,9 +17,9 @@ patches: path: /spec/config/runtimeClasses value: - name: "kata-qemu" - snapshotter: "overlayfs" + snapshotter: "nydus" - name: "kata-qemu-se" - snapshotter: "overlayfs" + snapshotter: "nydus" - op: add path: /spec/config/defaultRuntimeClassName value: "kata-qemu" diff --git a/config/samples/ccruntime/ssh-demo/kustomization.yaml b/config/samples/ccruntime/ssh-demo/kustomization.yaml index bb1b8f5d..8dd3b3e0 100644 --- a/config/samples/ccruntime/ssh-demo/kustomization.yaml +++ b/config/samples/ccruntime/ssh-demo/kustomization.yaml @@ -18,11 +18,11 @@ patches: path: /spec/config/runtimeClasses value: - name: "kata" - snapshotter: "overlayfs" + snapshotter: "nydus" - name: "kata-clh" - snapshotter: "overlayfs" + snapshotter: "nydus" - name: "kata-qemu" - snapshotter: "overlayfs" + snapshotter: "nydus" target: kind: CcRuntime From 17973974521a66b83f0eab754dfbeacaf25f883e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 11 Oct 2023 23:34:39 +0200 Subject: [PATCH 11/15] pre-install: Bump containerd to its latest release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fabiano Fidêncio --- install/pre-install-payload/Makefile | 2 +- install/pre-install-payload/payload.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/install/pre-install-payload/Makefile b/install/pre-install-payload/Makefile index a0aaac76..35288e68 100644 --- a/install/pre-install-payload/Makefile +++ b/install/pre-install-payload/Makefile @@ -1,5 +1,5 @@ COCO_CONTAINERD_VERSION = 1.6.8.2 -OFFICIAL_CONTAINERD_VERSION = 1.7.0 +OFFICIAL_CONTAINERD_VERSION = 1.7.7 VFIO_GPU_CONTAINERD_VERSION = 1.7.0.0 NYDUS_SNAPSHOTTER_VERSION = v0.13.3-multiarch diff --git a/install/pre-install-payload/payload.sh b/install/pre-install-payload/payload.sh index 8820c43b..23739147 100755 --- a/install/pre-install-payload/payload.sh +++ b/install/pre-install-payload/payload.sh @@ -9,7 +9,7 @@ script_dir=$(dirname "$(readlink -f "$0")") coco_containerd_repo=${coco_containerd_repo:-"https://github.com/confidential-containers/containerd"} coco_containerd_version=${coco_containerd_version:-"1.6.8.2"} official_containerd_repo=${official_containerd_repo:-"https://github.com/containerd/containerd"} -official_containerd_version=${official_containerd_version:-"1.7.0"} +official_containerd_version=${official_containerd_version:-"1.7.7"} vfio_gpu_containerd_repo=${vfio_gpu_containerd_repo:-"https://github.com/confidential-containers/containerd"} vfio_gpu_containerd_version=${vfio_gpu_containerd_version:-"1.7.0.0"} nydus_snapshotter_repo=${nydus_snapshotter_repo:-"https://github.com/fidencio/nydus-snapshotter"} From 6bec6fea64e921858bf5fe8cc39a9a9507061ddd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Mon, 9 Oct 2023 17:45:00 +0200 Subject: [PATCH 12/15] controllers:: Fix snapshotter setup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The condition to set the snapshotter to be used is defintely wrong, as it'd never ever set the snapshotter variable. Signed-off-by: Fabiano Fidêncio --- controllers/ccruntime_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/ccruntime_controller.go b/controllers/ccruntime_controller.go index 00be51d7..06387fc0 100644 --- a/controllers/ccruntime_controller.go +++ b/controllers/ccruntime_controller.go @@ -622,7 +622,7 @@ func (r *CcRuntimeReconciler) processDaemonset(operation DaemonOperation) *appsv // payload script supports setting one snapshotter per runtime handler. // For now, for the v0.8.0 release, we're fine assuming that all the // set snapshotters are going to be the same. - if snapshotter != "" { + if snapshotter == "" && runtimeClass.Snapshotter != "" { snapshotter = runtimeClass.Snapshotter } } From 51ec98c162d06ef0be2efc41bc91b227a0f4837b Mon Sep 17 00:00:00 2001 From: Huang Huang Date: Fri, 22 Sep 2023 21:35:55 +0800 Subject: [PATCH 13/15] Fix `panic: odd number of arguments passed as key-value pairs for logging` ``` panic: odd number of arguments passed as key-value pairs for logging [recovered] panic: odd number of arguments passed as key-value pairs for logging ... github.com/confidential-containers/operator/controllers.(*CcRuntimeReconciler).monitorCcRuntimeInstallation(0xc0002d4190) /workspace/controllers/ccruntime_controller.go:473 +0x3bf ``` Signed-off-by: Huang Huang --- controllers/ccruntime_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/ccruntime_controller.go b/controllers/ccruntime_controller.go index 06387fc0..ce292ec4 100644 --- a/controllers/ccruntime_controller.go +++ b/controllers/ccruntime_controller.go @@ -470,7 +470,7 @@ func (r *CcRuntimeReconciler) monitorCcRuntimeInstallation() (ctrl.Result, error foundRc := &nodeapi.RuntimeClass{} err := r.Client.Get(context.TODO(), types.NamespacedName{Name: runtimeClass.Name}, foundRc) if errors.IsNotFound(err) { - r.Log.Info("The runtime payload failed to create the runtime class named %s", runtimeClass.Name) + r.Log.Info("The runtime payload failed to create the runtime class", "runtimeClassName", runtimeClass.Name) return ctrl.Result{}, err } runtimeClassNames = append(runtimeClassNames, runtimeClass.Name) From a9ee6b0fb1c4000480f2899ee89799177367c73d Mon Sep 17 00:00:00 2001 From: Wainer dos Santos Moschetta Date: Tue, 24 Oct 2023 09:53:38 -0300 Subject: [PATCH 14/15] tests/e2e: describe all pods on teardown On operator_tests.bats's teardown let's print the description of all pods in the confidential-containers namespace to help on debug fails. Signed-off-by: Wainer dos Santos Moschetta --- tests/e2e/operator_tests.bats | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/e2e/operator_tests.bats b/tests/e2e/operator_tests.bats index d2a040c3..d6fa8351 100644 --- a/tests/e2e/operator_tests.bats +++ b/tests/e2e/operator_tests.bats @@ -44,4 +44,7 @@ systemctl is-active "$container_runtime" teardown() { # For debugging sake. kubectl get pods -A || true + echo "::group::Describe all pods of confidential-containers namespace" + kubectl -n confidential-containers describe pods || true + echo "::endgroup::" } From 619eb49aae6493d4f5b66ee249cc80fc73da01c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabiano=20Fid=C3=AAncio?= Date: Wed, 25 Oct 2023 20:53:20 +0200 Subject: [PATCH 15/15] controllers: Workaround Uninstall / postUninstall race condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This should be treated in a better way than adding an one minute sleep, as just having the sleep here won't do us any good in the future. What's basically happening, and forcing us to do this, is the fact that the Uninstall and postUninstall daemonsets are being started at exactly the same time, leading to a race condition when changing the containerd configuration. When looking at the kata-containers payload code, we see that the the label is only set after containerd is successfully reconfigured, and looking at this function we see we shouldn't reach this part before the label is set. However, that's not what we're facing ... In order to unblock this PR to get merged, we're good enough. But this needs proper investigation and hopefully we'll see this patch being reverted sooner than later. Signed-off-by: Fabiano Fidêncio --- controllers/ccruntime_controller.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/controllers/ccruntime_controller.go b/controllers/ccruntime_controller.go index ce292ec4..e3928096 100644 --- a/controllers/ccruntime_controller.go +++ b/controllers/ccruntime_controller.go @@ -226,6 +226,20 @@ func (r *CcRuntimeReconciler) processCcRuntimeDeleteRequest() (ctrl.Result, erro if r.ccRuntime.Spec.Config.PostUninstall.Image == "" { controllerutil.RemoveFinalizer(r.ccRuntime, RuntimeConfigFinalizer) } else if r.ccRuntime.Spec.Config.PostUninstall.Image != "" { + // FXIME: This should be treated in a better way, as just having the sleep + // here won't do us any good in the future. + // + // What's basically happening, and forcing us to do this, is the + // fact that the Uninstall and postUninstall daemonsets are being + // started at exactly the same time, leading to a race condition + // when changing the containerd configuration. + // + // When looking at the kata-containers payload code, we see that the + // the label is only set after containerd is successfully reconfigured, + // and looking at this function we see we shouldn't reach this part + // before the label is set. However, that's not what we're facing ... + time.Sleep(time.Second * 60) + result, err = handlePostUninstall(r) if !result.Requeue { controllerutil.RemoveFinalizer(r.ccRuntime, RuntimeConfigFinalizer)