diff --git a/Makefile b/Makefile index 48b892219..b2a5c07e8 100644 --- a/Makefile +++ b/Makefile @@ -104,7 +104,7 @@ cc-containerd: # Run the Confidential Containers tests for kubernetes. cc-kubernetes: bash -f .ci/install_bats.sh - K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats" \ + K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats confidential/image_pulling_with_snapshotter.bats" \ bash integration/kubernetes/run_kubernetes_tests.sh # Run the Confidential Containers AMD SEV specific tests. diff --git a/integration/confidential/lib.sh b/integration/confidential/lib.sh index 6e0873d05..63f74407f 100644 --- a/integration/confidential/lib.sh +++ b/integration/confidential/lib.sh @@ -12,6 +12,10 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash" source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh" FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures" SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures" +NYDUS_SNAPSHOTTER_BINARY="/opt/confidential-containers/bin/containerd-nydus-grpc" +NYDUS_SNAPSHOTTER_TARFS_CONFIG="/opt/confidential-containers/share/nydus-snapshotter/config-coco-host-sharing.toml" +NYDUS_SNAPSHOTTER_GUEST_CONFIG="/opt/confidential-containers/share/nydus-snapshotter/config-coco-guest-pulling.toml" +NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG" # Toggle between true and false the service_offload configuration of # the Kata agent. @@ -223,6 +227,8 @@ configure_cc_containerd() { sudo tee -a "$containerd_conf_file" fi + configure_containerd_for_nydus_snapshotter "$containerd_conf_file" + sudo systemctl restart containerd if ! waitForProcess 30 5 "sudo crictl info >/dev/null"; then die "containerd seems not operational after reconfigured" @@ -445,3 +451,85 @@ EOF EOF fi } + +############################################################################### + +# remote-snapshotter + +configure_remote_snapshotter() { + case "${SNAPSHOTTER:-}" in + "nydus") + configure_nydus_snapshotter + ;; + *) ;; + + esac +} + +is_containerd_support_per_runtime_snapshotter () { + containerd_version=$(containerd --version | awk '{print $3}') + required_version="v1.7.0" + printf '%s\n' ${required_version} ${containerd_version} | sort --check=quiet -V +} + +configure_containerd_for_nydus_snapshotter() { + if [ "${SNAPSHOTTER:-}" = "nydus" ]; then + local containerd_config="$1" + sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g' "$containerd_config" + if is_containerd_support_per_runtime_snapshotter; then + sudo sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/a\ snapshotter = "nydus"' "$containerd_config" + else + sudo sed -i 's/snapshotter = .*/snapshotter = "nydus"/g' "$containerd_config" + fi + fi +} + +kill_nydus_snapshotter_process() { + echo "Kill nydus snapshotter" + bin="containerd-nydus-grpc" + sudo kill -9 $(pidof $bin) || true + kill -9 $(pidof $bin) || true + rm -rf "/var/lib/containerd-nydus" || true +} + +restore_containerd_config() { + local saved_containerd_conf_file="$1" + echo "Restore containerd to pre-test state." + if [ -f "$saved_containerd_conf_file" ]; then + systemctl stop containerd || true + sleep 5 + sudo mv -f "$saved_containerd_conf_file" "/etc/containerd/config.toml" + systemctl start containerd || true + fi +} + +remove_test_image() { + local test_image="$1" + if ctr -n k8s.io i ls | grep -q "$test_image"; then + crictl rmi "$1" + fi + + pause_name=$(grep 'sandbox_image' /etc/containerd/config.toml | awk -F'"' '{print $2}') + if ctr -n k8s.io i ls | grep -q "$pause_name"; then + crictl rmi "$pause_name" + fi +} + +restart_nydus_snapshotter() { + kill_nydus_snapshotter_process || true + echo "Restart nydus snapshotter" + sudo "$NYDUS_SNAPSHOTTER_BINARY" --config "$NYDUS_SNAPSHOTTER_CONFIG" >/dev/stdout 2>&1 & +} + +configure_nydus_snapshotter() { + if [ "${SNAPSHOTTER:-}" = "nydus" ]; then + echo "Configure nydus snapshotter" + if [ "$EXPORT_MODE" == "image_guest_pull" ]; then + NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_GUEST_CONFIG" + else + NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG" + sudo sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" "$NYDUS_SNAPSHOTTER_CONFIG" + fi + restart_nydus_snapshotter + fi +} \ No newline at end of file diff --git a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in index 3c8e9d3c1..1d0b7def0 100644 --- a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in +++ b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in @@ -5,7 +5,7 @@ apiVersion: v1 kind: Pod metadata: - name: busybox-cc + name: busybox-cc$INDEX spec: runtimeClassName: $RUNTIMECLASS containers: diff --git a/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats b/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats new file mode 100644 index 000000000..7ac6f34f7 --- /dev/null +++ b/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats @@ -0,0 +1,89 @@ +#!/usr/bin/env bats +# Copyright (c) 2023 Intel Corporation +# Copyright (c) 2023 IBM Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +load "${BATS_TEST_DIRNAME}/lib.sh" +load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh" +load "${BATS_TEST_DIRNAME}/tests_common.sh" + +tag_suffix="" +if [ "$(uname -m)" != "x86_64" ]; then + tag_suffix="-$(uname -m)" +fi + +# Images used on the tests. + +image_unsigned_protected="quay.io/kata-containers/confidential-containers:unsigned${tag_suffix}" + +original_kernel_params=$(get_kernel_params) +# Allow to configure the runtimeClassName on pod configuration. +RUNTIMECLASS="${RUNTIMECLASS:-kata}" +test_tag="[cc][agent][kubernetes][containerd]" + +setup() { + remove_test_image "$image_unsigned_protected" + kill_nydus_snapshotter_process + setup_common +} + +@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" { + if [ "$(uname -m)" = s390x ]; then + skip "test for s390x as nydus-image doesn't currently support this platform" + fi + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + pod_config="$(new_pod_config "$image_unsigned_protected")" + echo $pod_config + create_test_pod "$pod_config" + fi +} + +@test "$test_tag Test can pull an image as a raw block disk image to guest without dm-verity" { + if [ "$(uname -m)" = s390x ]; then + skip "test for s390x as nydus-image doesn't currently support this platform" + fi + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_block" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + pod_config="$(new_pod_config "$image_unsigned_protected")" + echo $pod_config + create_test_pod "$pod_config" + fi +} + +@test "$test_tag Test can create two pods with pulling the image only once with dm-verity enabled" { + if [ "$(uname -m)" = s390x ]; then + skip "test for s390x as nydus-image doesn't currently support this platform" + fi + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + + pod_config_1="$(new_pod_config "$image_unsigned_protected" "1")" + echo $pod_config_1 + create_test_pod $pod_config_1 + pod_config_2="$(new_pod_config "$image_unsigned_protected" "2")" + echo $pod_config_2 + create_test_pod $pod_config_2 + + pull_times=$(journalctl -g "PullImage \"$image_unsigned_protected\" with snapshotter nydus" | wc -l) + [ ${pull_times} -eq 1 ] + fi +} + +@test "$test_tag Test can pull an image inside the guest with remote-snapshotter" { + switch_image_service_offload on + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_guest_pull" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + pod_config="$(new_pod_config "$image_unsigned_protected")" + echo $pod_config + create_test_pod $pod_config + fi +} + +teardown() { + remove_test_image "$image_unsigned_protected" + kill_nydus_snapshotter_process + teardown_common +} diff --git a/integration/kubernetes/confidential/lib.sh b/integration/kubernetes/confidential/lib.sh index 9e9d904cc..aed45f0fb 100755 --- a/integration/kubernetes/confidential/lib.sh +++ b/integration/kubernetes/confidential/lib.sh @@ -46,7 +46,9 @@ kubernetes_create_cc_pod() { fi kubectl apply -f ${config_file} - if ! pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}'); then + pod_name=$(${GOPATH}/bin/yq r ${config_file} 'metadata.name') + echo $(kubectl get pod "$pod_name" &> /dev/null) + if ! kubectl get pod "$pod_name" &> /dev/null; then echo "Failed to create the pod" return 1 fi diff --git a/integration/kubernetes/confidential/tests_common.sh b/integration/kubernetes/confidential/tests_common.sh index 177ed450e..50f106dac 100644 --- a/integration/kubernetes/confidential/tests_common.sh +++ b/integration/kubernetes/confidential/tests_common.sh @@ -97,8 +97,9 @@ create_test_pod() { new_pod_config() { local base_config="${FIXTURES_DIR}/pod-config.yaml.in" local image="$1" + local index="${2:-}" local new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename ${base_config}).XXX") - IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" envsubst < "$base_config" > "$new_config" + IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" INDEX="$index" envsubst < "$base_config" > "$new_config" echo "$new_config" }