From 502f11efa524ea6c9ee6f3fb7224a46a5d54ac09 Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Wed, 6 Sep 2023 11:14:23 +0800 Subject: [PATCH] ci: Add test cases for CoCo image pulling without forked containerd Additional tests are necessary to verify new feature that pulling image without forked containerd in CoCo. 1)image sharing on the host without dm-verity. 2)image sharing on the host with dm-verity. 3)image pulling in the guest with nydus-snapshotter. Fixes #5763 Depends-on:github.com/kata-containers/kata-containers/pull/7676 Depends-on:github.com/confidential-containers/operator#251 Signed-off-by: ChengyuZhu6 --- Makefile | 2 +- integration/confidential/lib.sh | 88 +++++++++ .../confidential/fixtures/pod-config.yaml.in | 2 +- .../image_pulling_with_snapshotter.bats | 182 ++++++++++++++++++ integration/kubernetes/confidential/lib.sh | 4 +- 5 files changed, 275 insertions(+), 3 deletions(-) create mode 100644 integration/kubernetes/confidential/image_pulling_with_snapshotter.bats diff --git a/Makefile b/Makefile index 48b892219..8b8ebfc94 100644 --- a/Makefile +++ b/Makefile @@ -104,7 +104,7 @@ cc-containerd: # Run the Confidential Containers tests for kubernetes. cc-kubernetes: bash -f .ci/install_bats.sh - K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats" \ + K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats" confidential/image_pulling_with_snapshotter.bats" \ bash integration/kubernetes/run_kubernetes_tests.sh # Run the Confidential Containers AMD SEV specific tests. diff --git a/integration/confidential/lib.sh b/integration/confidential/lib.sh index 6e0873d05..8b8467a28 100644 --- a/integration/confidential/lib.sh +++ b/integration/confidential/lib.sh @@ -12,6 +12,10 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash" source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh" FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures" SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures" +NYDUS_SNAPSHOTTER_BINARY="/opt/confidential-containers/bin/containerd-nydus-grpc" +NYDUS_SNAPSHOTTER_TARFS_CONFIG="/opt/confidential-containers/share/nydus-snapshotter/config-coco-host-sharing.toml" +NYDUS_SNAPSHOTTER_GUEST_CONFIG="/opt/confidential-containers/share/nydus-snapshotter/config-coco-guest-pulling.toml" +NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG" # Toggle between true and false the service_offload configuration of # the Kata agent. @@ -223,6 +227,8 @@ configure_cc_containerd() { sudo tee -a "$containerd_conf_file" fi + configure_containerd_for_nydus_snapshotter "$containerd_conf_file" + sudo systemctl restart containerd if ! waitForProcess 30 5 "sudo crictl info >/dev/null"; then die "containerd seems not operational after reconfigured" @@ -445,3 +451,85 @@ EOF EOF fi } + +############################################################################### + +# remote-snapshotter + +configure_remote_snapshotter() { + case "${SNAPSHOTTER:-}" in + "nydus") + configure_nydus_snapshotter + ;; + *) ;; + + esac +} + +is_containerd_support_per_runtime_snapshotter () { + containerd_version=$(containerd --version | awk '{print $3}') + required_version="v1.7.0" + printf '%s\n' ${required_version} ${containerd_version} | sort --check=quiet -V +} + +configure_containerd_for_nydus_snapshotter() { + if [ "${SNAPSHOTTER:-}" = "nydus" ]; then + local containerd_config="$1" + sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g' "$containerd_config" + if is_containerd_support_per_runtime_snapshotter; then + sudo sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/a\ snapshotter = "nydus"' "$containerd_config" + else + sudo sed -i 's/snapshotter = .*/snapshotter = "nydus"/g' "$containerd_config" + fi + fi +} + +kill_nydus_snapshotter_process() { + echo "Kill nydus snapshotter" + bin="containerd-nydus-grpc" + sudo kill -9 $(pidof $bin) || true + kill -9 $(pidof $bin) || true + rm -r "/var/lib/containerd-nydus" || true +} + +restore_containerd_config() { + local saved_containerd_conf_file="$1" + echo "Restore containerd to pre-test state." + if [ -f "$saved_containerd_conf_file" ]; then + systemctl stop containerd || true + sleep 5 + sudo mv -f "$saved_containerd_conf_file" "/etc/containerd/config.toml" + systemctl start containerd || true + fi +} + +remove_test_image() { + local test_image="$1" + if ctr -n k8s.io i ls | grep -q "$test_image"; then + crictl rmi "$1" + fi + + pause_name=$(grep 'sandbox_image' /etc/containerd/config.toml | awk -F'"' '{print $2}') + if ctr -n k8s.io i ls | grep -q "$pause_name"; then + crictl rmi "$pause_name" + fi +} + +restart_nydus_snapshotter() { + kill_nydus_snapshotter_process || true + echo "Restart nydus snapshotter" + sudo "$NYDUS_SNAPSHOTTER_BINARY" --config "$NYDUS_SNAPSHOTTER_CONFIG" >/dev/stdout 2>&1 & +} + +configure_nydus_snapshotter() { + if [ "${SNAPSHOTTER:-}" = "nydus" ]; then + echo "Configure nydus snapshotter" + if [ "$EXPORT_MODE" == "image_guest_pull" ]; then + NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_GUEST_CONFIG" + else + NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG" + sudo sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" "$NYDUS_SNAPSHOTTER_CONFIG" + fi + restart_nydus_snapshotter + fi +} \ No newline at end of file diff --git a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in index 3c8e9d3c1..1d0b7def0 100644 --- a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in +++ b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in @@ -5,7 +5,7 @@ apiVersion: v1 kind: Pod metadata: - name: busybox-cc + name: busybox-cc$INDEX spec: runtimeClassName: $RUNTIMECLASS containers: diff --git a/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats b/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats new file mode 100644 index 000000000..91f4ce4d6 --- /dev/null +++ b/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats @@ -0,0 +1,182 @@ +#!/usr/bin/env bats +# Copyright (c) 2023 Intel Corporation +# Copyright (c) 2023 IBM Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +load "${BATS_TEST_DIRNAME}/lib.sh" +load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh" + +tag_suffix="" +if [ "$(uname -m)" != "x86_64" ]; then + tag_suffix="-$(uname -m)" +fi + +# Images used on the tests. + +image_unsigned_protected="quay.io/kata-containers/confidential-containers:unsigned${tag_suffix}" + +original_kernel_params=$(get_kernel_params) +# Allow to configure the runtimeClassName on pod configuration. +RUNTIMECLASS="${RUNTIMECLASS:-kata}" +test_tag="[cc][agent][kubernetes][containerd]" + +# Create the test pod. +# +# Note: the global $sandbox_name, $pod_config should be set +# already. It also relies on $CI and $DEBUG exported by CI scripts or +# the developer, to decide how to set debug flags. +# +create_test_pod() { + local pod_config="$1" + # On CI mode we only want to enable the agent debug for the case of + # the test failure to obtain logs. + if [ "${CI:-}" == "true" ]; then + enable_full_debug + elif [ "${DEBUG:-}" == "true" ]; then + enable_full_debug + enable_agent_console + fi + + echo "Create the test sandbox" + echo "Pod config is: $pod_config" + kubernetes_create_cc_pod $pod_config +} + +# Create a pod configuration out of a template file. +# +# Parameters: +# $1 - the container image. +# Return: +# the path to the configuration file. The caller should not care about +# its removal afterwards as it is created under the bats temporary +# directory. +# +# Environment variables: +# RUNTIMECLASS: set the runtimeClassName value from $RUNTIMECLASS. +# +new_pod_config() { + local base_config="${FIXTURES_DIR}/pod-config.yaml.in" + local image="$1" + local index="$2" + + local new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename ${base_config}).XXX") + IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" INDEX="$2" envsubst <"$base_config" >"$new_config" + echo "$new_config" +} + +setup() { + start_date=$(date +"%Y-%m-%d %H:%M:%S") + + pod_config="$(new_pod_config "$image_unsigned_protected")" + pod_id="" + + kubernetes_delete_all_cc_pods_if_any_exists || true + remove_test_image "$image_unsigned_protected" + kill_nydus_snapshotter_process + rm -rf "/var/lib/containerd-nydus" + + echo "Prepare containerd for Confidential Container" + SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$" + configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE" + + echo "Reconfigure Kata Containers" + switch_image_service_offload off + clear_kernel_params + add_kernel_params "${original_kernel_params}" + + setup_proxy + switch_measured_rootfs_verity_scheme none +} + +# Check the logged messages on host have a given message. +# Parameters: +# $1 - the message +# +# Note: get the logs since the global $start_date. +# +assert_logs_contain() { + local message="$1" + # Note: with image-rs we get more that the default 1000 lines of logs + journalctl -x -t kata --since "$start_date" -n 100000 | grep "$message" +} + +@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" { + if [ "$(uname -m)" = s390x ]; then + skip "test for s390x as nydus-image doesn't currently support this platform" + fi + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + pod_config="$(new_pod_config "$image_unsigned_protected")" + echo $pod_config + create_test_pod "$pod_config" + fi +} + +@test "$test_tag Test can pull an image as a raw block disk image to guest without dm-verity" { + if [ "$(uname -m)" = s390x ]; then + skip "test for s390x as nydus-image doesn't currently support this platform" + fi + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_block" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + pod_config="$(new_pod_config "$image_unsigned_protected")" + echo $pod_config + create_test_pod "$pod_config" + fi +} + +@test "$test_tag Test can create two pods with pulling the image only once with dm-verity enabled" { + if [ "$(uname -m)" = s390x ]; then + skip "test for s390x as nydus-image doesn't currently support this platform" + fi + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + + pod_config_1="$(new_pod_config "$image_unsigned_protected" "1")" + echo $pod_config_1 + create_test_pod $pod_config_1 + pod_config_2="$(new_pod_config "$image_unsigned_protected" "2")" + echo $pod_config_2 + create_test_pod $pod_config_2 + + pull_times=$(journalctl -g "PullImage \"$image_unsigned_protected\" with snapshotter nydus" | wc -l) + [ ${pull_times} -eq 1 ] + fi +} + +@test "$test_tag Test can pull an image inside the guest with remote-snapshotter" { + switch_image_service_offload on + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_guest_pull" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + pod_config="$(new_pod_config "$image_unsigned_protected")" + echo $pod_config + create_test_pod $pod_config + fi +} + +teardown() { + # Print the logs and cleanup resources. + echo "-- Kata logs:" + sudo journalctl -xe -t kata --since "$start_date" -n 100000 + + remove_test_image "$image_unsigned_protected" + kill_nydus_snapshotter_process + + # Allow to not destroy the environment if you are developing/debugging + # tests. + if [[ "${CI:-false}" == "false" && "${DEBUG:-}" == true ]]; then + echo "Leaving changes and created resources untouched" + return + fi + + kubernetes_delete_all_cc_pods_if_any_exists || true + remove_test_image "$image_unsigned_protected" + kill_nydus_snapshotter_process + rm -rf "/var/lib/containerd-nydus" + clear_kernel_params + add_kernel_params "${original_kernel_params}" + switch_image_service_offload off + disable_full_debug + restore_containerd_config "$SAVED_CONTAINERD_CONF_FILE" +} diff --git a/integration/kubernetes/confidential/lib.sh b/integration/kubernetes/confidential/lib.sh index 1082530bd..17e000932 100755 --- a/integration/kubernetes/confidential/lib.sh +++ b/integration/kubernetes/confidential/lib.sh @@ -46,7 +46,9 @@ kubernetes_create_cc_pod() { fi kubectl apply -f ${config_file} - if ! pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}'); then + pod_name=$(${GOPATH}/bin/yq r ${config_file} 'metadata.name') + echo $(kubectl get pod "$pod_name" &> /dev/null) + if ! kubectl get pod "$pod_name" &> /dev/null; then echo "Failed to create the pod" return 1 fi