From 0523d88cb7f784dc8d282d24f74fdc72ed02dfd2 Mon Sep 17 00:00:00 2001 From: Gabriela Cervantes Date: Wed, 18 Sep 2024 22:53:56 +0000 Subject: [PATCH] tests/e2e: Add encrypted image test for operator This PR adds an encrypted image test for the operator repository. Signed-off-by: Gabriela Cervantes --- .github/workflows/ccruntime_e2e.yaml | 7 + tests/e2e/Dockerfile | 2 + tests/e2e/ansible/group_vars/all | 1 + tests/e2e/ansible/install_test_deps.yaml | 66 ++++++++ tests/e2e/encrypted.sh | 182 +++++++++++++++++++++++ tests/e2e/nginx-encrypted.yaml | 22 +++ tests/e2e/operator_tests.bats | 6 + 7 files changed, 286 insertions(+) create mode 100644 tests/e2e/Dockerfile create mode 100755 tests/e2e/encrypted.sh create mode 100644 tests/e2e/nginx-encrypted.yaml diff --git a/.github/workflows/ccruntime_e2e.yaml b/.github/workflows/ccruntime_e2e.yaml index 1d66b93f..3e08e085 100644 --- a/.github/workflows/ccruntime_e2e.yaml +++ b/.github/workflows/ccruntime_e2e.yaml @@ -55,6 +55,13 @@ jobs: env: TARGET_BRANCH: ${{ inputs.target-branch }} + - name: Login to Confidential Containers ghcr.io + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Install deps run: | sudo apt-get update -y diff --git a/tests/e2e/Dockerfile b/tests/e2e/Dockerfile new file mode 100644 index 00000000..7e287b16 --- /dev/null +++ b/tests/e2e/Dockerfile @@ -0,0 +1,2 @@ +FROM nginx:stable +RUN echo "something confidential" > /secret diff --git a/tests/e2e/ansible/group_vars/all b/tests/e2e/ansible/group_vars/all index 3a786520..d43cd759 100644 --- a/tests/e2e/ansible/group_vars/all +++ b/tests/e2e/ansible/group_vars/all @@ -15,6 +15,7 @@ build_pkgs: - gcc container_runtime: containerd go_version: 1.22.6 +rust_version: 1.75.0 # conntrack and socat are needed by the `kubeadm init` preflight checks kubeadm_pkgs: ubuntu: diff --git a/tests/e2e/ansible/install_test_deps.yaml b/tests/e2e/ansible/install_test_deps.yaml index 919a6ee6..cc5a0e32 100644 --- a/tests/e2e/ansible/install_test_deps.yaml +++ b/tests/e2e/ansible/install_test_deps.yaml @@ -33,6 +33,48 @@ path: bats-core state: absent when: bats_exist.rc != 0 + - block: + - name: Check skopeo is installed + shell: command -v skopeo >/dev/null 2>&1 + register: skopeo_exist + ignore_errors: yes + - name: Install skopeo + shell: | + sudo apt-get install -y build-essential libgpgme-dev libassuan-dev libbtrfs-dev pkg-config go-md2man + git clone https://github.com/containers/skopeo + cd skopeo + DISABLE_DOCS=1 make + DISABLE_DOCS=1 sudo make install + - block: + - name: Install kbs test dependencies + shell: | + sudo apt-get install -y build-essential pkg-config libssl-dev + - block: + - name: Check rust is installed + shell: command -v rustc >/dev/null 2>&1 + register: rustc_exist + ignore_errors: yes + - name: Install rust + shell: | + curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain {{ rust_version }} + - block: + - name: Create temp trustee directory + shell: | + mkdir /tmp/trustee + - name: Clone trustee repository + git: + repo: https://github.com/confidential-containers/trustee.git + dest: /tmp/trustee + clone: yes + force: yes + retries: 3 + delay: 10 + - name: Build and install the kbs client + shell: | + export PATH="${PATH}:${HOME}/.cargo/bin" + cd /tmp/trustee/kbs + make CLI_FEATURES=sample_only cli + sudo make install-cli - block: - name: Download Go tarball get_url: @@ -52,6 +94,19 @@ src: /usr/local/go/bin/go dest: /usr/local/bin/go state: link + - block: + - name: Check yq is installed + shell: command -v yq >/dev/null 2>&1 + register: yq_exist + ignore_errors: yes + - name: Install yq + shell: | + export yq_pkg="github.com/mikefarah/yq" + export yq_version=v4.40.7 + export yq_path="/usr/local/bin/yq" + yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_linux_{{ target_arch }}" + curl -o "${yq_path}" -LSsf "${yq_url}" + chmod +x "${yq_path}" - block: - name: Check kustomize is installed shell: command -v kustomize >/dev/null 2>&1 @@ -81,3 +136,14 @@ - /usr/local/go - /usr/local/bin/bats - /usr/local/bin/kustomize + - name: Uninstall kbs-client + shell: | + cd /tmp/trustee/kbs + sudo make uninstall + - name: Remove temporary file + file: + path: "{{ item }}" + state: absent + with_items: + - /tmp/trustee + diff --git a/tests/e2e/encrypted.sh b/tests/e2e/encrypted.sh new file mode 100755 index 00000000..e29a601f --- /dev/null +++ b/tests/e2e/encrypted.sh @@ -0,0 +1,182 @@ +#!/bin/bash +# +# Copyright Confidential Containers Contributors +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail + +script_dir="$(dirname "$(readlink -f "$0")")" +source "${script_dir}/lib.sh" + +export key_file="${key_file:-image_key}" +export key_path="${key_path:-/default/image_key/nginx}" +export trustee_repo="${trustee_repo:-https://github.com/confidential-containers/trustee.git}" +export kbs_namespace="${kbs_namespace:-coco-tenant}" +export aa_kbc="${aa_kbc:-cc_kbc}" +export kbs_svc_name="${kbs_svc_name:-kbs}" +export kbs_ingress_name="${kbs_ingress_name:-kbs}" +export runtimeclass="${runtimeclass:-kata-qemu}" +export username="${username:-}" +export encrypted_image="${encrypted_image:-ghcr.io/$username/nginx:latest}" + +build_encryption_key() { + docker build -t unencrypted . + # Encryption key needs to be 32 byte sequence + head -c 32 /dev/urandom | openssl enc > "${key_file}" + # Encryption key needs to be provided as base 64 enconded string + key_b64="$(base64 < ${key_file})" + key_id="kbs://${key_path}" + + # Image encryption logic + guest_components_repo="https://github.com/confidential-containers/guest-components.git" + guest_components_path="${script_dir}/guest-components" + if [ -d "${guest_components_path}" ]; then + echo "Guest components repo directory exists, removing it" + rm -rf "${guest_components_path}" + fi + + git clone "${guest_components_repo}" + + pushd "${guest_components_path}" + export tag_name="coco-keyprovider" + docker build -t "${tag_name}" -f ./attestation-agent/docker/Dockerfile.keyprovider . + mkdir -p oci/{input,output} + skopeo copy docker-daemon:unencrypted:latest dir:./oci/input + docker run -v "${PWD}/oci:/oci" "${tag_name}" /encrypt.sh -k "${key_b64}" -i "${key_id}" -s dir:/oci/input -d dir:/oci/output + skopeo inspect dir:./oci/output | jq '.LayersData[0].Annotations["org.opencontainers.image.enc.keys.provider.attestation-agent"] | @base64d | fromjson' + skopeo copy dir:./oci/output "docker://${encrypted_image}" + skopeo inspect "docker://${encrypted_image}" | jq -r '.Digest' + # Verify layer is encrypted + skopeo inspect "docker://${encrypted_image}" | jq -r '.LayersData[].MIMEType' | grep encrypted + skopeo inspect "docker://${encrypted_image}" | jq -r '.LayersData[].Annotations."org.opencontainers.image.enc.keys.provider.attestation-agent"' | base64 -d + skopeo inspect "docker://${encrypted_image}" | jq -r '.LayersData[].Annotations."org.opencontainers.image.enc.pubopts"' | base64 -d + popd +} + +deploy_k8s_kbs() { + if [ ! -d "${script_dir}/trustee" ]; then + git clone "${trustee_repo}" + fi + + pushd "${script_dir}/trustee/kbs/config/kubernetes" + echo "somesecret" > overlays/$(uname -m)/key.bin + export DEPLOYMENT_DIR=nodeport + ./deploy-kbs.sh + kbs_pod=$(kubectl -n "${kbs_namespace}" get pods -o NAME) + kubectl wait --for=condition=Ready -n "${kbs_namespace}" "${kbs_pod}" --timeout=300s + popd +} + +delete_k8s_kbs() { + pushd "${script_dir}/trustee/kbs/config/kubernetes" + kubectl delete -k overlays/$(uname -m) + popd + rm -rf "${script_dir}/trustee" +} + +provide_image_key() { + kubectl exec -n "${kbs_namespace}" "${kbs_pod}" -- mkdir -p "/opt/confidential-containers/kbs/repository/$(dirname "$key_path")" + cat "${key_file}" | kubectl exec -i -n "${kbs_namespace}" "${kbs_pod}" -- tee "/opt/confidential-containers/kbs/repository/${key_path}" > /dev/null +} + +launch_pod() { + kubectl apply -f "${script_dir}/nginx-encrypted.yaml" + export pod_name=$(kubectl -n default get pods -o NAME) +} + +delete_pod() { + kubectl delete "${pod_name}" +} + +check_image_key() { + kubectl logs -n "${kbs_namespace}" "${kbs_pod}" | grep "${key_path}" +} + +set_metadata_annotation() { + local yaml="${1}" + local key="${2}" + local value="${3}" + local metadata_path="${4:-}" + local annotation_key="" + + [ -n "$metadata_path" ] && annotation_key+="${metadata_path}." + + # yaml annotation key name. + annotation_key+="metadata.annotations.\"${key}\"" + + echo "$annotation_key" + # yq set annotations in yaml. Quoting the key because it can have + # dots. + yq -i ".${annotation_key} = \"${value}\"" "${yaml}" +} + +set_aa_kbc() { + local cc_kbs_addr + export cc_kbs_addr=$(kbs_k8s_svc_http_addr) + kernel_params_annotation="io.katacontainers.config.hypervisor.kernel_params" + kernel_params_value="agent.guest_components_rest_api=resource" + if [ "${aa_kbc}" = "cc_kbc" ]; then + kernel_params_value+=" agent.aa_kbc_params=cc_kbc::${cc_kbs_addr}" + fi + set_metadata_annotation "${script_dir}/nginx-encrypted.yaml" \ + "${kernel_params_annotation}" \ + "${kernel_params_value}" +} + +kbs_k8s_svc_http_addr() { + local host + local port + + host=$(kbs_k8s_svc_host) + port=$(kbs_k8s_svc_port) + + echo "http://${host}:${port}" +} + +kbs_k8s_svc_host() { + if kubectl get ingress -n "${kbs_namespace}" 2>/dev/null | grep -q kbs; then + kubectl get ingress "${kbs_ingress_name}" -n "${kbs_namespace}" \ + -o jsonpath='{.spec.rules[0].host}' 2>/dev/null + elif kubectl get svc "${kbs_svc_name}" -n "${kbs_namespace}" &>/dev/null; then + local host + host=$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}' -n "${kbs_namespace}") + echo "$host" + else + kubectl get svc "${kbs_svc_name}" -n "${kbs_namespace}" \ + -o jsonpath='{.spec.clusterIP}' 2>/dev/null + fi +} + +kbs_k8s_svc_port() { + if kubectl get ingress -n "${kbs_namespace}" 2>/dev/null | grep -q kbs; then + echo "80" + elif kubectl get svc "${kbs_svc_name}" -n "${kbs_namespace}" &>/dev/null; then + kubectl get svc "${kbs_svc_name}" -n "${kbs_namespace}" -o jsonpath='{.spec.ports[0].nodePort}' + else + kubectl get svc "${kbs_svc_name}" -n "${kbs_namespace}" \ + -o jsonpath='{.spec.ports[0].port}' 2>/dev/null + fi +} + +set_runtimeclass() { + sudo sed -i "s/RUNTIMECLASS/${runtimeclass}/g" "${script_dir}/nginx-encrypted.yaml" + sudo sed -i "s/USERNAME/${username}/g" "${script_dir}/nginx-encrypted.yaml" +} + +main() { + build_encryption_key + deploy_k8s_kbs + set_aa_kbc + set_runtimeclass + provide_image_key + launch_pod + check_image_key + delete_k8s_kbs + delete_pod +} + +main "$@" diff --git a/tests/e2e/nginx-encrypted.yaml b/tests/e2e/nginx-encrypted.yaml new file mode 100644 index 00000000..dee4ce29 --- /dev/null +++ b/tests/e2e/nginx-encrypted.yaml @@ -0,0 +1,22 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: nginx + name: nginx-encrypted +spec: + replicas: 1 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + annotations: + io.containerd.cri.runtime-handler: RUNTIMECLASS + spec: + runtimeClassName: RUNTIMECLASS + containers: + - image: docker://ghcr.io/USERNAME/nginx:latest + name: nginx diff --git a/tests/e2e/operator_tests.bats b/tests/e2e/operator_tests.bats index 0cc7449c..72890e68 100644 --- a/tests/e2e/operator_tests.bats +++ b/tests/e2e/operator_tests.bats @@ -21,6 +21,12 @@ setup() { ns="confidential-containers-system" } +@test "$test_tag Test encrypt image" { +is_operator_installed + +"${BATS_TEST_DIRNAME}/encrypted.sh" +} + @test "$test_tag Test can uninstall the operator" { # Assume the operator is installed, otherwise fail.