From a9d2204a54806deef3b11f41af584241dd5afeb8 Mon Sep 17 00:00:00 2001 From: Alberto Faria Date: Fri, 12 Apr 2024 23:19:21 +0100 Subject: [PATCH] tests: Switch to a VM-based test harness This allows us to further control the test environment and write more involved tests. Signed-off-by: Alberto Faria --- .config/nextest.toml | 2 - Makefile | 4 +- plans/main.fmf | 1 - test.sh | 53 ------ tests/env.sh | 431 ++++++++++++++++++++++++++++++++++++++++++ tests/run.rs | 164 ---------------- tests/t/cloud-init.sh | 30 +++ tests/t/hostname.sh | 47 +++++ tests/t/ignition.sh | 39 ++++ tests/t/mount.sh | 47 +++++ tests/t/persistent.sh | 27 +++ tests/t/stop-start.sh | 14 ++ 12 files changed, 638 insertions(+), 221 deletions(-) delete mode 100644 .config/nextest.toml delete mode 100755 test.sh create mode 100755 tests/env.sh delete mode 100644 tests/run.rs create mode 100644 tests/t/cloud-init.sh create mode 100644 tests/t/hostname.sh create mode 100644 tests/t/ignition.sh create mode 100644 tests/t/mount.sh create mode 100644 tests/t/persistent.sh create mode 100644 tests/t/stop-start.sh diff --git a/.config/nextest.toml b/.config/nextest.toml deleted file mode 100644 index 3ca5346..0000000 --- a/.config/nextest.toml +++ /dev/null @@ -1,2 +0,0 @@ -[profile.default] -slow-timeout = { period = "60s", terminate-after = 2 } diff --git a/Makefile b/Makefile index 41ead85..e96a603 100644 --- a/Makefile +++ b/Makefile @@ -67,7 +67,9 @@ uninstall: #.PHONY: unit unit: $(CARGO_TARGET_DIR) - $(SHELL) test.sh podman + $(SHELL) tests/env.sh build + $(SHELL) tests/env.sh start + $(SHELL) tests/env.sh run all all #.PHONY: code_coverage code_coverage: $(CARGO_TARGET_DIR) diff --git a/plans/main.fmf b/plans/main.fmf index ab77ccd..099194a 100644 --- a/plans/main.fmf +++ b/plans/main.fmf @@ -13,7 +13,6 @@ prepare: execute: how: tmt script: | - cargo install nextest PATH=$PATH:/root/.cargo/bin make unit /validate_test: diff --git a/test.sh b/test.sh deleted file mode 100755 index 38b5041..0000000 --- a/test.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: GPL-2.0-or-later - -set -o errexit -o pipefail -o nounset - -if (( $# == 0 )); then - >&2 echo -n "\ -Usage: $0 - -Examples: - $ $0 podman # rootless Podman - $ sudo $0 podman # rootful Podman - $ $0 docker # Docker - $ $0 podman docker # rootless Podman and Docker -" - exit 2 -fi - -function __log_and_run() { - printf '\033[0;33m%s\033[0m\n' "$*" - "$@" -} - -export CARGO_TERM_COLOR=always - -script_dir="$( dirname "$0" )" -if [[ "${script_dir}" != . ]]; then - __log_and_run cd "${script_dir}" -fi - -images=( - quay.io/containerdisks/fedora:39 - quay.io/crun-vm/example-fedora-coreos:39 -) - -# ensure that tests don't timeout because they're pulling images -for engine in "$@"; do - for image in "${images[@]}"; do - __log_and_run "${engine}" pull "${image}" - done -done - -nextest_run=( - nextest run \ - --all-targets --all-features --test-threads $(( $(nproc) / 2 )) \ - -- "${@/#/test_run::engine_}" - ) - -if command -v cargo-nextest &> /dev/null; then - __log_and_run cargo "${nextest_run[@]}" -else - __log_and_run "${nextest_run[@]}" -fi diff --git a/tests/env.sh b/tests/env.sh new file mode 100755 index 0000000..766853e --- /dev/null +++ b/tests/env.sh @@ -0,0 +1,431 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0-or-later + +set -o errexit -o pipefail -o nounset + +start_time="$( date +%s.%N )" + +env_image_base=quay.io/containerdisks/fedora:39 +env_image=quay.io/crun-vm/test-env:latest +container_name=crun-vm-test-env + +declare -A TEST_IMAGES +TEST_IMAGES=( + [fedora]=quay.io/containerdisks/fedora:39 # uses cloud-init + [coreos]=quay.io/crun-vm/example-fedora-coreos:39 # uses Ignition +) + +declare -A TEST_IMAGES_DEFAULT_USER +TEST_IMAGES_DEFAULT_USER=( + [fedora]=fedora + [coreos]=core +) + +declare -A TEST_IMAGES_DEFAULT_USER_HOME +TEST_IMAGES_DEFAULT_USER_HOME=( + [fedora]=/home/fedora + [coreos]=/var/home/core +) + +__bad_usage() { + >&2 echo -n "\ +Usage: $0 [] + +build + $0 run + $0 start + $0 stop + +COMMANDS + + build + Build the test env VM container image. + + start + Start the test env VM. + + restart + Stop the test env VM if running, then start it. + + stop + Stop the test env VM if running. + + run + run all + Run a test script in the test env VM under the given engine. must + be one of 'docker', 'podman', 'rootful-podman', or 'all'. + + ssh + SSH into the test env VM for debugging. +" + exit 2 +} + +# Usage: __elapsed +__elapsed() { + bc -l <<< "$( date +%s.%N ) - ${start_time}" +} + +# Usage: __small_log_without_time +__small_log_without_time() { + # shellcheck disable=SC2059 + printf "\033[%sm--- %s\033[0m\n" \ + "$1" "$( printf "${@:2}" )" +} + +# Usage: __log +__small_log() { + # shellcheck disable=SC2059 + printf "\033[%sm--- [%6.1f] %s\033[0m\n" \ + "$1" "$( __elapsed )" "$( printf "${@:2}" )" +} + +# Usage: __big_log +__big_log() { + local text term_cols sep_len + text="$( printf "${@:2}" )" + term_cols="$( tput cols 2> /dev/null )" || term_cols=80 + sep_len="$(( term_cols - ${#text} - 16 ))" + printf "\033[%sm--- [%6.1f] %s " "$1" "$( __elapsed )" "${text}" + printf '%*s\033[0m\n' "$(( sep_len < 0 ? 0 : sep_len ))" '' | tr ' ' - +} + +__log_without_time_and_run() { + __small_log_without_time 36 '$ %s' "$*" + "$@" +} + +__log_and_run() { + __small_log 36 '$ %s' "$*" + "$@" +} + +__rel() { + realpath -s --relative-to=. "$1" +} + +__build_runtime() { + __big_log 33 'Building crun-vm...' + __log_and_run cargo build --manifest-path "$( __rel "$repo_root/Cargo.toml" )" + runtime=$repo_root/target/debug/crun-vm +} + +__extra_cleanup() { :; } + +repo_root=$( readlink -e "$( dirname "$0" )/.." ) + +temp_dir=$( mktemp -d ) +trap '__extra_cleanup; rm -fr "$temp_dir"' EXIT + +case "${1:-}" in +build) + if (( $# != 1 )); then + __bad_usage + fi + + __build_runtime + + __big_log 33 'Building test env image...' + + # extract base image file + + __log_and_run "$( __rel "$repo_root/util/extract-vm-image.sh" )" \ + "$env_image_base" \ + "$temp_dir/image" + + # expand base image + + __log_and_run qemu-img create -f qcow2 "$temp_dir/resized-image.qcow2" 20G + __log_and_run virt-resize --expand /dev/sda5 "$temp_dir/image" "$temp_dir/resized-image.qcow2" + + rm "$temp_dir/image" + + # launch VM from base image file + + __log_and_run podman run \ + --name "$container_name-build" \ + --runtime "$runtime" \ + --memory 8g \ + --rm -dit \ + --rootfs "$temp_dir" \ + --persistent + + # shellcheck disable=SC2317 + __extra_cleanup() { + __log_and_run podman stop --time 0 "$container_name-build" + } + + # customize VM + + __exec() { + __log_and_run podman exec "$container_name-build" --as fedora "$@" + } + + # generate an ssh keypair for users fedora and root so crun-vm containers + # get a predictable keypair + __exec 'ssh-keygen -q -f .ssh/id_rsa -N "" && sudo cp -r .ssh /root/' + + __exec sudo dnf update -y + __exec sudo dnf install -y \ + bash \ + bc \ + coreutils \ + crun \ + docker \ + genisoimage \ + grep \ + htop \ + libselinux-devel \ + libvirt-client \ + libvirt-daemon-driver-qemu \ + libvirt-daemon-log \ + lsof \ + openssh-clients \ + podman \ + qemu-img \ + qemu-system-x86-core \ + shadow-utils \ + util-linux \ + virtiofsd + __exec sudo dnf clean all + + daemon_json='{ "runtimes": { "crun-vm": { "path": "/home/fedora/target/debug/crun-vm" } } }' + __exec sudo mkdir -p /etc/docker + __exec "echo ${daemon_json@Q} | sudo tee /etc/docker/daemon.json" + + __exec sudo cloud-init clean --logs # run cloud-init again on next boot + + __exec sudo poweroff || true + + # sparsify image file + + __log_and_run podman wait --ignore "$container_name-build" + __extra_cleanup() { :; } + + __log_and_run virt-sparsify "$temp_dir/resized-image.qcow2" "$temp_dir/final-image.qcow2" + + rm "$temp_dir/resized-image.qcow2" + + # package new image file + + __log_and_run "$( __rel "$repo_root/util/package-vm-image.sh" )" \ + "$temp_dir/final-image.qcow2" \ + "$env_image" + + __big_log 33 'Done.' + ;; + +start) + if (( $# != 1 )); then + __bad_usage + fi + + if podman container exists "$container_name"; then + >&2 echo "Already started." + exit 0 + fi + + __build_runtime + + # launch VM + + __log_and_run podman run \ + --name "$container_name" \ + --pull never \ + --runtime "$runtime" \ + --memory 8g \ + --rm -dit \ + -v "$temp_dir":/home/fedora/images:z \ + -v "$repo_root/target":/home/fedora/target:z \ + "$env_image" \ + "" + + # shellcheck disable=SC2317 + __extra_cleanup() { + __log_and_run podman stop --time 0 "$container_name" + } + + # load test images onto VM + + __exec() { + __log_and_run podman exec "$container_name" --as fedora "$@" + } + + chmod a+rx "$temp_dir" # so user "fedora" in guest can access it + + for image in "${TEST_IMAGES[@]}"; do + __log_and_run podman pull "$image" + __log_and_run podman save "$image" -o "$temp_dir/image.tar" + + __exec cp /home/fedora/images/image.tar image.tar + __exec sudo docker load -i image.tar + __exec podman load -i image.tar + __exec sudo podman load -i image.tar + __exec rm image.tar + + rm "$temp_dir/image.tar" + done + + __extra_cleanup() { :; } + ;; + +restart) + "$0" stop + "$0" start + ;; + +stop) + if (( $# != 1 )); then + __bad_usage + fi + + __log_and_run podman stop --ignore "$container_name" + ;; + +run) + if (( $# < 3 )); then + __bad_usage + fi + + case "$2" in + docker|podman|rootful-podman) + engines=( "$2" ) + ;; + all) + engines=( docker podman rootful-podman ) + ;; + *) + __bad_usage + ;; + esac + + if (( $# == 3 )) && [[ "$3" == all ]]; then + mapfile -d '' -t tests < <( find "$repo_root/tests/t" -type f -print0 ) + else + tests=( "${@:3}" ) + fi + + if ! podman container exists "$container_name"; then + >&2 echo "The test environment VM isn't running. Start it with:" + >&2 echo " \$ $0 start" + exit 1 + fi + + __build_runtime + + for t in "${tests[@]}"; do + for engine in "${engines[@]}"; do + + __big_log 33 'Running test %s under %s...' "$( __rel "$t" )" "$engine" + + case "$engine" in + docker) + engine_cmd=( sudo docker ) + runtime_in_env=crun-vm + ;; + podman) + engine_cmd=( podman ) + runtime_in_env=/home/fedora/target/debug/crun-vm + ;; + rootful-podman) + engine_cmd=( sudo podman ) + runtime_in_env=/home/fedora/target/debug/crun-vm + ;; + esac + + # generate random label for containers created by test script + label=$( mktemp --dry-run | xargs basename ) + + # shellcheck disable=SC2317 + __engine() { + if [[ "$1" == run ]]; then + __command=( + "${engine_cmd[@]}" run + --runtime "$runtime_in_env" + --pull never + --label "$label" + "${@:2}" + ) + __small_log 36 '$ %s' "${__command[*]}" + # shellcheck disable=SC2034 + LAST_RUN_ID=$( "${__command[@]}" ) + else + __log_and_run "${engine_cmd[@]}" "$@" + fi + } + + __exec() { + podman exec -i "$container_name" --as fedora "$@" + } + + # shellcheck disable=SC2088 + __exec mkdir "$label.temp" "$label.util" + + # copy util scripts + + for file in $repo_root/util/*; do + contents=$( cat "$file" ) + path_in_vm=$label.util/$( basename "$file" ) + __exec "echo ${contents@Q} > $path_in_vm && chmod +x $path_in_vm" + done + + # run test + + full_script="\ + set -o errexit -o pipefail -o nounset + $( + declare -p \ + TEST_IMAGES TEST_IMAGES_DEFAULT_USER TEST_IMAGES_DEFAULT_USER_HOME \ + engine_cmd runtime_in_env label start_time + ) + $( declare -f __elapsed __engine __log_and_run __small_log ) + __skip() { + exit 0 + } + TEMP_DIR=~/$label.temp + UTIL_DIR=~/$label.util + ENGINE=$engine + export RUST_BACKTRACE=${RUST_BACKTRACE:-1} + $( cat "$t" )\ + " + + exit_code=0 + __exec <<< "$full_script" || exit_code=$? + + # remove any leftover containers + + __small_log 36 'Cleaning up...' + + full_script="\ + set -o errexit -o pipefail -o nounset + ${engine_cmd[*]} ps --filter label=$label --format '{{.Names}}' --all | + xargs --no-run-if-empty ${engine_cmd[*]} rm --force + sudo rm -fr $label.temp $label.util + " + + __exec <<< "$full_script" + + # report test result + + if (( exit_code == 0 )); then + __small_log 36 'Test succeeded.' + else + __small_log 36 'Test failed.' + __big_log 31 'A test failed.' + exit "$exit_code" + fi + + done + done + + __big_log 32 'All tests succeeded.' + ;; + +ssh) + __log_and_run podman exec -it "$container_name" --as fedora "${@:2}" + ;; + +*) + __bad_usage + ;; +esac diff --git a/tests/run.rs b/tests/run.rs deleted file mode 100644 index 0a39278..0000000 --- a/tests/run.rs +++ /dev/null @@ -1,164 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later - -#![allow(clippy::items_after_test_module)] - -use std::env; -use std::io::{BufWriter, Write}; -use std::process::{Command, Stdio}; - -use anyhow::{anyhow, Result}; -use camino::Utf8Path; -use test_case::test_matrix; -use uuid::Uuid; - -fn simple_test_case(image: &str, home_dir: &str) -> TestCase { - TestCase { - run_args: vec![image.to_string(), "".to_string()], - exec_user: Utf8Path::new(home_dir).file_name().unwrap().to_string(), - test_script: "".to_string(), - } -} - -fn complex_test_case(image: &str, home_dir: &str) -> TestCase { - TestCase { - run_args: vec![ - "-h=my-test-vm".to_string(), - format!("-v=./util:{home_dir}/util"), - format!("-v=./README.md:{home_dir}/README.md:z,ro"), // "ro" is so qemu uses shared lock - format!("--mount=type=tmpfs,dst={home_dir}/tmp"), - image.to_string(), - format!("--cloud-init={REPO_PATH}/examples/cloud-init/config"), - format!("--ignition={REPO_PATH}/examples/ignition/config.ign"), - ], - exec_user: Utf8Path::new(home_dir).file_name().unwrap().to_string(), - test_script: format!( - " - mount -l | grep '^virtiofs-0 on {home_dir}/util type virtiofs' - mount -l | grep '^tmpfs on {home_dir}/tmp type tmpfs' - [[ -b ~/README.md ]] - sudo grep 'This project is released under' ~/README.md - " - ), - } -} - -#[test_matrix( - // engines - [ - Engine::Podman, - Engine::Docker, - ], - - // cases - [ - simple_test_case("quay.io/containerdisks/fedora:39", "/home/fedora"), - simple_test_case("quay.io/crun-vm/example-fedora-coreos:39", "/var/home/core"), - - complex_test_case("quay.io/containerdisks/fedora:39", "/home/fedora"), - complex_test_case("quay.io/crun-vm/example-fedora-coreos:39", "/var/home/core"), - ] -)] -fn test_run(engine: Engine, case: TestCase) { - env::set_var("RUST_BACKTRACE", "1"); - - let container_name = get_random_container_name(); - - // launch VM - - let status = engine - .command("run") - .arg(format!("--name={}", container_name)) - .arg("--rm") - .arg("--detach") - .args(&case.run_args) - .spawn() - .unwrap() - .wait() - .unwrap(); - assert!(status.success()); - - // run the test script - - let result = (|| -> Result<()> { - let mut exec_child = engine - .command("exec") - .arg("-i") - .arg(&container_name) - .arg("--as") - .arg(&case.exec_user) - .arg("bash") - .arg("-s") - .stdin(Stdio::piped()) - .spawn()?; - - { - let mut writer = BufWriter::new(exec_child.stdin.take().unwrap()); - writer.write_all("set -ex\n".as_bytes())?; - writer.write_all("! command -v cloud-init || cloud-init status --wait\n".as_bytes())?; - writer.write_all(case.test_script.as_bytes())?; - writer.write_all("\n".as_bytes())?; - writer.flush()?; - // stdin is closed when writer is dropped - } - - match exec_child.wait()?.code().unwrap() { - 0 => Ok(()), - n => Err(anyhow!("test script failed with exit code {n}")), - } - })(); - - // terminate the VM - - let status = engine - .command("stop") - .arg(&container_name) - .stdin(Stdio::null()) - .spawn() - .unwrap() - .wait() - .unwrap(); - assert!(status.success()); - - result.unwrap(); -} - -const BINARY_PATH: &str = env!("CARGO_BIN_EXE_crun-vm"); -const REPO_PATH: &str = env!("CARGO_MANIFEST_DIR"); - -struct TestCase { - run_args: Vec, - exec_user: String, - test_script: String, -} - -fn get_random_container_name() -> String { - format!("crun-vm-test-{}", Uuid::new_v4()) -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum Engine { - Podman, - Docker, -} - -impl Engine { - pub fn command(self, subcommand: &str) -> Command { - let engine = match self { - Engine::Podman => "podman", - Engine::Docker => "docker", - }; - - let mut cmd = Command::new(engine); - cmd.arg(subcommand); - - if subcommand == "run" { - match self { - Engine::Podman => cmd.arg(format!("--runtime={}", BINARY_PATH)), - Engine::Docker => cmd.arg("--runtime=crun-vm"), - }; - } - - cmd.env("RUST_BACKTRACE", "1"); - cmd - } -} diff --git a/tests/t/cloud-init.sh b/tests/t/cloud-init.sh new file mode 100644 index 0000000..6ea51dd --- /dev/null +++ b/tests/t/cloud-init.sh @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: GPL-2.0-or-later + +image="${TEST_IMAGES[fedora]}" +user="${TEST_IMAGES_DEFAULT_USER[fedora]}" +home="${TEST_IMAGES_DEFAULT_USER_HOME[fedora]}" + +cat >"$TEMP_DIR/user-data" <"$TEMP_DIR/meta-data" <"$TEMP_DIR/config.ign" < "$TEMP_DIR/file" + + __engine run \ + --rm --detach \ + --name "mount-$os" \ + --volume "$TEMP_DIR/file:$home/file:z" \ + --volume "$TEMP_DIR:$home/dir:z" \ + --mount "type=tmpfs,dst=$home/tmp" \ + "$image" \ + "" + + __test() { + __engine exec "mount-$os" --as "$user" + + __engine exec "mount-$os" --as "$user" " + set -e + [[ -b $home/file ]] + sudo cmp -n 6 $home/file <<< hello + " + + __engine exec "mount-$os" --as "$user" " + set -e + mount -l | grep '^virtiofs-0 on $home/dir type virtiofs' + [[ -d $home/dir ]] + sudo cmp $home/dir/file <<< hello + " + + __engine exec "mount-$os" --as "$user" " + mount -l | grep '^tmpfs on $home/tmp type tmpfs' + " + } + + __test + __engine restart "mount-$os" + __test + + __engine stop --time 0 "mount-$os" + +done diff --git a/tests/t/persistent.sh b/tests/t/persistent.sh new file mode 100644 index 0000000..3171290 --- /dev/null +++ b/tests/t/persistent.sh @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: GPL-2.0-or-later + +if [[ "$ENGINE" == docker ]]; then + # docker doesn't support --rootfs + __skip +fi + +"$UTIL_DIR/extract-vm-image.sh" "${TEST_IMAGES[fedora]}" "$TEMP_DIR/image" + +__test() { + id=$( __engine run --detach --name persistent --rootfs "$TEMP_DIR" "$1" ) + + __engine exec persistent --as fedora "$2" + + __engine stop persistent + __engine rm persistent + + if [[ "$ENGINE" != rootful-podman ]]; then + # ensure user that invoked `engine run` can delete crun-vm state + rm -r "$TEMP_DIR/crun-vm-$id" + fi +} + +__test "" '[[ ! -e i-was-here ]] && touch i-was-here' +__test --persistent '[[ ! -e i-was-here ]] && touch i-was-here' +__test --persistent '[[ -e i-was-here ]]' +__test "" '[[ -e i-was-here ]]' diff --git a/tests/t/stop-start.sh b/tests/t/stop-start.sh new file mode 100644 index 0000000..b4787b1 --- /dev/null +++ b/tests/t/stop-start.sh @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0-or-later + +__engine run --detach --name stop-start "${TEST_IMAGES[fedora]}" "" + +__engine exec stop-start --as fedora '[[ ! -e i-was-here ]] && touch i-was-here' + +for (( i = 0; i < 2; ++i )); do + + __engine stop stop-start + __engine start stop-start + + __engine exec stop-start --as fedora '[[ -e i-was-here ]]' + +done