From b5cdc57ae3f215db592073312f379678ab499dae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Harry=20M=C3=ADchal?= Date: Wed, 7 Aug 2019 11:48:27 +0200 Subject: [PATCH] Add system test scripts These tests are written using BATS (Bash Automated Testing System). I used a very helpful helpers.bash script from the libpod project (Thank you!) that I tweaked slightly. https://github.com/containers/toolbox/issues/68 --- test/system/001-basics.bats | 26 +++ test/system/101-create.bats | 20 +++ test/system/102-list.bats | 44 +++++ test/system/103-remove.bats | 56 +++++++ test/system/104-run.bats | 12 ++ test/system/helpers.bash | 320 ++++++++++++++++++++++++++++++++++++ 6 files changed, 478 insertions(+) create mode 100644 test/system/001-basics.bats create mode 100644 test/system/101-create.bats create mode 100644 test/system/102-list.bats create mode 100644 test/system/103-remove.bats create mode 100644 test/system/104-run.bats create mode 100644 test/system/helpers.bash diff --git a/test/system/001-basics.bats b/test/system/001-basics.bats new file mode 100644 index 000000000..8c3417d77 --- /dev/null +++ b/test/system/001-basics.bats @@ -0,0 +1,26 @@ +#!/usr/bin/env bats + +load helpers + +function setup() { + : +} + +function teardown() { + : +} + +@test "Output version number using full flag" { + skip "Not implemented" + run_toolbox --version +} + +@test "Output version number using command" { + skip "Not implemented" + run_toolbox version +} + +@test "Show usage screen when no command is given" { + run_toolbox 1 + is "${lines[0]}" "toolbox: missing command" "Usage line 1" +} diff --git a/test/system/101-create.bats b/test/system/101-create.bats new file mode 100644 index 000000000..bced2303b --- /dev/null +++ b/test/system/101-create.bats @@ -0,0 +1,20 @@ +#!/usr/bin/env bats + +load helpers + +@test "Create the default container." { + run_toolbox -y create +} + +@test "Create a container with a valid custom name (whole word)" { + run_toolbox -y create -c "customname" +} + +@test "Try to create a container with a bad custom name (with special characters)" { + run_toolbox 1 -y create -c "ßpeci@l.Nam€" + is "${lines[0]}" "toolbox: invalid argument for '--container'" "Toolbox reports invalid argument for --container" +} + +@test "Create a container with a custom image (f29)" { + run_toolbox -y create -i fedora-toolbox:29 +} diff --git a/test/system/102-list.bats b/test/system/102-list.bats new file mode 100644 index 000000000..c587039b9 --- /dev/null +++ b/test/system/102-list.bats @@ -0,0 +1,44 @@ +#!/usr/bin/env bats + +load helpers + +@test "Run list with zero containers and zero images" { + remove_all_images + remove_all_containers + run_toolbox list + is "$output" "" "Output of list should be blank" +} + +@test "Run list with zero containers (-c flag)" { + remove_all_containers + run_toolbox list -c + is "$output" "" "Output of list should be blank" +} + +@test "Run list with zero images (-i flag)" { + remove_all_images + run_toolbox list -i + is "$output" "" "Output of list should be blank" +} + +@test "Run list with 1 default container and 1 default image" { + create_toolbox + run_toolbox list + is "${lines[1]}" ".*registry.fedoraproject.org/.*" "Default image" + is "${lines[3]}" ".*fedora-toolbox-.*" "Default container" + is "${#lines[@]}" "4" "Expected length of output is 4" +} + +@test "Run list with 3 containers (-c flag)" { + create_toolbox 3 fedora + run_toolbox list -c + for i in $(seq 1 3); do + is "${lines[$i]}" ".*fedora-$((i)) \+" "One of the containers" + done +} + +@test "Run list with 3 images (-i flag)" { + get_images 3 + run_toolbox list -i + is "${#lines[@]}" "4" "Expected length of output is 4" +} diff --git a/test/system/103-remove.bats b/test/system/103-remove.bats new file mode 100644 index 000000000..c31b2a209 --- /dev/null +++ b/test/system/103-remove.bats @@ -0,0 +1,56 @@ +#!/usr/bin/env bats + +load helpers + +@test "Remove a specific container (called fedora-2)" { + create_toolbox 2 fedora + run_toolbox rm fedora-2 + is "$output" "" "Successfull removal shouldn't print anything" +} + +@test "Remove a specific image (default image called by name)" { + get_images 1 + run_toolbox rmi "$TOOLBOX_DEFAULT_IMAGE" +} + +@test "Try to remove a nonexistent container" { + local todelete="nonexistentcontainer" + run_toolbox 1 rm "$todelete" + is "$output" "toolbox: failed to inspect $todelete" "Toolbox should fail with: no such container" +} + +@test "Try to remove a nonexistent image" { + local todelete="nonexistentimage" + run_toolbox 1 rmi "$todelete" +} + +@test "Try to remove a running container (called fedora-1)" { + create_toolbox 1 fedora + run_toolbox run -c fedora-1 echo "WAKE UP" + run_toolbox 1 rm fedora-1 + is "$output" "toolbox: failed to remove container fedora-1" "Toolbox should fail to remove the container" +} + +@test "Remove all containers (2 present)" { + create_toolbox 2 fedora + run_toolbox rm --all + is "$output" "" "" +} + +@test "Remove all images" { + get_images 2 + run_toolbox rmi --all +} + +@test "Try to remove all containers (running containers)" { + create_toolbox 2 fedora + run_toolbox run -c fedora-1 echo "WAKE UP" + run_toolbox run -c fedora-2 echo "WAKE UP" + run_toolbox 1 rm --all +} + +@test "Try to remove all images with present containers" { + get_images 2 + create_toolbox 2 fedora + run_toolbox 1 rmi --all +} diff --git a/test/system/104-run.bats b/test/system/104-run.bats new file mode 100644 index 000000000..ba5dad390 --- /dev/null +++ b/test/system/104-run.bats @@ -0,0 +1,12 @@ +#!/usr/bin/env bats + +load helpers + +function setup() { + setup_with_one_container +} + +@test "Echo 'Hello World' inside of an container" { + run_toolbox run echo "Hello World" + is "$output" "Hello World" "Should say 'Hello World'" +} diff --git a/test/system/helpers.bash b/test/system/helpers.bash new file mode 100644 index 000000000..175b1c6a8 --- /dev/null +++ b/test/system/helpers.bash @@ -0,0 +1,320 @@ +#!/usr/bin/env bash + +# Podman and Toolbox commands to run +PODMAN=${PODMAN:-podman} +TOOLBOX=${TOOLBOX:-toolbox} + +# Helpful globals +LATEST_FEDORA_VERSION=${LATEST_FEDORA_VERSION:-"32"} +DEFAULT_FEDORA_VERSION=${DEFAULT_FEDORA_VERSION:-"f31"} +REGISTRY_URL=${REGISTRY_URL:-"registry.fedoraproject.org"} +TOOLBOX_DEFAULT_IMAGE=${TOOLBOX_DEFAULT_IMAGE:-"registry.fedoraproject.org/f31/fedora-toolbox:31"} +TOOLBOX_TIMEOUT=${TOOLBOX_TIMEOUT:-100} +PODMAN_TIMEOUT=${PODMAN_TIMEOUT:-100} + +# Colors +LGC='\033[1;32m' # Light Green Color +LBC='\033[1;34m' # Light Blue Color +NC='\033[0m' # No Color + +# Basic setup +function basic_setup() { + echo "# [basic_setup]" >&2 + # Make sure desired images are present + if [ -z "$found_needed_image" ]; then + run_podman pull "$TOOLBOX_DEFAULT_IMAGE" + fi +} + +function setup_with_one_container() { + echo "# [setup_with_one_container]" >&2 + # Clean up all images except for the default one + remove_all_images_but_default + # Create a new (default) container if no other are present + run_toolbox -y create +} + +function basic_teardown() { + echo "# [basic_teardown]" >&2 + # Clean up all containers + remove_all_containers + # Clean up all images except for the default one + remove_all_images_but_default +} + +# Set the default setup function +function setup() { + basic_setup +} + +function teardown() { + basic_teardown +} + + +################ +# run_podman # Invoke $PODMAN, with timeout, using BATS 'run' +################ +# +# This is the preferred mechanism for invoking podman: first, it +# invokes $PODMAN, which may be 'podman-remote' or '/some/path/podman'. +# +# Second, we log the command run and its output. This doesn't normally +# appear in BATS output, but it will if there's an error. +# +# Next, we check exit status. Since the normal desired code is 0, +# that's the default; but the first argument can override: +# +# run_podman 125 nonexistent-subcommand +# run_podman '?' some-other-command # let our caller check status +# +# Since we use the BATS 'run' mechanism, $output and $status will be +# defined for our caller. +# +function run_podman() { + # Number as first argument = expected exit code; default 0 + expected_rc=0 + case "$1" in + [0-9]) expected_rc=$1; shift;; + [1-9][0-9]) expected_rc=$1; shift;; + [12][0-9][0-9]) expected_rc=$1; shift;; + '?') expected_rc= ; shift;; # ignore exit code + esac + + # stdout is only emitted upon error; this echo is to help a debugger + echo "\$ $PODMAN $*" + run timeout --foreground -v --kill=10 $PODMAN_TIMEOUT $PODMAN "$@" 3>/dev/null + # without "quotes", multiple lines are glommed together into one + if [ -n "$output" ]; then + echo "$output" + fi + if [ "$status" -ne 0 ]; then + echo -n "[ rc=$status "; + if [ -n "$expected_rc" ]; then + if [ "$status" -eq "$expected_rc" ]; then + echo -n "(expected) "; + else + echo -n "(** EXPECTED $expected_rc **) "; + fi + fi + echo "]" + fi + + if [ -n "$expected_rc" ]; then + if [ "$status" -ne "$expected_rc" ]; then + die "exit code is $status; expected $expected_rc" + fi + fi +} + +function run_toolbox() { + # Number as first argument = expected exit code; default 0 + expected_rc=0 + case "$1" in + [0-9]) expected_rc=$1; shift;; + [1-9][0-9]) expected_rc=$1; shift;; + [12][0-9][0-9]) expected_rc=$1; shift;; + '?') expected_rc= ; shift;; # ignore exit code + esac + + # stdout is only emitted upon error; this echo is to help a debugger + echo "\$ $TOOLBOX $*" + run timeout --foreground -v --kill=10 $TOOLBOX_TIMEOUT $TOOLBOX "$@" 3>/dev/null + # without "quotes", multiple lines are glommed together into one + if [ -n "$output" ]; then + echo "$output" + fi + if [ "$status" -ne 0 ]; then + echo -n "[ rc=$status "; + if [ -n "$expected_rc" ]; then + if [ "$status" -eq "$expected_rc" ]; then + echo -n "(expected) "; + else + echo -n "(** EXPECTED $expected_rc **) "; + fi + fi + echo "]" + fi + + if [ -n "$expected_rc" ]; then + if [ "$status" -ne "$expected_rc" ]; then + die "exit code is $status; expected $expected_rc" + fi + fi +} + +# Functions to prepare environment + + +function create_toolbox() { + echo "# [create_toolbox]" + + local numberof="$1" + local naming="$2" + local image="$3" + + if [ "$numberof" = "" ]; then + numberof=${numberof:-1} + fi + + if [ "$image" = "" ]; then + image=$TOOLBOX_DEFAULT_IMAGE + fi + + for i in $(seq "$numberof"); do + if [ "$naming" = "" ]; then + run_toolbox '?' -y create -i "$image" + else + run_toolbox '?' -y create -c "$naming-$i" -i "$image" + fi + done +} + + +function get_images() { + echo "# [get_images]" + + local numberof="$1" + local image="" + + if [ "$numberof" = "" ]; then + numberof=${numberof:-1} + fi + + for i in $(seq $numberof); do + local version=$[$LATEST_FEDORA_VERSION-$i] + image="$REGISTRY_URL/f$version/fedora-toolbox:$version" + run_podman pull "$image" || echo "Podman couldn't pull the image." + done +} + + +function remove_all_images() { + echo "# [remove_all_images]" + run_podman images --all --format '{{.Repository}}:{{.Tag}} {{.ID}}' + for line in "${lines[@]}"; do + set $line + run_podman rmi --force "$1" >/dev/null 2>&1 || true + run_podman rmi --force "$2" >/dev/null 2>&1 || true + done +} + +function remove_all_images_but_default() { + echo "# [remove_all_images_but_default]" + found_needed_image=1 + run_podman images --all --format '{{.Repository}}:{{.Tag}} {{.ID}}' + for line in "${lines[@]}"; do + set $line + if [ "$1" == "$TOOLBOX_DEFAULT_IMAGE" ]; then + found_needed_image=1 + else + run_podman rmi --force "$1" >/dev/null 2>&1 || true + fi + done +} + + +function get_image_name() { + echo "# [get_image_name]" + local type="$1" + local version="$2" + + if [ -z "$type" ]; then + type=${type:-fedora} + fi + + if [ -z "$version" ]; then + version=${version:-$DEFAULT_FEDORA_VERSION} + fi + + case "$type" in + fedora) + echo "$REGISTRY_URL/f$version/fedora-toolbox:$version" + ;; + esac +} + +function remove_all_containers() { + echo "# [remove_all_containers]" + run_toolbox '?' rm --all --force +} + + +# BATS specific functions + +######### +# die # Abort with helpful message +######### +function die() { + echo "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv" >&2 + echo "#| FAIL: $*" >&2 + echo "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" >&2 + false +} + + +######## +# is # Compare actual vs expected string; fail w/diagnostic if mismatch +######## +# +# Compares given string against expectations, using 'expr' to allow patterns. +# +# Examples:failed to inspect +# +# is "$actual" "$expected" "descriptive test name" +# is "apple" "orange" "name of a test that will fail in most universes" +# is "apple" "[a-z]\+" "this time it should pass" +# +function is() { + local actual="$1" + local expect="$2" + local testname="${3:-FIXME}" + + if [ -z "$expect" ]; then + if [ -z "$actual" ]; then + return + fi + expect='[no output]' + elif expr "$actual" : "$expect" >/dev/null; then + return + fi + + # This is a multi-line message, which may in turn contain multi-line + # output, so let's format it ourself, readably + local -a actual_split + readarray -t actual_split <<<"$actual" + printf "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" >&2 + printf "#| FAIL: $testname\n" >&2 + printf "#| expected: '%s'\n" "$expect" >&2 + printf "#| actual: '%s'\n" "${actual_split[0]}" >&2 + local line + for line in "${actual_split[@]:1}"; do + printf "#| > '%s'\n" "$line" >&2 + done + printf "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n" >&2 + false +} + +################# +# parse_table # Split a table on '|' delimiters; return space-separated +################# +# +# See sample .bats scripts for examples. The idea is to list a set of +# tests in a table, then use simple logic to iterate over each test. +# Columns are separated using '|' (pipe character) because sometimes +# we need spaces in our fields. +# +function parse_table() { + while read line; do + test -z "$line" && continue + + declare -a row=() + while read col; do + dprint "col=<<$col>>" + row+=("$col") + done < <(echo "$line" | tr '|' '\012' | sed -e 's/^ *//' -e 's/\\/\\\\/g') + + printf "%q " "${row[@]}" + printf "\n" + done <<<"$1" +}