From aa5c4879b432a95333cb73b2816b97f7e0ebff4f Mon Sep 17 00:00:00 2001 From: Ed Santiago Date: Tue, 19 Mar 2024 10:02:43 -0600 Subject: [PATCH] minikube: instrument tests, to allow debugging failures New run_minikube() helper, modeled after run_podman(). Echoes each command being run and its output. On failure, runs minikube logs. Addresses (does not close) #21931 which is hitting us hard in CI. Probably quay flakes, but it's impossible to tell without logs. Also: bug fix: one "run podman" fixed to run_podman Signed-off-by: Ed Santiago --- test/minikube/001-kube.bats | 113 ++++++++++++------------------------ test/minikube/helpers.bash | 58 ++++++++++++++++-- 2 files changed, 91 insertions(+), 80 deletions(-) diff --git a/test/minikube/001-kube.bats b/test/minikube/001-kube.bats index 04137ddbad..2f3ffeec8b 100755 --- a/test/minikube/001-kube.bats +++ b/test/minikube/001-kube.bats @@ -9,11 +9,9 @@ load helpers.bash # BEGIN tests @test "minikube - check cluster is up" { - run minikube kubectl get nodes - assert "$status" -eq 0 "get status of nodes" + run_minikube kubectl get nodes assert "$output" =~ "Ready" - run minikube kubectl get pods - assert "$status" -eq 0 "get pods in the default namespace" + run_minikube kubectl get pods assert "$output" == "No resources found in default namespace." } @@ -25,14 +23,11 @@ load helpers.bash # deploy to the minikube cluster project="ctr-ns" - run minikube kubectl create namespace $project - assert "$status" -eq 0 "create new namespace $project" - run minikube kubectl -- apply -f $fname - assert "$status" -eq 0 "deploy $fname to the cluster" + run_minikube kubectl create namespace $project + run_minikube kubectl -- apply -f $fname assert "$output" == "pod/$cname-pod created" wait_for_pods_to_start - run minikube kubectl delete namespace $project - assert $status -eq 0 "delete namespace $project" + run_minikube kubectl delete namespace $project } @test "minikube - deploy generated pod yaml to minikube" { @@ -48,14 +43,11 @@ load helpers.bash # deploy to the minikube cluster project="pod-ns" - run minikube kubectl create namespace $project - assert "$status" -eq 0 "create new namespace $project" - run minikube kubectl -- apply -f $fname - assert "$status" -eq 0 "deploy $fname to the cluster" + run_minikube kubectl create namespace $project + run_minikube kubectl -- apply -f $fname assert "$output" == "pod/$pname created" wait_for_pods_to_start - run minikube kubectl delete namespace $project - assert $status -eq 0 "delete namespace $project" + run_minikube kubectl delete namespace $project } @test "minikube - apply podman ctr to cluster" { @@ -64,35 +56,29 @@ load helpers.bash # deploy to minikube cluster with kube apply project="ctr-apply" - run minikube kubectl create namespace $project - assert "$status" -eq 0 "create new namespace $project" + run_minikube kubectl create namespace $project run_podman kube apply --kubeconfig $KUBECONFIG --ns $project $cname assert "$output" =~ "Successfully deployed workloads to cluster!" - run minikube kubectl -- get pods --namespace $project - assert "$status" -eq 0 "kube apply $cname to the cluster" + run_minikube kubectl -- get pods --namespace $project assert "$output" =~ "$cname-pod" wait_for_pods_to_start - run minikube kubectl delete namespace $project - assert $status -eq 0 "delete namespace $project" + run_minikube kubectl delete namespace $project } @test "minikube - apply podman pod to cluster" { pname="test-pod-apply" run_podman pod create --name $pname - run podman container create --pod $pname $IMAGE top + run_podman container create --pod $pname $IMAGE top # deploy to minikube cluster with kube apply project="pod-apply" - run minikube kubectl create namespace $project - assert "$status" -eq 0 "create new namespace $project" + run_minikube kubectl create namespace $project run_podman kube apply --kubeconfig $KUBECONFIG --ns $project $pname assert "$output" =~ "Successfully deployed workloads to cluster!" - run minikube kubectl -- get pods --namespace $project - assert "$status" -eq 0 "kube apply $pname to the cluster" + run_minikube kubectl -- get pods --namespace $project assert "$output" =~ "$pname" wait_for_pods_to_start - run minikube kubectl delete namespace $project - assert $status -eq 0 "delete namespace $project" + run_minikube kubectl delete namespace $project } @test "minikube - deploy generated kube yaml with podman kube apply to cluster" { @@ -108,16 +94,13 @@ load helpers.bash # deploy to minikube cluster with kube apply project="yaml-apply" - run minikube kubectl create namespace $project - assert "$status" -eq 0 "create new namespace $project" + run_minikube kubectl create namespace $project run_podman kube apply --kubeconfig $KUBECONFIG --ns $project -f $fname assert "$output" =~ "Successfully deployed workloads to cluster!" - run minikube kubectl -- get pods --namespace $project - assert "$status" -eq 0 "kube apply $pname to the cluster" + run_minikube kubectl -- get pods --namespace $project assert "$output" =~ "$pname" wait_for_pods_to_start - run minikube kubectl delete namespace $project - assert $status -eq 0 "delete namespace $project" + run_minikube kubectl delete namespace $project } @test "minikube - apply podman ctr with volume to cluster" { @@ -127,19 +110,15 @@ load helpers.bash # deploy to minikube cluster with kube apply project="ctr-vol-apply" - run minikube kubectl create namespace $project - assert "$status" -eq 0 "create new namespace $project" + run_minikube kubectl create namespace $project run_podman kube apply --kubeconfig $KUBECONFIG --ns $project $cname $vname assert "$output" =~ "Successfully deployed workloads to cluster!" - run minikube kubectl -- get pods --namespace $project - assert "$status" -eq 0 "kube apply $cname to the cluster" + run_minikube kubectl -- get pods --namespace $project assert "$output" =~ "$cname-pod" - run minikube kubectl -- get pvc --namespace $project - assert "$status" -eq 0 "kube apply $vname to the cluster" + run_minikube kubectl -- get pvc --namespace $project assert "$output" =~ "$vname" wait_for_pods_to_start - run minikube kubectl delete namespace $project - assert $status -eq 0 "delete namespace $project" + run_minikube kubectl delete namespace $project } @test "minikube - apply podman ctr with service to cluster" { @@ -148,19 +127,15 @@ load helpers.bash # deploy to minikube cluster with kube apply project="ctr-svc-apply" - run minikube kubectl create namespace $project - assert "$status" -eq 0 "create new namespace $project" + run_minikube kubectl create namespace $project run_podman kube apply --kubeconfig $KUBECONFIG -s --ns $project $cname assert "$output" =~ "Successfully deployed workloads to cluster!" - run minikube kubectl -- get pods --namespace $project - assert "$status" -eq 0 "kube apply $cname to the cluster" + run_minikube kubectl -- get pods --namespace $project assert "$output" =~ "$cname-pod" - run minikube kubectl -- get svc --namespace $project - assert "$status" -eq 0 "kube apply service to the cluster" + run_minikube kubectl -- get svc --namespace $project assert "$output" =~ "$cname-pod" wait_for_pods_to_start - run minikube kubectl delete namespace $project - assert $status -eq 0 "delete namespace $project" + run_minikube kubectl delete namespace $project } @test "minikube - deploy generated container yaml to minikube --type=deployment" { @@ -171,14 +146,11 @@ load helpers.bash # deploy to the minikube cluster project="dep-ctr-ns" - run minikube kubectl create namespace $project - assert "$status" -eq 0 "create new namespace $project" - run minikube kubectl -- apply -f $fname - assert "$status" -eq 0 "deploy $fname to the cluster" + run_minikube kubectl create namespace $project + run_minikube kubectl -- apply -f $fname assert "$output" == "deployment.apps/$cname-pod-deployment created" wait_for_pods_to_start - run minikube kubectl delete namespace $project - assert $status -eq 0 "delete namespace $project" + run_minikube kubectl delete namespace $project } @test "minikube - deploy generated pod yaml to minikube --type=deployment" { @@ -194,14 +166,11 @@ load helpers.bash # deploy to the minikube cluster project="dep-pod-ns" - run minikube kubectl create namespace $project - assert "$status" -eq 0 "create new namespace $project" - run minikube kubectl -- apply -f $fname - assert "$status" -eq 0 "deploy $fname to the cluster" + run_minikube kubectl create namespace $project + run_minikube kubectl -- apply -f $fname assert "$output" == "deployment.apps/$pname-deployment created" wait_for_pods_to_start - run minikube kubectl delete namespace $project - assert $status -eq 0 "delete namespace $project" + run_minikube kubectl delete namespace $project } @test "minikube - deploy generated container yaml to minikube --type=daemonset" { @@ -212,14 +181,11 @@ load helpers.bash # deploy to the minikube cluster project="dep-ctr-ns" - run minikube kubectl create namespace $project - assert "$status" -eq 0 "create new namespace $project" - run minikube kubectl -- apply -f $fname - assert "$status" -eq 0 "deploy $fname to the cluster" + run_minikube kubectl create namespace $project + run_minikube kubectl -- apply -f $fname assert "$output" == "daemonset.apps/$cname-pod-daemonset created" wait_for_pods_to_start - run minikube kubectl delete namespace $project - assert $status -eq 0 "delete namespace $project" + run_minikube kubectl delete namespace $project } @test "minikube - deploy generated pod yaml to minikube --type=daemonset" { @@ -235,12 +201,9 @@ load helpers.bash # deploy to the minikube cluster project="dep-pod-ns" - run minikube kubectl create namespace $project - assert "$status" -eq 0 "create new namespace $project" - run minikube kubectl -- apply -f $fname - assert "$status" -eq 0 "deploy $fname to the cluster" + run_minikube kubectl create namespace $project + run_minikube kubectl -- apply -f $fname assert "$output" == "daemonset.apps/$pname-daemonset created" wait_for_pods_to_start - run minikube kubectl delete namespace $project - assert $status -eq 0 "delete namespace $project" + run_minikube kubectl delete namespace $project } diff --git a/test/minikube/helpers.bash b/test/minikube/helpers.bash index c47b711e3d..24e394647c 100644 --- a/test/minikube/helpers.bash +++ b/test/minikube/helpers.bash @@ -4,10 +4,55 @@ load ../system/helpers.bash KUBECONFIG="$HOME/.kube/config" +################## +# run_minikube # Local helper, with instrumentation for debugging failures +################## +function run_minikube() { + # Number as first argument = expected exit code; default 0 + local expected_rc=0 + case "$1" in + [0-9]) expected_rc=$1; shift;; + [1-9][0-9]) expected_rc=$1; shift;; + [12][0-9][0-9]) expected_rc=$1; shift;; + '?') expected_rc= ; shift;; # ignore exit code + esac + + # stdout is only emitted upon error; this printf is to help in debugging + printf "\n%s %s %s %s\n" "$(timestamp)" "\$" "minikube" "$*" + run minikube "$@" + # without "quotes", multiple lines are glommed together into one + if [[ -n "$output" ]]; then + echo "$(timestamp) $output" + fi + if [[ "$status" -ne 0 ]]; then + echo -n "$(timestamp) [ rc=$status "; + if [[ -n "$expected_rc" ]]; then + if [[ "$status" -eq "$expected_rc" ]]; then + echo -n "(expected) "; + else + echo -n "(** EXPECTED $expected_rc **) "; + fi + fi + echo "]" + fi + + if [[ -n "$expected_rc" ]]; then + if [[ "$status" -ne "$expected_rc" ]]; then + # Further debugging + echo "\$ minikube logs" + run minikube logs + echo "$output" + + die "exit code is $status; expected $expected_rc" + fi + fi +} + + function setup(){ # only set up the minikube cluster before the first test if [[ "$BATS_TEST_NUMBER" -eq 1 ]]; then - minikube start + run_minikube start wait_for_default_sa fi basic_setup @@ -17,8 +62,12 @@ function teardown(){ # only delete the minikube cluster if we are done with the last test # the $DEBUG_MINIKUBE env can be set to preserve the cluster to debug if needed if [[ "$BATS_TEST_NUMBER" -eq ${#BATS_TEST_NAMES[@]} ]] && [[ "$DEBUG_MINIKUBE" == "" ]]; then - minikube delete + run_minikube delete fi + + # Prevents nasty red warnings in log + run_podman rmi --ignore $(pause_image) + basic_teardown } @@ -29,8 +78,7 @@ function wait_for_default_sa(){ # if the default service account hasn't been created yet, there is something else wrong while [[ $count -lt 30 ]] && [[ $sa_ready == false ]] do - run minikube kubectl get sa - assert "$status" -eq 0 + run_minikube kubectl get sa if [[ "$output" != "No resources found in default namespace." ]]; then sa_ready=true fi @@ -49,7 +97,7 @@ function wait_for_pods_to_start(){ # if the pod hasn't started running after 30 seconds, there is something else wrong while [[ $count -lt 30 ]] && [[ $running == false ]] do - run minikube kubectl get pods + run_minikube kubectl get pods assert "$status" -eq 0 if [[ "$output" =~ "Running" ]]; then running=true