Merge pull request #20797 from edsantiago/defer_assert_failures

[systests] new defer-assertion-failure

test/system/005-info.bats

@@ -23,6 +23,8 @@ runRoot:
 cgroupManager: \\\(systemd\\\|cgroupfs\\\)
 cgroupVersion: v[12]
 "
+    defer-assertion-failures
+
     while read expect; do
         is "$output" ".*$expect" "output includes '$expect'"
     done < <(parse_table "$expected_keys")
@@ -52,11 +54,13 @@ store.imageStore.number   | 1
 host.slirp4netns.executable | $expr_path
 "
 
-    parse_table "$tests" | while read field expect; do
+    defer-assertion-failures
+
+    while read field expect; do
         actual=$(echo "$output" | jq -r ".$field")
         dprint "# actual=<$actual> expect=<$expect>"
         is "$actual" "$expect" "jq .$field"
-    done
+    done < <(parse_table "$tests")
 }
 
 @test "podman info - confirm desired runtime" {

test/system/010-images.bats

@@ -26,10 +26,12 @@ load helpers
 {{.Labels.created_at}}   | 20[0-9-]\\\+T[0-9:]\\\+Z
 "
 
-    parse_table "$tests" | while read fmt expect; do
+    defer-assertion-failures
+
+    while read fmt expect; do
         run_podman images --format "$fmt"
         is "$output" "$expect" "podman images --format '$fmt'"
-    done
+    done < <(parse_table "$tests")
 
     run_podman images --format "{{.ID}}" --no-trunc
     is "$output" "sha256:[0-9a-f]\\{64\\}\$" "podman images --no-trunc"
@@ -49,12 +51,11 @@ Labels.created_at | 20[0-9-]\\\+T[0-9:]\\\+Z
 
     run_podman images -a --format json
 
-    parse_table "$tests" | while read field expect; do
+    while read field expect; do
         actual=$(echo "$output" | jq -r ".[0].$field")
         dprint "# actual=<$actual> expect=<$expect}>"
         is "$actual" "$expect" "jq .$field"
-    done
-
+    done < <(parse_table "$tests")
 }
 
 @test "podman images - history output" {

test/system/030-run.bats

@@ -21,6 +21,8 @@ echo $rand        |   0 | $rand
 /etc              | 126 | $err_no_exec_dir
 "
 
+    defer-assertion-failures
+
     tests_run=0
     while read cmd expected_rc expected_output; do
         if [ "$expected_output" = "''" ]; then expected_output=""; fi
@@ -389,6 +391,9 @@ journald  | -
 k8s-file  | y
 json-file | f
 "
+
+    defer-assertion-failures
+
     while read driver do_check; do
         msg=$(random_string 15)
         run_podman run --name myctr --log-driver $driver $IMAGE echo $msg
@@ -1285,6 +1290,8 @@ search               | $IMAGE           |
     bogus=$PODMAN_TMPDIR/bogus-authfile
     touch $PODMAN_TMPDIR/Containerfile
 
+    defer-assertion-failures
+
     while read command args local_only;do
         # skip commands that don't work in podman-remote
         if [[ "$local_only" = "-" ]]; then

test/system/065-cp.bats

@@ -47,6 +47,8 @@ load helpers
 0 | subdir               | /srv/subdir/hostfile0 | copy to workdir/subdir
 "
 
+    defer-assertion-failures
+
     # RUNNING container
     while read id dest dest_fullname description; do
         run_podman cp $srcdir/hostfile$id destrunning:$dest
@@ -192,6 +194,8 @@ load helpers
 2 | subdir/containerfile2 | /        | /containerfile2 | copy from workdir/subdir (rel path) to srcdir
 "
 
+    defer-assertion-failures
+
     # RUNNING container
     while read id src dest dest_fullname description; do
         # dest may be "''" for empty table cells
@@ -251,6 +255,8 @@ load helpers
 2 | subdir/containerfile2 | /        | /containerfile2 | copy from workdir/subdir (rel path) to /
 "
 
+    defer-assertion-failures
+
     # From RUNNING container
     local -a destcontainers=()
     while read id src dest dest_fullname description; do
@@ -342,6 +348,8 @@ load helpers
  dir/.     | /newdir3 | /newdir3/sub | copy dir/. to newdir3
 "
 
+    defer-assertion-failures
+
     # RUNNING container
     while read src dest dest_fullname description; do
         run_podman cp $srcdir/$src destrunning:$dest
@@ -400,6 +408,8 @@ load helpers
 /tmp/subdir.  |         | /subdir.       | copy /tmp/subdir.
 "
 
+    defer-assertion-failures
+
     # RUNNING container
     while read src dest dest_fullname description; do
         if [[ $dest == "''" ]];then
@@ -467,6 +477,8 @@ load helpers
 /tmp/subdir.  | /       | /subdir.       | copy /tmp/subdir.
 "
 
+    defer-assertion-failures
+
     # From RUNNING container
     local -a destcontainers=()
     while read src dest dest_fullname description; do

test/system/075-exec.bats

@@ -167,6 +167,9 @@ load helpers
 }
 
 @test "podman exec --tty" {
+    # Run all tests, report failures at end
+    defer-assertion-failures
+
     # Outer loops: different variations on the RUN container
     for run_opt_t in "" "-t"; do
         for run_term_env in "" "explicit_RUN_term"; do

test/system/110-history.bats

@@ -11,14 +11,16 @@ load helpers
 --no-trunc                       | .*[0-9a-f]\\\{64\\\}
 "
 
-    parse_table "$tests" | while read options expect; do
+    defer-assertion-failures
+
+    while read options expect; do
         if [ "$options" = "''" ]; then options=; fi
 
         eval set -- "$options"
 
         run_podman history "$@" $IMAGE
         is "$output" "$expect" "podman history $options"
-    done
+    done < <(parse_table "$tests")
 }
 
 @test "podman history - custom format" {
@@ -42,7 +44,9 @@ size      | -\\\?[0-9]\\\+
 
     run_podman history --format json $IMAGE
 
-    parse_table "$tests" | while read field expect; do
+    defer-assertion-failures
+
+    while read field expect; do
         # HACK: we can't include '|' in the table
         if [ "$field" = "id" ]; then expect="$expect\|<missing>";fi
 
@@ -54,8 +58,7 @@ size      | -\\\?[0-9]\\\+
             is "$actual" "$expect\$" "jq .[$i].$field"
             i=$(expr $i + 1)
         done
-    done
-
+    done < <(parse_table "$tests")
 }
 
 @test "podman image history Created" {

test/system/620-option-conflicts.bats

@@ -18,6 +18,9 @@ container cleanup | --all          | --exec=foo
 container cleanup | --exec=foo     | --rmi                  | foo
 "
 
+    # Run all tests, continue even if any fail
+    defer-assertion-failures
+
     # FIXME: parse_table is what does all the work, giving us test cases.
     while read subcommands opt1 opt2 args; do
         opt1_name=${opt1%=*}

test/system/helpers.bash

@@ -201,6 +201,34 @@ function basic_setup() {
     # Test filenames must match ###-name.bats; use "[###] " as prefix
     run expr "$BATS_TEST_FILENAME" : "^.*/\([0-9]\{3\}\)-[^/]\+\.bats\$"
     BATS_TEST_NAME_PREFIX="[${output}] "
+
+    # By default, assert() and die() cause an immediate test failure.
+    # Under special circumstances (usually long test loops), tests
+    # can call defer-assertion-failures() to continue going, the
+    # idea being that a large number of failures can show patterns.
+    ASSERTION_FAILURES=
+    immediate-assertion-failures
+}
+
+function immediate-assertion-failures() {
+    function bail-now() {
+        # "false" does not apply to "bail now"! It means "nonzero exit",
+        # which BATS interprets as "yes, bail immediately".
+        false
+    }
+
+    # Any backlog?
+    if [[ -n "$ASSERTION_FAILURES" ]]; then
+        local n=${#ASSERTION_FAILURES}
+        ASSERTION_FAILURES=
+        die "$n test assertions failed. Search for 'FAIL:' above this line." >&2
+    fi
+}
+
+function defer-assertion-failures() {
+    function bail-now() {
+        ASSERTION_FAILURES+="!"
+    }
 }
 
 # Basic teardown: remove all pods and containers
@@ -237,6 +265,7 @@ function basic_teardown() {
     done
 
     command rm -rf $PODMAN_TMPDIR
+    immediate-assertion-failures
 }
 
 
@@ -745,7 +774,7 @@ function die() {
     echo "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv"  >&2
     echo "#| FAIL: $*"                                           >&2
     echo "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" >&2
-    false
+    bail-now
 }
 
 ############
@@ -861,7 +890,7 @@ function assert() {
         printf "#|         > %s%s\n" "$ws" "$line"                >&2
     done
     printf "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"   >&2
-    false
+    bail-now
 }
 
 ########
@@ -911,7 +940,7 @@ function is() {
         printf "#|         > '%s'\n" "$line"                   >&2
     done
     printf "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n" >&2
-    false
+    bail-now
 }
 
 ####################