From 1f3599d635b86410a2ca34b7b669df1e0984c705 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Sat, 12 Oct 2024 17:40:50 +0200 Subject: [PATCH 1/6] feat: display "totals" even on stop on failure --- src/env.sh | 4 ++-- src/main.sh | 10 +++++++++- src/runner.sh | 3 ++- ...h.test_bashunit_when_stop_on_failure_env.snapshot | 12 ++++++++++++ ...est_bashunit_when_stop_on_failure_option.snapshot | 12 ++++++++++++ 5 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/env.sh b/src/env.sh index 3ed96928..dfc6d56e 100644 --- a/src/env.sh +++ b/src/env.sh @@ -12,7 +12,6 @@ _DEFAULT_LOG_PATH="out.log" _DEFAULT_LOAD_FILE="tests/bootstrap.sh" _DEFAULT_LOG_JUNIT="" _DEFAULT_REPORT_HTML="" -_DEFAULT_TERMINAL_WIDTH=100 : "${BASHUNIT_DEFAULT_PATH:=${DEFAULT_PATH:=$_DEFAULT_DEFAULT_PATH}}" : "${BASHUNIT_LOG_JUNIT:=${LOG_JUNIT:=$_DEFAULT_LOG_JUNIT}}" @@ -76,9 +75,10 @@ function env::find_terminal_width() { fi # Directly echo the value with fallback - echo "${cols:-$_DEFAULT_TERMINAL_WIDTH}" + echo "${cols:-100}" } +EXIT_CODE_STOP_ON_FAILURE=4 TEMP_DIR_PARALLEL_TEST_SUITE="/tmp/bashunit/parallel/${_OS:-Unknown}" TERMINAL_WIDTH="$(env::find_terminal_width)" FAILURES_OUTPUT_PATH=$(mktemp) diff --git a/src/main.sh b/src/main.sh index 0d87998b..ee52dcd2 100644 --- a/src/main.sh +++ b/src/main.sh @@ -16,7 +16,8 @@ function main::exec_tests() { fi # Trap SIGINT (Ctrl-C) and call the cleanup function - trap main::cleanup SIGINT + trap 'main::cleanup' SIGINT + trap '[[ $? -eq $EXIT_CODE_STOP_ON_FAILURE ]] && main::handle_stop_on_failure' EXIT if env::is_parallel_run_enabled && check_os::is_alpine; then printf "%sWarning: Parallel test execution on Alpine Linux is currently" "${_COLOR_INCOMPLETE}" @@ -54,6 +55,13 @@ function main::cleanup() { exit 1 } +function main::handle_stop_on_failure() { + console_results::print_failing_tests_and_reset + console_results::render_result + cleanup_temp_files + exit 1 +} + function main::exec_assert() { local original_assert_fn=$1 local args=("${@:2}") diff --git a/src/runner.sh b/src/runner.sh index 609258af..d2a11ea2 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -29,6 +29,7 @@ function runner::load_test_files() { runner::clean_set_up_and_tear_down_after_script done + # Tests Results if env::is_parallel_run_enabled; then wait @@ -205,7 +206,7 @@ function runner::run_test() { reports::add_test_failed "$test_file" "$function_name" "$duration" "$total_assertions" runner::write_failure_result_output "$test_file" "$subshell_output" if env::is_stop_on_failure_enabled; then - exit 1 + exit "$EXIT_CODE_STOP_ON_FAILURE" fi return fi diff --git a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot index b13b332f..3db6168a 100644 --- a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot +++ b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot @@ -3,3 +3,15 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh ✗ Failed: B error Expected '1' but got  '2' + +There was 1 failure: + +|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh +|✗ Failed: B error +| Expected '1' +| but got  '2' + +Tests:  1 passed, 1 failed, 2 total +Assertions: 3 passed, 1 failed, 4 total + + Some tests failed  diff --git a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot index b13b332f..3db6168a 100644 --- a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot +++ b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot @@ -3,3 +3,15 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh ✗ Failed: B error Expected '1' but got  '2' + +There was 1 failure: + +|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh +|✗ Failed: B error +| Expected '1' +| but got  '2' + +Tests:  1 passed, 1 failed, 2 total +Assertions: 3 passed, 1 failed, 4 total + + Some tests failed  From ab22ed40fecefc3ef2faf356bb2c6aa4ce44bc00 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Sat, 12 Oct 2024 17:54:58 +0200 Subject: [PATCH 2/6] feat: add Stop on failure enabled text --- src/main.sh | 1 + ...ilure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot | 1 + ...re_test_sh.test_bashunit_when_stop_on_failure_option.snapshot | 1 + 3 files changed, 3 insertions(+) diff --git a/src/main.sh b/src/main.sh index ee52dcd2..daaf9832 100644 --- a/src/main.sh +++ b/src/main.sh @@ -56,6 +56,7 @@ function main::cleanup() { } function main::handle_stop_on_failure() { + printf "%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}" console_results::print_failing_tests_and_reset console_results::render_result cleanup_temp_files diff --git a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot index 3db6168a..7f8e30c1 100644 --- a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot +++ b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot @@ -3,6 +3,7 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh ✗ Failed: B error Expected '1' but got  '2' +Stop on failure enabled... There was 1 failure: diff --git a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot index 3db6168a..7f8e30c1 100644 --- a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot +++ b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot @@ -3,6 +3,7 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh ✗ Failed: B error Expected '1' but got  '2' +Stop on failure enabled... There was 1 failure: From e8a94a9fce81ae054ca7ef11c631370fbd4cc882 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Sat, 12 Oct 2024 19:12:50 +0200 Subject: [PATCH 3/6] feat: create .stop-on-failure shared on parallel tests --- src/env.sh | 1 + src/main.sh | 14 +++++++++++--- src/parallel.sh | 14 ++++++++++++++ src/runner.sh | 15 ++++++++++----- ...est_bashunit_when_stop_on_failure_env.snapshot | 1 + ..._bashunit_when_stop_on_failure_option.snapshot | 1 + 6 files changed, 38 insertions(+), 8 deletions(-) diff --git a/src/env.sh b/src/env.sh index dfc6d56e..3188eaf8 100644 --- a/src/env.sh +++ b/src/env.sh @@ -80,6 +80,7 @@ function env::find_terminal_width() { EXIT_CODE_STOP_ON_FAILURE=4 TEMP_DIR_PARALLEL_TEST_SUITE="/tmp/bashunit/parallel/${_OS:-Unknown}" +TEMP_FILE_PARALLEL_STOP_ON_FAILURE="$TEMP_DIR_PARALLEL_TEST_SUITE/.stop-on-failure" TERMINAL_WIDTH="$(env::find_terminal_width)" FAILURES_OUTPUT_PATH=$(mktemp) CAT="$(which cat)" diff --git a/src/main.sh b/src/main.sh index daaf9832..f610b726 100644 --- a/src/main.sh +++ b/src/main.sh @@ -17,7 +17,7 @@ function main::exec_tests() { # Trap SIGINT (Ctrl-C) and call the cleanup function trap 'main::cleanup' SIGINT - trap '[[ $? -eq $EXIT_CODE_STOP_ON_FAILURE ]] && main::handle_stop_on_failure' EXIT + trap '[[ $? -eq $EXIT_CODE_STOP_ON_FAILURE ]] && main::handle_stop_on_failure_sync' EXIT if env::is_parallel_run_enabled && check_os::is_alpine; then printf "%sWarning: Parallel test execution on Alpine Linux is currently" "${_COLOR_INCOMPLETE}" @@ -25,12 +25,20 @@ function main::exec_tests() { printf "particularly involving race conditions.%s\n" "${_COLOR_DEFAULT}" fi + if env::is_parallel_run_enabled; then + parallel::reset + fi + console_header::print_version_with_env "$filter" "${test_files[@]}" runner::load_test_files "$filter" "${test_files[@]}" if env::is_parallel_run_enabled; then wait fi + if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then + printf "\r%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}" + fi + console_results::print_failing_tests_and_reset console_results::render_result exit_code=$? @@ -55,8 +63,8 @@ function main::cleanup() { exit 1 } -function main::handle_stop_on_failure() { - printf "%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}" +function main::handle_stop_on_failure_sync() { + printf "\n%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}" console_results::print_failing_tests_and_reset console_results::render_result cleanup_temp_files diff --git a/src/parallel.sh b/src/parallel.sh index e25250b7..6d6aea19 100755 --- a/src/parallel.sh +++ b/src/parallel.sh @@ -64,3 +64,17 @@ function parallel::aggregate_test_results() { export _ASSERTIONS_INCOMPLETE=$total_incomplete export _ASSERTIONS_SNAPSHOT=$total_snapshot } + +function parallel::mark_stop_on_failure() { + touch "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" +} + +function parallel::must_stop_on_failure() { + [[ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ]] +} + +function parallel::reset() { + # shellcheck disable=SC2153 + rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE" + [ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ] && rm "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" +} diff --git a/src/runner.sh b/src/runner.sh index d2a11ea2..f1c9d5d2 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -6,10 +6,6 @@ function runner::load_test_files() { local files=("${@}") local pids=() - if env::is_parallel_run_enabled; then - rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE" - fi - for test_file in "${files[@]}"; do if [[ ! -f $test_file ]]; then continue @@ -89,6 +85,7 @@ function runner::call_test_functions() { # shellcheck disable=SC2207 local functions_to_run=($(runner::functions_for_script "$script" "$filtered_functions")) + # todo: refactor -> invert condition and reduce indentation level if [[ "${#functions_to_run[@]}" -gt 0 ]]; then if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then echo "Running $script" @@ -97,6 +94,10 @@ function runner::call_test_functions() { helper::check_duplicate_functions "$script" || true for function_name in "${functions_to_run[@]}"; do + if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then + break + fi + local provider_data=() while IFS=" " read -r line; do provider_data+=("$line") @@ -206,7 +207,11 @@ function runner::run_test() { reports::add_test_failed "$test_file" "$function_name" "$duration" "$total_assertions" runner::write_failure_result_output "$test_file" "$subshell_output" if env::is_stop_on_failure_enabled; then - exit "$EXIT_CODE_STOP_ON_FAILURE" + if env::is_parallel_run_enabled; then + parallel::mark_stop_on_failure + else + exit "$EXIT_CODE_STOP_ON_FAILURE" + fi fi return fi diff --git a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot index 7f8e30c1..4d22f06a 100644 --- a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot +++ b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot @@ -3,6 +3,7 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh ✗ Failed: B error Expected '1' but got  '2' + Stop on failure enabled... There was 1 failure: diff --git a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot index 7f8e30c1..4d22f06a 100644 --- a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot +++ b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot @@ -3,6 +3,7 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh ✗ Failed: B error Expected '1' but got  '2' + Stop on failure enabled... There was 1 failure: From c1f1609c7879241a3e4c21685495bf307a3727fa Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Sat, 12 Oct 2024 19:16:56 +0200 Subject: [PATCH 4/6] refactor: runner::call_test_functions --- src/runner.sh | 61 ++++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/src/runner.sh b/src/runner.sh index f1c9d5d2..0f0445af 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -85,43 +85,44 @@ function runner::call_test_functions() { # shellcheck disable=SC2207 local functions_to_run=($(runner::functions_for_script "$script" "$filtered_functions")) - # todo: refactor -> invert condition and reduce indentation level - if [[ "${#functions_to_run[@]}" -gt 0 ]]; then - if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then - echo "Running $script" - fi + if [[ "${#functions_to_run[@]}" -le 0 ]]; then + return + fi - helper::check_duplicate_functions "$script" || true + if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then + echo "Running $script" + fi - for function_name in "${functions_to_run[@]}"; do - if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then - break - fi + helper::check_duplicate_functions "$script" || true - local provider_data=() - while IFS=" " read -r line; do - provider_data+=("$line") - done <<< "$(helper::get_provider_data "$function_name" "$script")" + for function_name in "${functions_to_run[@]}"; do + if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then + break + fi - # No data provider found - if [[ "${#provider_data[@]}" -eq 0 ]]; then - runner::run_test "$script" "$function_name" - unset function_name - continue - fi + local provider_data=() + while IFS=" " read -r line; do + provider_data+=("$line") + done <<< "$(helper::get_provider_data "$function_name" "$script")" - # Execute the test function for each line of data - for data in "${provider_data[@]}"; do - IFS=" " read -r -a args <<< "$data" - if [ "${#args[@]}" -gt 1 ]; then - runner::run_test "$script" "$function_name" "${args[@]}" - else - runner::run_test "$script" "$function_name" "$data" - fi - done + # No data provider found + if [[ "${#provider_data[@]}" -eq 0 ]]; then + runner::run_test "$script" "$function_name" unset function_name + continue + fi + + # Execute the test function for each line of data + for data in "${provider_data[@]}"; do + IFS=" " read -r -a args <<< "$data" + if [ "${#args[@]}" -gt 1 ]; then + runner::run_test "$script" "$function_name" "${args[@]}" + else + runner::run_test "$script" "$function_name" "$data" + fi done - fi + unset function_name + done } # shellcheck disable=SC2155 From 498a50081c8d21793e161f524bda427411b1aef8 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Sat, 12 Oct 2024 19:17:26 +0200 Subject: [PATCH 5/6] refactor: remove unused function run_test --- src/runner.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/runner.sh b/src/runner.sh index 0f0445af..1de0a628 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -67,11 +67,6 @@ function runner::functions_for_script() { shopt -u extdebug } -# todo: remove me; deprecated. Helper function for test authors to invoke a named test case -function run_test() { - runner::run_test "testing-fn" "$function_name" "$@" -} - function runner::call_test_functions() { local script="$1" local filter="$2" From ef1b60133fd6b5716bd0c176249e98a42a1a3ba4 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Sat, 12 Oct 2024 19:22:50 +0200 Subject: [PATCH 6/6] docs: update changelog --- CHANGELOG.md | 1 + src/runner.sh | 20 ++++---------------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb17b52b..d12d655d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - `LOG_PATH="out.log"` - `LOAD_FILE="tests/bootstrap.sh"` - Add check that git is installed to `install.sh` +- Fixed `-S|--stop-on-failure` behaviour - Improved time taken display - Improved clean up temporal files and directories - Improved CI test speed by running them in parallel diff --git a/src/runner.sh b/src/runner.sh index 1de0a628..e46c3124 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -1,23 +1,20 @@ #!/bin/bash +# shellcheck disable=SC2155 function runner::load_test_files() { local filter=$1 shift local files=("${@}") - local pids=() for test_file in "${files[@]}"; do if [[ ! -f $test_file ]]; then continue fi - # shellcheck source=/dev/null source "$test_file" - runner::run_set_up_before_script if env::is_parallel_run_enabled; then runner::call_test_functions "$test_file" "$filter" 2>/dev/null & - pids+=($!) else runner::call_test_functions "$test_file" "$filter" fi @@ -25,15 +22,11 @@ function runner::load_test_files() { runner::clean_set_up_and_tear_down_after_script done - # Tests Results if env::is_parallel_run_enabled; then wait - runner::spinner & local spinner_pid=$! - parallel::aggregate_test_results "$TEMP_DIR_PARALLEL_TEST_SUITE" - # Kill the spinner once the aggregation finishes disown "$spinner_pid" && kill "$spinner_pid" &>/dev/null printf "\r " # Clear the spinner output @@ -72,11 +65,8 @@ function runner::call_test_functions() { local filter="$2" local prefix="test" # Use declare -F to list all function names - local all_function_names - all_function_names=$(declare -F | awk '{print $3}') - local filtered_functions - filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names") - + local all_function_names=$(declare -F | awk '{print $3}') + local filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names") # shellcheck disable=SC2207 local functions_to_run=($(runner::functions_for_script "$script" "$filtered_functions")) @@ -120,7 +110,6 @@ function runner::call_test_functions() { done } -# shellcheck disable=SC2155 function runner::run_test() { local start_time start_time=$(clock::now) @@ -202,6 +191,7 @@ function runner::run_test() { state::add_tests_failed reports::add_test_failed "$test_file" "$function_name" "$duration" "$total_assertions" runner::write_failure_result_output "$test_file" "$subshell_output" + if env::is_stop_on_failure_enabled; then if env::is_parallel_run_enabled; then parallel::mark_stop_on_failure @@ -266,7 +256,6 @@ function runner::parse_result() { fi } -# shellcheck disable=SC2155 function runner::parse_result_parallel() { local function_name=$1 shift @@ -306,7 +295,6 @@ function runner::parse_result_parallel() { echo "$execution_result" > "$unique_test_result_file" } -# shellcheck disable=SC2155 function runner::parse_result_sync() { local function_name=$1 local execution_result=$2