From 98c326ba242f0428a41e31f0834cfaae0fbfd024 Mon Sep 17 00:00:00 2001 From: "Jose M. Valera Reales" Date: Sat, 12 Oct 2024 19:42:42 +0200 Subject: [PATCH] Stop on failure when running on parallel (#367) --- CHANGELOG.md | 1 + src/env.sh | 5 +- src/main.sh | 19 +++- src/parallel.sh | 14 +++ src/runner.sh | 92 +++++++++---------- ...bashunit_when_stop_on_failure_env.snapshot | 14 +++ ...hunit_when_stop_on_failure_option.snapshot | 14 +++ 7 files changed, 105 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb17b52b..d12d655d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - `LOG_PATH="out.log"` - `LOAD_FILE="tests/bootstrap.sh"` - Add check that git is installed to `install.sh` +- Fixed `-S|--stop-on-failure` behaviour - Improved time taken display - Improved clean up temporal files and directories - Improved CI test speed by running them in parallel diff --git a/src/env.sh b/src/env.sh index 3ed96928..3188eaf8 100644 --- a/src/env.sh +++ b/src/env.sh @@ -12,7 +12,6 @@ _DEFAULT_LOG_PATH="out.log" _DEFAULT_LOAD_FILE="tests/bootstrap.sh" _DEFAULT_LOG_JUNIT="" _DEFAULT_REPORT_HTML="" -_DEFAULT_TERMINAL_WIDTH=100 : "${BASHUNIT_DEFAULT_PATH:=${DEFAULT_PATH:=$_DEFAULT_DEFAULT_PATH}}" : "${BASHUNIT_LOG_JUNIT:=${LOG_JUNIT:=$_DEFAULT_LOG_JUNIT}}" @@ -76,10 +75,12 @@ function env::find_terminal_width() { fi # Directly echo the value with fallback - echo "${cols:-$_DEFAULT_TERMINAL_WIDTH}" + echo "${cols:-100}" } +EXIT_CODE_STOP_ON_FAILURE=4 TEMP_DIR_PARALLEL_TEST_SUITE="/tmp/bashunit/parallel/${_OS:-Unknown}" +TEMP_FILE_PARALLEL_STOP_ON_FAILURE="$TEMP_DIR_PARALLEL_TEST_SUITE/.stop-on-failure" TERMINAL_WIDTH="$(env::find_terminal_width)" FAILURES_OUTPUT_PATH=$(mktemp) CAT="$(which cat)" diff --git a/src/main.sh b/src/main.sh index 0d87998b..f610b726 100644 --- a/src/main.sh +++ b/src/main.sh @@ -16,7 +16,8 @@ function main::exec_tests() { fi # Trap SIGINT (Ctrl-C) and call the cleanup function - trap main::cleanup SIGINT + trap 'main::cleanup' SIGINT + trap '[[ $? -eq $EXIT_CODE_STOP_ON_FAILURE ]] && main::handle_stop_on_failure_sync' EXIT if env::is_parallel_run_enabled && check_os::is_alpine; then printf "%sWarning: Parallel test execution on Alpine Linux is currently" "${_COLOR_INCOMPLETE}" @@ -24,12 +25,20 @@ function main::exec_tests() { printf "particularly involving race conditions.%s\n" "${_COLOR_DEFAULT}" fi + if env::is_parallel_run_enabled; then + parallel::reset + fi + console_header::print_version_with_env "$filter" "${test_files[@]}" runner::load_test_files "$filter" "${test_files[@]}" if env::is_parallel_run_enabled; then wait fi + if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then + printf "\r%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}" + fi + console_results::print_failing_tests_and_reset console_results::render_result exit_code=$? @@ -54,6 +63,14 @@ function main::cleanup() { exit 1 } +function main::handle_stop_on_failure_sync() { + printf "\n%sStop on failure enabled...%s\n" "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}" + console_results::print_failing_tests_and_reset + console_results::render_result + cleanup_temp_files + exit 1 +} + function main::exec_assert() { local original_assert_fn=$1 local args=("${@:2}") diff --git a/src/parallel.sh b/src/parallel.sh index e25250b7..6d6aea19 100755 --- a/src/parallel.sh +++ b/src/parallel.sh @@ -64,3 +64,17 @@ function parallel::aggregate_test_results() { export _ASSERTIONS_INCOMPLETE=$total_incomplete export _ASSERTIONS_SNAPSHOT=$total_snapshot } + +function parallel::mark_stop_on_failure() { + touch "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" +} + +function parallel::must_stop_on_failure() { + [[ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ]] +} + +function parallel::reset() { + # shellcheck disable=SC2153 + rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE" + [ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ] && rm "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" +} diff --git a/src/runner.sh b/src/runner.sh index 609258af..e46c3124 100755 --- a/src/runner.sh +++ b/src/runner.sh @@ -1,27 +1,20 @@ #!/bin/bash +# shellcheck disable=SC2155 function runner::load_test_files() { local filter=$1 shift local files=("${@}") - local pids=() - - if env::is_parallel_run_enabled; then - rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE" - fi for test_file in "${files[@]}"; do if [[ ! -f $test_file ]]; then continue fi - # shellcheck source=/dev/null source "$test_file" - runner::run_set_up_before_script if env::is_parallel_run_enabled; then runner::call_test_functions "$test_file" "$filter" 2>/dev/null & - pids+=($!) else runner::call_test_functions "$test_file" "$filter" fi @@ -31,12 +24,9 @@ function runner::load_test_files() { if env::is_parallel_run_enabled; then wait - runner::spinner & local spinner_pid=$! - parallel::aggregate_test_results "$TEMP_DIR_PARALLEL_TEST_SUITE" - # Kill the spinner once the aggregation finishes disown "$spinner_pid" && kill "$spinner_pid" &>/dev/null printf "\r " # Clear the spinner output @@ -70,59 +60,56 @@ function runner::functions_for_script() { shopt -u extdebug } -# todo: remove me; deprecated. Helper function for test authors to invoke a named test case -function run_test() { - runner::run_test "testing-fn" "$function_name" "$@" -} - function runner::call_test_functions() { local script="$1" local filter="$2" local prefix="test" # Use declare -F to list all function names - local all_function_names - all_function_names=$(declare -F | awk '{print $3}') - local filtered_functions - filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names") - + local all_function_names=$(declare -F | awk '{print $3}') + local filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names") # shellcheck disable=SC2207 local functions_to_run=($(runner::functions_for_script "$script" "$filtered_functions")) - if [[ "${#functions_to_run[@]}" -gt 0 ]]; then - if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then - echo "Running $script" - fi + if [[ "${#functions_to_run[@]}" -le 0 ]]; then + return + fi - helper::check_duplicate_functions "$script" || true + if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then + echo "Running $script" + fi - for function_name in "${functions_to_run[@]}"; do - local provider_data=() - while IFS=" " read -r line; do - provider_data+=("$line") - done <<< "$(helper::get_provider_data "$function_name" "$script")" + helper::check_duplicate_functions "$script" || true - # No data provider found - if [[ "${#provider_data[@]}" -eq 0 ]]; then - runner::run_test "$script" "$function_name" - unset function_name - continue - fi + for function_name in "${functions_to_run[@]}"; do + if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then + break + fi + + local provider_data=() + while IFS=" " read -r line; do + provider_data+=("$line") + done <<< "$(helper::get_provider_data "$function_name" "$script")" - # Execute the test function for each line of data - for data in "${provider_data[@]}"; do - IFS=" " read -r -a args <<< "$data" - if [ "${#args[@]}" -gt 1 ]; then - runner::run_test "$script" "$function_name" "${args[@]}" - else - runner::run_test "$script" "$function_name" "$data" - fi - done + # No data provider found + if [[ "${#provider_data[@]}" -eq 0 ]]; then + runner::run_test "$script" "$function_name" unset function_name + continue + fi + + # Execute the test function for each line of data + for data in "${provider_data[@]}"; do + IFS=" " read -r -a args <<< "$data" + if [ "${#args[@]}" -gt 1 ]; then + runner::run_test "$script" "$function_name" "${args[@]}" + else + runner::run_test "$script" "$function_name" "$data" + fi done - fi + unset function_name + done } -# shellcheck disable=SC2155 function runner::run_test() { local start_time start_time=$(clock::now) @@ -204,8 +191,13 @@ function runner::run_test() { state::add_tests_failed reports::add_test_failed "$test_file" "$function_name" "$duration" "$total_assertions" runner::write_failure_result_output "$test_file" "$subshell_output" + if env::is_stop_on_failure_enabled; then - exit 1 + if env::is_parallel_run_enabled; then + parallel::mark_stop_on_failure + else + exit "$EXIT_CODE_STOP_ON_FAILURE" + fi fi return fi @@ -264,7 +256,6 @@ function runner::parse_result() { fi } -# shellcheck disable=SC2155 function runner::parse_result_parallel() { local function_name=$1 shift @@ -304,7 +295,6 @@ function runner::parse_result_parallel() { echo "$execution_result" > "$unique_test_result_file" } -# shellcheck disable=SC2155 function runner::parse_result_sync() { local function_name=$1 local execution_result=$2 diff --git a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot index b13b332f..4d22f06a 100644 --- a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot +++ b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot @@ -3,3 +3,17 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh ✗ Failed: B error Expected '1' but got  '2' + +Stop on failure enabled... + +There was 1 failure: + +|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh +|✗ Failed: B error +| Expected '1' +| but got  '2' + +Tests:  1 passed, 1 failed, 2 total +Assertions: 3 passed, 1 failed, 4 total + + Some tests failed  diff --git a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot index b13b332f..4d22f06a 100644 --- a/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot +++ b/tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot @@ -3,3 +3,17 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh ✗ Failed: B error Expected '1' but got  '2' + +Stop on failure enabled... + +There was 1 failure: + +|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh +|✗ Failed: B error +| Expected '1' +| but got  '2' + +Tests:  1 passed, 1 failed, 2 total +Assertions: 3 passed, 1 failed, 4 total + + Some tests failed