Stop on failure when running on parallel (#367)

CHANGELOG.md

@@ -21,6 +21,7 @@
     - `LOG_PATH="out.log"`
     - `LOAD_FILE="tests/bootstrap.sh"`
 - Add check that git is installed to `install.sh`
+- Fixed `-S|--stop-on-failure` behaviour
 - Improved time taken display
 - Improved clean up temporal files and directories
 - Improved CI test speed by running them in parallel

src/env.sh

@@ -12,7 +12,6 @@ _DEFAULT_LOG_PATH="out.log"
 _DEFAULT_LOAD_FILE="tests/bootstrap.sh"
 _DEFAULT_LOG_JUNIT=""
 _DEFAULT_REPORT_HTML=""
-_DEFAULT_TERMINAL_WIDTH=100
 
 : "${BASHUNIT_DEFAULT_PATH:=${DEFAULT_PATH:=$_DEFAULT_DEFAULT_PATH}}"
 : "${BASHUNIT_LOG_JUNIT:=${LOG_JUNIT:=$_DEFAULT_LOG_JUNIT}}"
@@ -76,10 +75,12 @@ function env::find_terminal_width() {
   fi
 
   # Directly echo the value with fallback
-  echo "${cols:-$_DEFAULT_TERMINAL_WIDTH}"
+  echo "${cols:-100}"
 }
 
+EXIT_CODE_STOP_ON_FAILURE=4
 TEMP_DIR_PARALLEL_TEST_SUITE="/tmp/bashunit/parallel/${_OS:-Unknown}"
+TEMP_FILE_PARALLEL_STOP_ON_FAILURE="$TEMP_DIR_PARALLEL_TEST_SUITE/.stop-on-failure"
 TERMINAL_WIDTH="$(env::find_terminal_width)"
 FAILURES_OUTPUT_PATH=$(mktemp)
 CAT="$(which cat)"

src/main.sh

@@ -16,20 +16,29 @@ function main::exec_tests() {
   fi
 
   # Trap SIGINT (Ctrl-C) and call the cleanup function
-  trap main::cleanup SIGINT
+  trap 'main::cleanup' SIGINT
+  trap '[[ $? -eq $EXIT_CODE_STOP_ON_FAILURE ]] && main::handle_stop_on_failure_sync' EXIT
 
   if env::is_parallel_run_enabled && check_os::is_alpine; then
     printf "%sWarning: Parallel test execution on Alpine Linux is currently" "${_COLOR_INCOMPLETE}"
     printf "in a beta stage.\nThis means there may be unresolved issues, "
     printf "particularly involving race conditions.%s\n" "${_COLOR_DEFAULT}"
   fi
 
+  if env::is_parallel_run_enabled; then
+    parallel::reset
+  fi
+
   console_header::print_version_with_env "$filter" "${test_files[@]}"
   runner::load_test_files "$filter" "${test_files[@]}"
   if env::is_parallel_run_enabled; then
     wait
   fi
 
+  if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then
+    printf "\r%sStop on failure enabled...%s\n"  "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}"
+  fi
+
   console_results::print_failing_tests_and_reset
   console_results::render_result
   exit_code=$?
@@ -54,6 +63,14 @@ function main::cleanup() {
   exit 1
 }
 
+function main::handle_stop_on_failure_sync() {
+  printf "\n%sStop on failure enabled...%s\n"  "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}"
+  console_results::print_failing_tests_and_reset
+  console_results::render_result
+  cleanup_temp_files
+  exit 1
+}
+
 function main::exec_assert() {
   local original_assert_fn=$1
   local args=("${@:2}")

src/parallel.sh

@@ -64,3 +64,17 @@ function parallel::aggregate_test_results() {
   export _ASSERTIONS_INCOMPLETE=$total_incomplete
   export _ASSERTIONS_SNAPSHOT=$total_snapshot
 }
+
+function parallel::mark_stop_on_failure() {
+  touch "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE"
+}
+
+function parallel::must_stop_on_failure() {
+  [[ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ]]
+}
+
+function parallel::reset() {
+  # shellcheck disable=SC2153
+  rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE"
+  [ -f "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE" ] && rm "$TEMP_FILE_PARALLEL_STOP_ON_FAILURE"
+}

src/runner.sh

@@ -1,27 +1,20 @@
 #!/bin/bash
+# shellcheck disable=SC2155
 
 function runner::load_test_files() {
   local filter=$1
   shift
   local files=("${@}")
-  local pids=()
-
-  if env::is_parallel_run_enabled; then
-    rm -rf "$TEMP_DIR_PARALLEL_TEST_SUITE"
-  fi
 
   for test_file in "${files[@]}"; do
     if [[ ! -f $test_file ]]; then
       continue
     fi
-
     # shellcheck source=/dev/null
     source "$test_file"
-
     runner::run_set_up_before_script
     if env::is_parallel_run_enabled; then
       runner::call_test_functions "$test_file" "$filter" 2>/dev/null &
-      pids+=($!)
     else
       runner::call_test_functions "$test_file" "$filter"
     fi
@@ -31,12 +24,9 @@ function runner::load_test_files() {
 
   if env::is_parallel_run_enabled; then
     wait
-
     runner::spinner &
     local spinner_pid=$!
-
     parallel::aggregate_test_results "$TEMP_DIR_PARALLEL_TEST_SUITE"
-
     # Kill the spinner once the aggregation finishes
     disown "$spinner_pid" && kill "$spinner_pid" &>/dev/null
     printf "\r " # Clear the spinner output
@@ -70,59 +60,56 @@ function runner::functions_for_script() {
   shopt -u extdebug
 }
 
-# todo: remove me; deprecated. Helper function for test authors to invoke a named test case
-function run_test() {
-  runner::run_test "testing-fn" "$function_name" "$@"
-}
-
 function runner::call_test_functions() {
   local script="$1"
   local filter="$2"
   local prefix="test"
   # Use declare -F to list all function names
-  local all_function_names
-  all_function_names=$(declare -F | awk '{print $3}')
-  local filtered_functions
-  filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names")
-
+  local all_function_names=$(declare -F | awk '{print $3}')
+  local filtered_functions=$(helper::get_functions_to_run "$prefix" "$filter" "$all_function_names")
   # shellcheck disable=SC2207
   local functions_to_run=($(runner::functions_for_script "$script" "$filtered_functions"))
 
-  if [[ "${#functions_to_run[@]}" -gt 0 ]]; then
-    if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then
-      echo "Running $script"
-    fi
+  if [[ "${#functions_to_run[@]}" -le 0 ]]; then
+    return
+  fi
 
-    helper::check_duplicate_functions "$script" || true
+  if ! env::is_simple_output_enabled && ! env::is_parallel_run_enabled; then
+    echo "Running $script"
+  fi
 
-    for function_name in "${functions_to_run[@]}"; do
-      local provider_data=()
-      while IFS=" " read -r line; do
-        provider_data+=("$line")
-      done <<< "$(helper::get_provider_data "$function_name" "$script")"
+  helper::check_duplicate_functions "$script" || true
 
-      # No data provider found
-      if [[ "${#provider_data[@]}" -eq 0 ]]; then
-        runner::run_test "$script" "$function_name"
-        unset function_name
-        continue
-      fi
+  for function_name in "${functions_to_run[@]}"; do
+    if env::is_parallel_run_enabled && parallel::must_stop_on_failure; then
+      break
+    fi
+
+    local provider_data=()
+    while IFS=" " read -r line; do
+      provider_data+=("$line")
+    done <<< "$(helper::get_provider_data "$function_name" "$script")"
 
-      # Execute the test function for each line of data
-      for data in "${provider_data[@]}"; do
-        IFS=" " read -r -a args <<< "$data"
-        if [ "${#args[@]}" -gt 1 ]; then
-          runner::run_test "$script" "$function_name" "${args[@]}"
-        else
-          runner::run_test "$script" "$function_name" "$data"
-        fi
-      done
+    # No data provider found
+    if [[ "${#provider_data[@]}" -eq 0 ]]; then
+      runner::run_test "$script" "$function_name"
       unset function_name
+      continue
+    fi
+
+    # Execute the test function for each line of data
+    for data in "${provider_data[@]}"; do
+      IFS=" " read -r -a args <<< "$data"
+      if [ "${#args[@]}" -gt 1 ]; then
+        runner::run_test "$script" "$function_name" "${args[@]}"
+      else
+        runner::run_test "$script" "$function_name" "$data"
+      fi
     done
-  fi
+    unset function_name
+  done
 }
 
-# shellcheck disable=SC2155
 function runner::run_test() {
   local start_time
   start_time=$(clock::now)
@@ -204,8 +191,13 @@ function runner::run_test() {
     state::add_tests_failed
     reports::add_test_failed "$test_file" "$function_name" "$duration" "$total_assertions"
     runner::write_failure_result_output "$test_file" "$subshell_output"
+
     if env::is_stop_on_failure_enabled; then
-      exit 1
+      if env::is_parallel_run_enabled; then
+        parallel::mark_stop_on_failure
+      else
+        exit "$EXIT_CODE_STOP_ON_FAILURE"
+      fi
     fi
     return
   fi
@@ -264,7 +256,6 @@ function runner::parse_result() {
   fi
 }
 
-# shellcheck disable=SC2155
 function runner::parse_result_parallel() {
   local function_name=$1
   shift
@@ -304,7 +295,6 @@ function runner::parse_result_parallel() {
   echo "$execution_result" > "$unique_test_result_file"
 }
 
-# shellcheck disable=SC2155
 function runner::parse_result_sync() {
   local function_name=$1
   local execution_result=$2

tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_env.snapshot

@@ -3,3 +3,17 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
 ✗ Failed: B error
     Expected '1'
     but got  '2'
+
+Stop on failure enabled...
+
+There was 1 failure:
+
+|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
+|✗ Failed: B error
+|    Expected '1'
+|    but got  '2'
+
+Tests:      1 passed, 1 failed, 2 total
+Assertions: 3 passed, 1 failed, 4 total
+
+ Some tests failed 

tests/acceptance/snapshots/bashunit_stop_on_failure_test_sh.test_bashunit_when_stop_on_failure_option.snapshot

@@ -3,3 +3,17 @@ Running ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
 ✗ Failed: B error
     Expected '1'
     but got  '2'
+
+Stop on failure enabled...
+
+There was 1 failure:
+
+|1) ./tests/acceptance/fixtures/test_bashunit_when_stop_on_failure.sh
+|✗ Failed: B error
+|    Expected '1'
+|    but got  '2'
+
+Tests:      1 passed, 1 failed, 2 total
+Assertions: 3 passed, 1 failed, 4 total
+
+ Some tests failed