Running in parallel (#358)

.env.example

@@ -4,8 +4,10 @@ BASHUNIT_REPORT_HTML=
 BASHUNIT_LOAD_FILE=
 
 # Booleans
+BASHUNIT_PARALLEL_RUN=
 BASHUNIT_SHOW_HEADER=
 BASHUNIT_HEADER_ASCII_ART=
 BASHUNIT_SIMPLE_OUTPUT=
 BASHUNIT_STOP_ON_FAILURE=
 BASHUNIT_SHOW_EXECUTION_TIME=
+BASHUNIT_DEV_MODE=

.github/workflows/tests.yml

@@ -74,3 +74,29 @@ jobs:
       - name: Run Tests
         run: |
           ./bashunit --simple tests/
+
+  simple-output-parallel:
+    name: "Simple output in parallel"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Run Tests
+        run: |
+          ./bashunit --parallel --simple tests/
+
+  extended-output-parallel:
+    name: "Extended output in parallel"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Run Tests
+        run: |
+          ./bashunit --parallel tests/

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## [Unreleased](https://github.com/TypedDevs/bashunit/compare/0.17.0...main)
 
+- Added `-p|--parallel` to enable running tests in parallel
 - Added `assert_file_contains` and `assert_file_not_contains`
 - Added `assert_true` and `assert_false`
 - Added `BASHUNIT_LOG_PATH`

Makefile

@@ -62,7 +62,7 @@ test/list:
 	@echo $(TEST_SCRIPTS) | tr ' ' '\n'
 
 test: $(TEST_SCRIPTS)
-	@./bashunit $(TEST_SCRIPTS) -e tests/bootstrap.sh
+	@./bashunit $(TEST_SCRIPTS)
 
 test/watch: $(TEST_SCRIPTS)
 	@./bashunit $(TEST_SCRIPTS)

adrs/adr-003-parallel-testing.md

@@ -0,0 +1,50 @@
+# Title: Parallel testing
+
+* Status: accepted
+* Authors: @Chemaclass
+* Date: 2024-10-11
+
+Technical Story:
+- Pull Request: [TypedDevs/bashunit#358](https://github.com/TypedDevs/bashunit/pull/358)
+
+## Context and Problem Statement
+
+We aim to enhance testing performance by running tests in parallel processes while capturing and aggregating results effectively.
+
+## Considered Options
+
+- Implement parallel execution using subprocesses.
+- Aggregate test results from temporary files.
+- Use a spinner for user feedback during result aggregation.
+
+## Decision Outcome
+
+- Implemented parallel test execution using subprocesses.
+- Each test creates a temporary directory to store results, later aggregated.
+
+### Positive Consequences
+
+- Reduced test execution time considerably.
+- Clear feedback via a spinner during aggregation.
+
+### Negative Consequences
+
+- Potential complexity
+    - with handling temporary files during interruptions.
+    - in handling temporary files and managing subprocesses.
+
+## Technical Details
+
+When the `--parallel` flag is used, each test is run in its own subprocess by calling:
+
+> runner::call_test_functions "$test_file" "$filter" 2>/dev/null &
+
+Each test script creates a temporary directory and stores individual test results in temp files.
+After all tests finish, the results are aggregated by traversing these directories and files.
+This approach ensures isolation of test execution while improving performance by running tests concurrently.
+
+The aggregation (which collects all test outcomes into a final result set) is handled by the function:
+
+> parallel::aggregate_test_results "$TEMP_DIR_PARALLEL_TEST_SUITE"
+
+

bashunit

@@ -9,13 +9,14 @@ declare -r BASHUNIT_ROOT_DIR="$(dirname "${BASH_SOURCE[0]}")"
 export BASHUNIT_ROOT_DIR
 
 source "$BASHUNIT_ROOT_DIR/src/dev/debug.sh"
+source "$BASHUNIT_ROOT_DIR/src/check_os.sh"
 source "$BASHUNIT_ROOT_DIR/src/str.sh"
 source "$BASHUNIT_ROOT_DIR/src/globals.sh"
 source "$BASHUNIT_ROOT_DIR/src/dependencies.sh"
 source "$BASHUNIT_ROOT_DIR/src/io.sh"
 source "$BASHUNIT_ROOT_DIR/src/math.sh"
+source "$BASHUNIT_ROOT_DIR/src/parallel.sh"
 source "$BASHUNIT_ROOT_DIR/src/env.sh"
-source "$BASHUNIT_ROOT_DIR/src/check_os.sh"
 source "$BASHUNIT_ROOT_DIR/src/clock.sh"
 source "$BASHUNIT_ROOT_DIR/src/state.sh"
 source "$BASHUNIT_ROOT_DIR/src/colors.sh"
@@ -63,6 +64,12 @@ while [[ $# -gt 0 ]]; do
     -S|--stop-on-failure)
       export BASHUNIT_STOP_ON_FAILURE=true
       ;;
+    -p|--parallel)
+      export BASHUNIT_PARALLEL_RUN=true
+      ;;
+    --no-parallel)
+      export BASHUNIT_PARALLEL_RUN=false
+      ;;
     -e|--env|--load)
       # shellcheck disable=SC1090
       source "$2"

docs/command-line.md

@@ -99,6 +99,28 @@ Creates a report XML file that follows the JUnit XML format and contains informa
 ```
 :::
 
+## Parallel
+
+> `bashunit -p|--parallel`
+
+bashunit provides an option to run each test in a separate child process, allowing you to parallelize the test execution and potentially speed up the testing process. When running in parallel mode, the execution order of tests is randomized.
+
+::: code-group
+```bash [Example]
+./bashunit ./tests --parallel
+```
+:::
+
+This runs the tests in child processes with randomized execution, which may improve overall testing speed, especially for larger test suites.
+
+You can use `BASHUNIT_PARALLEL_RUN` option in your [configuration](/configuration#parallel).
+
+### Disabling Parallel Testing
+
+> `bashunit --no-parallel`
+
+If parallel testing is enabled by default or within a script, you can disable it using the --no-parallel option. This is useful if you need to run tests in sequence or if parallel execution is causing issues during debugging.
+
 ## Report
 
 > `bashunit -r|--report-html <out.html>`

docs/configuration.md

@@ -62,6 +62,15 @@ BASHUNIT_SIMPLE_OUTPUT=false
 ```
 :::
 
+## Parallel
+
+> `BASHUNIT_PARALLEL_RUN=true|false`
+
+Runs the tests in child processes with randomized execution, which may improve overall testing speed, especially for larger test suites.
+
+Similar as using `-p|--parallel` option on the [command line](/command-line#parallel).
+
+
 ## Stop on failure
 
 > `BASHUNIT_STOP_ON_FAILURE=true|false`

src/console_header.sh

@@ -76,6 +76,12 @@ Options:
   -l|--log-junit <out.xml>
     Create a report JUnit XML file that contains information about the test results.
 
+  -p|--parallel
+    Run each test in child process, randomizing the tests execution order.
+
+  --no-parallel
+    Disable the --parallel option. Util to disable parallel tests from within another test.
+
   -r|--report-html <out.html>
     Create a report HTML file that contains information about the test results.
 

src/env.sh

@@ -27,13 +27,15 @@ _DEFAULT_HEADER_ASCII_ART="false"
 _DEFAULT_SIMPLE_OUTPUT="false"
 _DEFAULT_STOP_ON_FAILURE="false"
 _DEFAULT_SHOW_EXECUTION_TIME="true"
+_DEFAULT_DEV_MODE="false"
 
 : "${BASHUNIT_PARALLEL_RUN:=${PARALLEL_RUN:=$_DEFAULT_PARALLEL_RUN}}"
 : "${BASHUNIT_SHOW_HEADER:=${SHOW_HEADER:=$_DEFAULT_SHOW_HEADER}}"
 : "${BASHUNIT_HEADER_ASCII_ART:=${HEADER_ASCII_ART:=$_DEFAULT_HEADER_ASCII_ART}}"
 : "${BASHUNIT_SIMPLE_OUTPUT:=${SIMPLE_OUTPUT:=$_DEFAULT_SIMPLE_OUTPUT}}"
 : "${BASHUNIT_STOP_ON_FAILURE:=${STOP_ON_FAILURE:=$_DEFAULT_STOP_ON_FAILURE}}"
 : "${BASHUNIT_SHOW_EXECUTION_TIME:=${SHOW_EXECUTION_TIME:=$_DEFAULT_SHOW_EXECUTION_TIME}}"
+: "${BASHUNIT_DEV_MODE:=${DEV_MODE:=$_DEFAULT_DEV_MODE}}"
 
 function env::is_parallel_run_enabled() {
   [[ "$BASHUNIT_PARALLEL_RUN" == "true" ]]
@@ -59,6 +61,10 @@ function env::is_show_execution_time_enabled() {
   [[ "$BASHUNIT_SHOW_EXECUTION_TIME" == "true" ]]
 }
 
+function env::is_dev_mode_enabled() {
+  [[ "$BASHUNIT_DEV_MODE" == "true" ]]
+}
+
 function env::find_terminal_width() {
   local cols=""
 
@@ -73,6 +79,7 @@ function env::find_terminal_width() {
   echo "${cols:-$_DEFAULT_TERMINAL_WIDTH}"
 }
 
+TEMP_DIR_PARALLEL_TEST_SUITE="/tmp/bashunit/parallel/${_OS:-Unknown}"
 TERMINAL_WIDTH="$(env::find_terminal_width)"
 FAILURES_OUTPUT_PATH=$(mktemp)
 CAT="$(which cat)"

src/globals.sh

@@ -25,17 +25,17 @@ function random_str() {
 }
 
 function temp_file() {
-  mkdir -p /tmp/bashunit-tmp && chmod -R 777 /tmp/bashunit-tmp
-  mktemp /tmp/bashunit-tmp/bashunit.XXXXXXX
+  mkdir -p /tmp/bashunit/tmp && chmod -R 777 /tmp/bashunit/tmp
+  mktemp /tmp/bashunit/tmp/bashunit.XXXXXXX
 }
 
 function temp_dir() {
-  mkdir -p /tmp/bashunit-tmp && chmod -R 777 /tmp/bashunit-tmp
-  mktemp -d /tmp/bashunit-tmp/bashunit.XXXXXXX
+  mkdir -p /tmp/bashunit/tmp && chmod -R 777 /tmp/bashunit/tmp
+  mktemp -d /tmp/bashunit/tmp/bashunit.XXXXXXX
 }
 
 function cleanup_temp_files() {
-  rm -rf /tmp/bashunit-tmp/*
+  rm -rf /tmp/bashunit/tmp/*
 }
 
 # shellcheck disable=SC2145

src/main.sh

@@ -15,8 +15,21 @@ function main::exec_tests() {
     exit 1
   fi
 
+  # Trap SIGINT (Ctrl-C) and call the cleanup function
+  trap main::cleanup SIGINT
+
+  if env::is_parallel_run_enabled && check_os::is_alpine; then
+    printf "%sWarning: Parallel test execution on Alpine Linux is currently" "${_COLOR_INCOMPLETE}"
+    printf "in a beta stage.\nThis means there may be unresolved issues, "
+    printf "particularly involving race conditions.%s\n" "${_COLOR_DEFAULT}"
+  fi
+
   console_header::print_version_with_env "$filter" "${test_files[@]}"
   runner::load_test_files "$filter" "${test_files[@]}"
+  if env::is_parallel_run_enabled; then
+    wait
+  fi
+
   console_results::print_failing_tests_and_reset
   console_results::render_result
   exit_code=$?
@@ -30,10 +43,17 @@ function main::exec_tests() {
   fi
 
   cleanup_temp_files
-
   exit $exit_code
 }
 
+function main::cleanup() {
+  printf "%sCaught Ctrl-C, killing all child processes...%s\n"  "${_COLOR_SKIPPED}" "${_COLOR_DEFAULT}"
+  # Kill all child processes of this script
+  pkill -P $$
+  cleanup_temp_files
+  exit 1
+}
+
 function main::exec_assert() {
   local original_assert_fn=$1
   local args=("${@:2}")

src/parallel.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+function parallel::aggregate_test_results() {
+  local temp_dir_parallel_test_suite=$1
+
+  local total_failed=0
+  local total_passed=0
+  local total_skipped=0
+  local total_incomplete=0
+  local total_snapshot=0
+
+  for script_dir in "$temp_dir_parallel_test_suite"/*; do
+    for result_file in "$script_dir"/*.result; do
+      while IFS= read -r line; do
+        # Extract assertion counts from the result lines using sed
+        failed=$(echo "$line" | sed -n 's/.*##ASSERTIONS_FAILED=\([0-9]*\)##.*/\1/p')
+        passed=$(echo "$line" | sed -n 's/.*##ASSERTIONS_PASSED=\([0-9]*\)##.*/\1/p')
+        skipped=$(echo "$line" | sed -n 's/.*##ASSERTIONS_SKIPPED=\([0-9]*\)##.*/\1/p')
+        incomplete=$(echo "$line" | sed -n 's/.*##ASSERTIONS_INCOMPLETE=\([0-9]*\)##.*/\1/p')
+        snapshot=$(echo "$line" | sed -n 's/.*##ASSERTIONS_SNAPSHOT=\([0-9]*\)##.*/\1/p')
+
+        # Default to 0 if no match is found
+        failed=${failed:-0}
+        passed=${passed:-0}
+        skipped=${skipped:-0}
+        incomplete=${incomplete:-0}
+        snapshot=${snapshot:-0}
+
+        # Add to the total counts
+        total_failed=$((total_failed + failed))
+        total_passed=$((total_passed + passed))
+        total_skipped=$((total_skipped + skipped))
+        total_incomplete=$((total_incomplete + incomplete))
+        total_snapshot=$((total_snapshot + snapshot))
+      done < "$result_file"
+
+      if [ "${failed:-0}" -gt 0 ]; then
+        state::add_tests_failed
+        continue
+      fi
+
+      if [ "${snapshot:-0}" -gt 0 ]; then
+        state::add_tests_snapshot
+        continue
+      fi
+
+      if [ "${incomplete:-0}" -gt 0 ]; then
+        state::add_tests_incomplete
+        continue
+      fi
+
+      if [ "${skipped:-0}" -gt 0 ]; then
+        state::add_tests_skipped
+        continue
+      fi
+
+      state::add_tests_passed
+    done
+  done
+
+  export _ASSERTIONS_FAILED=$total_failed
+  export _ASSERTIONS_PASSED=$total_passed
+  export _ASSERTIONS_SKIPPED=$total_skipped
+  export _ASSERTIONS_INCOMPLETE=$total_incomplete
+  export _ASSERTIONS_SNAPSHOT=$total_snapshot
+}