From 45608a739e3fd2a14d26aa3160b138ff21182d80 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Sun, 9 Aug 2026 14:30:57 +0200 Subject: [PATCH] fix(reports): every report format was empty under --parallel Closes #1004. --report-junit, --report-tap, --report-json, --report-html and --log-junit all produced a report of zero tests whenever the run was parallel. The tests ran and passed; only the report was empty. ./bashunit --report-junit r.xml tests/unit/util tests="49" ./bashunit --parallel --report-junit r.xml tests/unit/util tests="0" That is worse than an obviously broken file. --parallel is the mode people reach for in CI and junit is what CI consumes, and a junit file reporting no tests renders as a green, empty run in most systems. A suite could be fully green, fully reported, and communicating nothing. bashunit::reports::add_test is called from runner/exec.sh, which under --parallel runs inside the per-test worker. The arrays it appends to died with the worker, and state/parallel.sh -- which rebuilds counters from the .result files -- had no knowledge of them. Counters crossed the fork boundary; report rows did not. Each row is now also spooled to a run-scoped file, the same mechanism --snapshot-report-unused already uses to cross that boundary, and replayed in the parent immediately before the writers run. Fields are base64-encoded because a failure message carries newlines and arbitrary text. The arrays are still filled in add_test rather than skipped when parallel. The first version returned early there, which was wrong: the reports unit tests call add_test directly in the parent, and 32 of them started asserting against arrays nothing had touched. The parent never reaches that path for a real parallel test, so replaying the spool cannot double-count. Verified by generating all three text reports from the same fixture both ways and comparing: identical for junit, and identical in content for tap and json -- they differ only in row order, which is inherent to parallel execution. Statuses, durations, assertion counts and the multi-line failure message all survive. The new acceptance test runs the same suite sequentially and in parallel and compares the counts, so the two cannot drift apart again. 1735 sequential / 1694 parallel; fork budget unchanged. --- CHANGELOG.md | 3 ++ src/config/env.sh | 1 + src/main/run.sh | 2 + src/reports/collect.sh | 50 +++++++++++++++++++ .../bashunit_parallel_reports_test.sh | 37 ++++++++++++++ .../fixtures/parallel_reports/mixed.sh | 6 +++ 6 files changed, 99 insertions(+) create mode 100644 tests/acceptance/bashunit_parallel_reports_test.sh create mode 100644 tests/acceptance/fixtures/parallel_reports/mixed.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 29c99bce..9f93f21f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +### Fixed +- Report formats are no longer empty under `--parallel`. `--report-junit`, `--report-tap`, `--report-json`, `--report-html` and `--log-junit` all recorded zero tests, because the rows were collected inside the per-test worker and nothing rebuilt them in the parent (#1004) + ## [0.45.0](https://github.com/TypedDevs/bashunit/compare/0.44.0...0.45.0) - 2026-08-09 ### Added diff --git a/src/config/env.sh b/src/config/env.sh index 64e5326e..6d38bd6b 100644 --- a/src/config/env.sh +++ b/src/config/env.sh @@ -689,6 +689,7 @@ RERUN_FAILED_OUTPUT_PATH="$_BASHUNIT_RUN_OUTPUT_DIR/rerun-failed" # diff it against what is on disk. Appended from the test subshells, only when # the flag is on, so a normal run pays nothing. SNAPSHOT_USED_OUTPUT_PATH="$_BASHUNIT_RUN_OUTPUT_DIR/snapshots-used" +REPORTS_OUTPUT_PATH="$_BASHUNIT_RUN_OUTPUT_DIR/reports" # Shared temp directory, initialized once at startup for performance. BASHUNIT_TEMP_DIR="${TMPDIR:-/tmp}/bashunit/tmp" diff --git a/src/main/run.sh b/src/main/run.sh index 5fe6a046..153c97ab 100644 --- a/src/main/run.sh +++ b/src/main/run.sh @@ -127,6 +127,8 @@ function bashunit::main::exec_tests() { bashunit::snapshot::report_unused ${test_files[@]+"${test_files[@]}"} fi + bashunit::reports::load_spooled + if [ -n "$BASHUNIT_LOG_JUNIT" ]; then bashunit::reports::generate_junit_xml "$BASHUNIT_LOG_JUNIT" fi diff --git a/src/reports/collect.sh b/src/reports/collect.sh index 11bc4ddd..e636bf41 100644 --- a/src/reports/collect.sh +++ b/src/reports/collect.sh @@ -70,6 +70,33 @@ function bashunit::reports::add_test() { "$file":*) line="${_BASHUNIT_TEST_LOCATION##*:}" ;; esac + # Under --parallel this runs inside the per-test worker, so the arrays below + # are appended to in a process that is about to exit and the parent rebuilds + # nothing -- every report came out with zero tests while the run stayed green. + # Spool the row to a run-scoped file as well, the same way + # --snapshot-report-unused crosses the fork boundary, and replay it in the + # parent before the writers run. + # + # The arrays are still filled here rather than skipped: this function is also + # called directly, in the parent, by the reports unit tests, and returning + # early left them asserting against arrays nothing had touched. The parent + # never reaches this path for a real parallel test, so replaying the spool + # cannot double-count. + # + # Fields are base64-encoded because a failure message carries newlines and + # arbitrary text, either of which would break a delimited line. + if bashunit::parallel::is_enabled; then + printf '%s|%s|%s|%s|%s|%s|%s\n' \ + "$(bashunit::helper::encode_base64 "$file")" \ + "$(bashunit::helper::encode_base64 "$test_name")" \ + "$(bashunit::helper::encode_base64 "$status")" \ + "$(bashunit::helper::encode_base64 "$duration")" \ + "$(bashunit::helper::encode_base64 "$assertions")" \ + "$(bashunit::helper::encode_base64 "$failure_message")" \ + "$(bashunit::helper::encode_base64 "$line")" \ + >>"${REPORTS_OUTPUT_PATH:-/dev/null}" 2>/dev/null || true + fi + _BASHUNIT_REPORTS_TEST_FILES[${#_BASHUNIT_REPORTS_TEST_FILES[@]}]="$file" _BASHUNIT_REPORTS_TEST_NAMES[${#_BASHUNIT_REPORTS_TEST_NAMES[@]}]="$test_name" _BASHUNIT_REPORTS_TEST_STATUSES[${#_BASHUNIT_REPORTS_TEST_STATUSES[@]}]="$status" @@ -78,3 +105,26 @@ function bashunit::reports::add_test() { _BASHUNIT_REPORTS_TEST_FAILURES[${#_BASHUNIT_REPORTS_TEST_FAILURES[@]}]="$failure_message" _BASHUNIT_REPORTS_TEST_LINES[${#_BASHUNIT_REPORTS_TEST_LINES[@]}]="$line" } + +## +# Replays rows spooled by parallel workers into the report arrays, in the order +# they were written. Called once in the parent before any report is generated; +# a no-op sequentially, where add_test filled the arrays directly. +## +function bashunit::reports::load_spooled() { + bashunit::reports::is_enabled || return 0 + [ -f "${REPORTS_OUTPUT_PATH:-}" ] || return 0 + + local file test_name status duration assertions failure_message line n + while IFS='|' read -r file test_name status duration assertions failure_message line; do + [ -n "$file" ] || continue + local n=${#_BASHUNIT_REPORTS_TEST_FILES[@]} + _BASHUNIT_REPORTS_TEST_FILES[n]=$(bashunit::helper::decode_base64 "$file") + _BASHUNIT_REPORTS_TEST_NAMES[n]=$(bashunit::helper::decode_base64 "$test_name") + _BASHUNIT_REPORTS_TEST_STATUSES[n]=$(bashunit::helper::decode_base64 "$status") + _BASHUNIT_REPORTS_TEST_DURATIONS[n]=$(bashunit::helper::decode_base64 "$duration") + _BASHUNIT_REPORTS_TEST_ASSERTIONS[n]=$(bashunit::helper::decode_base64 "$assertions") + _BASHUNIT_REPORTS_TEST_FAILURES[n]=$(bashunit::helper::decode_base64 "$failure_message") + _BASHUNIT_REPORTS_TEST_LINES[n]=$(bashunit::helper::decode_base64 "$line") + done <"$REPORTS_OUTPUT_PATH" +} diff --git a/tests/acceptance/bashunit_parallel_reports_test.sh b/tests/acceptance/bashunit_parallel_reports_test.sh new file mode 100644 index 00000000..538c7b79 --- /dev/null +++ b/tests/acceptance/bashunit_parallel_reports_test.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Report writers collect rows via bashunit::reports::add_test, which runs inside +# the per-test worker under --parallel. The arrays died with the worker and +# nothing rebuilt them, so every report came out with zero tests while the run +# itself was green -- and a junit file reporting no tests reads as a passing +# empty run in most CI systems. +FIXTURE=tests/acceptance/fixtures/parallel_reports/mixed.sh + +function report_counts() { # $1 = extra flags -> "junit_total tap_plan json_total" + local dir junit tap json + dir=$(bashunit::temp_dir) + junit="$dir/r.xml"; tap="$dir/r.tap"; json="$dir/r.json" + + # shellcheck disable=SC2086 + NO_COLOR=1 ./bashunit --skip-env-file $1 \ + --report-junit "$junit" --report-tap "$tap" --report-json "$json" \ + "$FIXTURE" >/dev/null 2>&1 || true + + printf '%s %s %s' \ + "$("$GREP" -o 'tests="[0-9]*"' "$junit" | head -1)" \ + "$("$GREP" -m1 -o '^1\.\.[0-9]*' "$tap")" \ + "$("$GREP" -o '"total": *[0-9]*' "$json" | head -1)" +} + +function test_parallel_reports_match_sequential() { + assert_same "$(report_counts '--no-parallel')" "$(report_counts '--parallel')" +} + +function test_parallel_reports_are_not_empty() { + local counts + counts=$(report_counts '--parallel') + + assert_not_contains 'tests="0"' "$counts" + assert_not_contains '1..0' "$counts" +} diff --git a/tests/acceptance/fixtures/parallel_reports/mixed.sh b/tests/acceptance/fixtures/parallel_reports/mixed.sh new file mode 100644 index 00000000..1336ef43 --- /dev/null +++ b/tests/acceptance/fixtures/parallel_reports/mixed.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +function test_one_passes() { assert_same "a" "a"; } +function test_two_passes() { assert_same "b" "b"; } +function test_three_fails() { assert_same "expected" "actual"; } +function test_four_skips() { bashunit::skip "on purpose"; }