From 9c319cc1ecc5517404aa61105bacc58e829de302 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Mon, 10 Aug 2026 20:36:15 +0200 Subject: [PATCH 1/3] feat(reports): --report-md and an automatic GitHub step summary Every existing report format targets a machine, and the HTML one has to be uploaded and downloaded to be read. Nothing targeted the page a developer actually looks at first. The summary carries the verdict, a counts table and each failure with its file:line and message. Coverage and slowest-tests sections appear only when that data exists. Failure messages go inside a fence with ANSI stripped so they render verbatim, and test names are escaped so a name containing a pipe, asterisk, underscore or backtick cannot break the table. With GITHUB_STEP_SUMMARY set and no explicit path, the summary is appended there. Appended, never written: that file belongs to the whole job, so truncating it would discard the other steps' output. Only the outermost run writes it, for the same reason the annotations do -- the variable is inherited by every child, so a nested run would append its own fixtures' results to the parent's job page. Generation moved ahead of print_profile_and_reset, which deletes the profile records the summary reads, and load_spooled moved with it so a --parallel run still reports the rows its workers spooled (#1004). Closes #1015 --- .env.example | 1 + CHANGELOG.md | 1 + ...dr-011-source-layout-and-build-pipeline.md | 2 +- completions/_bashunit | 1 + completions/bashunit.bash | 2 +- docs/command-line.md | 49 +++++ src/config/env.sh | 15 ++ src/console/header.sh | 1 + src/main/run.sh | 12 +- src/main/test.sh | 5 + src/main/validate.sh | 2 +- src/reports/collect.sh | 2 + src/reports/index.sh | 1 + src/reports/markdown.sh | 173 ++++++++++++++++++ tests/acceptance/bashunit_report_md_test.sh | 106 +++++++++++ tests/unit/reports/report_md_test.sh | 138 ++++++++++++++ 16 files changed, 506 insertions(+), 5 deletions(-) create mode 100644 src/reports/markdown.sh create mode 100644 tests/acceptance/bashunit_report_md_test.sh create mode 100644 tests/unit/reports/report_md_test.sh diff --git a/.env.example b/.env.example index d2ea2eea..0eb11ea5 100644 --- a/.env.example +++ b/.env.example @@ -46,6 +46,7 @@ BASHUNIT_ORDER_BY= # Default: defined (or defects, random) BASHUNIT_FAIL_ON_FLAKY= # Default: false (treat retry-passed tests as failed) BASHUNIT_REPEAT= # Default: 1 (run each test N times) BASHUNIT_GHA_ANNOTATIONS= # Default: auto (or always, never) +BASHUNIT_REPORT_MD= # Default: empty (Markdown summary path) BASHUNIT_CHANGED= # Default: false (run only test files changed since a git ref) BASHUNIT_CHANGED_REF= # Default: empty (--changed ref: origin/HEAD, then HEAD) BASHUNIT_COVERAGE_DIFF= # Default: empty (restrict coverage to lines changed since this ref) diff --git a/CHANGELOG.md b/CHANGELOG.md index d027617e..37df1681 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added - `--coverage-diff ` restricts the coverage console report to lines changed since a base ref, and `--coverage-min` then gates on that diff percentage (#1032) +- `--report-md ` writes a Markdown run summary: verdict, counts table, failures with their message, plus coverage and slowest tests when those ran. Inside GitHub Actions it is appended to `$GITHUB_STEP_SUMMARY` automatically, so failures render on the job page (#1015) - `--gha-annotations ` controls GitHub Actions annotations on stdout; `auto` turns them on inside GitHub Actions and stays quiet everywhere else (#1014) - `--repeat ` runs each selected test n times so flakiness can be hunted before it reaches CI. The test is reported once with the aggregate outcome, a failure names the iteration it happened on, and repeat wraps `--retry` rather than the other way round (#1013) - Flaky is a first-class outcome: a test that only passed after a retry is counted separately, kept inside the pass total so the exit code is unchanged, and carried into JUnit (``), TAP, JSON, HTML and GitHub Actions along with the first attempt's failure message. `--fail-on-flaky` turns such a run red (#1012) diff --git a/adrs/adr-011-source-layout-and-build-pipeline.md b/adrs/adr-011-source-layout-and-build-pipeline.md index 9fb34ee5..31cef951 100644 --- a/adrs/adr-011-source-layout-and-build-pipeline.md +++ b/adrs/adr-011-source-layout-and-build-pipeline.md @@ -60,7 +60,7 @@ Seventeen, in load order. The order is the dependency layering: leaves first. | 10 | `cli/` | 5 | 445 | the `doc`/`init`/`upgrade`/`watch` subcommand implementations | | 11 | `assert/` | 11 | 2326 | every assertion | | 12 | `doubles/` | 4 | 505 | spies and mocks — **sourced by `assert/index.sh`, not the entrypoint** | -| 13 | `reports/` | 7 | 465 | JUnit, TAP, JSON, GHA and HTML writers | +| 13 | `reports/` | 8 | 738 | JUnit, TAP, JSON, GHA, HTML and Markdown writers | | 14 | `runner/` | 12 | 2434 | the file loop, per-test execution, retry, result parsing, `--list` | | 15 | `benchmark/` | 4 | 219 | the bench implementation (`runner/bench.sh` is its loop) | | 16 | `learn/` | 14 | 1296 | the interactive tutorial (9 of those files are `learn/lessons/`) | diff --git a/completions/_bashunit b/completions/_bashunit index cfb8519d..444447ae 100644 --- a/completions/_bashunit +++ b/completions/_bashunit @@ -78,6 +78,7 @@ _bashunit() { '(-r --report-html)'{-r,--report-html}'[Write HTML report]:file:_files' \ '--report-tap[Write TAP version 13 report]:file:_files' \ '--report-json[Write machine-readable JSON report]:file:_files' \ + '--report-md[Write a Markdown summary]:file:_files' \ '(-s --simple)'{-s,--simple}'[Simple output with dots]' \ '--detailed[Detailed output, the default]' \ '--output[Output format]:format:(tap)' \ diff --git a/completions/bashunit.bash b/completions/bashunit.bash index 3e000baa..c52cc3af 100644 --- a/completions/bashunit.bash +++ b/completions/bashunit.bash @@ -21,7 +21,7 @@ _BASHUNIT_COMPLETIONS_TEST_OPTS="--assert --boot --changed --coverage --coverage --no-coverage-report --no-output --no-output-on-failure --no-parallel \ --no-progress --no-snapshot-create --order-by --output --parallel --profile \ --random-order --repeat --report-html \ ---report-json --report-junit --report-tap --rerun-failed --retry --run-all \ +--report-json --report-junit --report-md --report-tap --rerun-failed --retry --run-all \ --seed --shard --show-incomplete --show-output --show-skipped --simple \ --skip-env-file --snapshot-report-unused --snapshot-update \ --stop-on-failure --strict --tag \ diff --git a/docs/command-line.md b/docs/command-line.md index 6b378a28..7f40f86f 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -75,6 +75,7 @@ bashunit test tests/ --parallel --simple | `-r, --report-html ` | Write HTML report | | `--report-tap ` | Write TAP version 13 report to a file | | `--report-json ` | Write machine-readable JSON report to a file | +| `--report-md ` | Write a Markdown summary (auto-appended to `$GITHUB_STEP_SUMMARY`) | | `-R, --run-all` | Run all assertions (don't stop on first failure) | | `-s, --simple` | Simple output (dots) | | `--detailed` | Detailed output (default) | @@ -458,6 +459,54 @@ bashunit test tests/ --report-json report.json ``` ::: +### Markdown summary + +> `bashunit test --report-md ` + +Every other report format targets a machine. This one targets the page a +developer actually looks at first: + +```markdown +## bashunit + +❌ **3 failed**, 409 passed in 12.3s + +| Result | Count | +|--------|-------| +| Passed | 409 | +| Failed | 3 | + +## Failures + +### Sums two numbers + +`tests/math_test.sh:42` + +``` +Expected '4' +but got '5' +``` +``` + +Inside GitHub Actions there is **nothing to configure**: with +`GITHUB_STEP_SUMMARY` set and no explicit path, the summary is appended to it +and renders on the job page. Appended, never written, because that file belongs +to the whole job and truncating it would discard the other steps' output. An +explicit `--report-md` path wins over the step summary. + +The report always carries the verdict, a counts table and the failures with +their `file:line` and message. Two sections appear only when the data exists: +coverage percentage after a `--coverage` run, and the slowest tests under +[`--profile`](#profile). + +Failure messages are ANSI-stripped and go inside a fence, so they render +verbatim. Test names are escaped, so a name containing `|`, `*`, `_` or a +backtick cannot break the table. + +Like the annotations, only the outermost run writes the step summary: a nested +bashunit run inherits `GITHUB_STEP_SUMMARY` and would otherwise append its own +fixtures' results to the parent's job page. + ### GitHub Actions annotations Inside GitHub Actions, bashunit annotates failing tests on the pull request by diff --git a/src/config/env.sh b/src/config/env.sh index 030a02d2..c56842f2 100644 --- a/src/config/env.sh +++ b/src/config/env.sh @@ -164,6 +164,7 @@ _BASHUNIT_DEFAULT_LOG_GHA="" _BASHUNIT_DEFAULT_REPORT_HTML="" _BASHUNIT_DEFAULT_REPORT_TAP="" _BASHUNIT_DEFAULT_REPORT_JSON="" +_BASHUNIT_DEFAULT_REPORT_MD="" # Coverage defaults (following kcov, bashcov, SimpleCov conventions) _BASHUNIT_DEFAULT_COVERAGE="false" @@ -323,6 +324,8 @@ _BASHUNIT_DEFAULT_SNAPSHOT_REPORT_UNUSED="false" : "${BASHUNIT_FAIL_ON_FLAKY:=$_BASHUNIT_DEFAULT_FAIL_ON_FLAKY}" : "${BASHUNIT_REPEAT:=$_BASHUNIT_DEFAULT_REPEAT}" : "${BASHUNIT_GHA_ANNOTATIONS:=$_BASHUNIT_DEFAULT_GHA_ANNOTATIONS}" +# No bare REPORT_MD alias: the newer report flags never grew one. +: "${BASHUNIT_REPORT_MD:=$_BASHUNIT_DEFAULT_REPORT_MD}" # GITHUB_ACTIONS is inherited by every child process, so a nested bashunit run # (bashunit's own acceptance suite, or a user's script under test that calls @@ -672,6 +675,18 @@ function bashunit::env::should_print_gha_annotations() { ! bashunit::env::is_tap_output_enabled } +## +# Whether the Markdown summary is appended to the job's step summary. Same +# outermost-run posture as the annotations: GITHUB_STEP_SUMMARY is inherited by +# every child, so a nested run would append its own fixtures' results to the +# parent's job page. +## +function bashunit::env::should_append_step_summary() { + [ -n "${GITHUB_STEP_SUMMARY:-}" ] || return 1 + + [ "${_BASHUNIT_IS_OUTERMOST_RUN:-true}" = true ] +} + function bashunit::env::is_fail_on_flaky_enabled() { [ "${BASHUNIT_FAIL_ON_FLAKY:-false}" = "true" ] } diff --git a/src/console/header.sh b/src/console/header.sh index d7f5b060..13f76937 100644 --- a/src/console/header.sh +++ b/src/console/header.sh @@ -133,6 +133,7 @@ Options: -r, --report-html Write HTML report --report-tap Write TAP version 13 report --report-json Write machine-readable JSON report + --report-md Write a Markdown summary (auto-appended to \$GITHUB_STEP_SUMMARY) -s, --simple Simple output (dots) --detailed Detailed output (default) --output Output format: tap (TAP version 13) diff --git a/src/main/run.sh b/src/main/run.sh index a12ada9b..95b9dd75 100644 --- a/src/main/run.sh +++ b/src/main/run.sh @@ -148,6 +148,16 @@ function bashunit::main::exec_tests() { bashunit::console_results::render_result exit_code=$? + # Rows first, and the Markdown summary before print_profile_and_reset below + # removes the profile records that summary reads. + bashunit::reports::load_spooled + + if [ -n "$BASHUNIT_REPORT_MD" ]; then + bashunit::reports::generate_report_md "$BASHUNIT_REPORT_MD" + elif bashunit::env::should_append_step_summary; then + bashunit::reports::append_step_summary + fi + if bashunit::env::is_profile_enabled; then bashunit::console_results::print_profile_and_reset fi @@ -158,8 +168,6 @@ function bashunit::main::exec_tests() { bashunit::snapshot::report_unused ${test_files[@]+"${test_files[@]}"} fi - bashunit::reports::load_spooled - # To stdout, not to a file: GitHub reads workflow commands from the job log. # After load_spooled so a --parallel run annotates the rows its workers # spooled, which the parent would otherwise never have seen (#1004). diff --git a/src/main/test.sh b/src/main/test.sh index 38c15e4a..05abe74a 100644 --- a/src/main/test.sh +++ b/src/main/test.sh @@ -255,6 +255,11 @@ function bashunit::main::cmd_test() { export -n BASHUNIT_REPORT_TAP shift ;; + --report-md) + BASHUNIT_REPORT_MD="$2" + export -n BASHUNIT_REPORT_MD + shift + ;; --report-json) BASHUNIT_REPORT_JSON="$2" export -n BASHUNIT_REPORT_JSON diff --git a/src/main/validate.sh b/src/main/validate.sh index 4dc3845a..ba4df117 100644 --- a/src/main/validate.sh +++ b/src/main/validate.sh @@ -135,7 +135,7 @@ function bashunit::main::validate_config_or_exit() { local _report_var _report_path for _report_var in BASHUNIT_LOG_JUNIT BASHUNIT_LOG_GHA BASHUNIT_REPORT_HTML \ - BASHUNIT_REPORT_TAP BASHUNIT_REPORT_JSON; do + BASHUNIT_REPORT_TAP BASHUNIT_REPORT_JSON BASHUNIT_REPORT_MD; do _report_path=${!_report_var:-} if [ -n "$_report_path" ]; then bashunit::main::require_writable_path_or_exit "$_report_path" "$_report_var" diff --git a/src/reports/collect.sh b/src/reports/collect.sh index 2d542684..6c4ec79d 100644 --- a/src/reports/collect.sh +++ b/src/reports/collect.sh @@ -61,6 +61,8 @@ function bashunit::reports::is_enabled() { [ -n "${BASHUNIT_LOG_GHA:-}" ] || [ -n "${BASHUNIT_REPORT_TAP:-}" ] || [ -n "${BASHUNIT_REPORT_JSON:-}" ] || + [ -n "${BASHUNIT_REPORT_MD:-}" ] || + bashunit::env::should_append_step_summary || bashunit::env::should_print_gha_annotations } diff --git a/src/reports/index.sh b/src/reports/index.sh index d1a72ab4..21207129 100644 --- a/src/reports/index.sh +++ b/src/reports/index.sh @@ -11,3 +11,4 @@ source "$BASHUNIT_ROOT_DIR/src/reports/tap.sh" source "$BASHUNIT_ROOT_DIR/src/reports/json.sh" source "$BASHUNIT_ROOT_DIR/src/reports/gha.sh" source "$BASHUNIT_ROOT_DIR/src/reports/html.sh" +source "$BASHUNIT_ROOT_DIR/src/reports/markdown.sh" diff --git a/src/reports/markdown.sh b/src/reports/markdown.sh new file mode 100644 index 00000000..eae4d3d3 --- /dev/null +++ b/src/reports/markdown.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash + +# Markdown run summary, for the one surface aimed at a human rather than a +# machine: the pull request page. GitHub Actions renders anything appended to +# $GITHUB_STEP_SUMMARY as Markdown on the job page; GitLab and Azure have +# equivalents that consume the same file. + +## +# Escapes the Markdown metacharacters that would otherwise change the rendering +# of a value taken from a test name. The backslash goes first, or it would +# escape the escapes added after it. +## +function bashunit::reports::__md_escape() { + local text="$1" + text=$(bashunit::reports::__strip_ansi "$text") + text="${text//\\/\\\\}" + text="${text//|/\\|}" + text="${text//\`/\\\`}" + text="${text//\*/\\*}" + text="${text//_/\\_}" + printf '%s' "$text" +} + +## +# Emits one `| Label | N |` row, but only when the count is non-zero. Passed and +# failed are always printed by the caller: a run with neither is the interesting +# case, not a row worth hiding. +## +function bashunit::reports::__md_count_row() { + local label=$1 + local count=$2 + if [ "${count:-0}" -gt 0 ]; then + printf '| %s | %s |\n' "$label" "$count" + fi +} + +## +# Writes the whole summary to stdout. Callers redirect: `>` for an explicit +# --report-md path, `>>` for the step summary, which belongs to the whole job. +## +function bashunit::reports::print_report_md() { + local passed failed skipped incomplete snapshot risky flaky duration_ms + passed=$(bashunit::state::get_tests_passed) + failed=$(bashunit::state::get_tests_failed) + skipped=$(bashunit::state::get_tests_skipped) + incomplete=$(bashunit::state::get_tests_incomplete) + snapshot=$(bashunit::state::get_tests_snapshot) + risky=$(bashunit::state::get_tests_risky) + flaky=$(bashunit::state::get_tests_flaky) + duration_ms=$(bashunit::clock::total_runtime_in_milliseconds) + + local duration + duration=$(bashunit::console_results::format_duration "$duration_ms") + + echo "## bashunit" + echo "" + if [ "${failed:-0}" -gt 0 ]; then + printf '❌ **%s failed**, %s passed in %s\n' "$failed" "$passed" "$duration" + else + printf '✅ **%s passed** in %s\n' "$passed" "$duration" + fi + echo "" + + echo "| Result | Count |" + echo "|--------|-------|" + printf '| Passed | %s |\n' "$passed" + printf '| Failed | %s |\n' "$failed" + bashunit::reports::__md_count_row "Skipped" "$skipped" + bashunit::reports::__md_count_row "Incomplete" "$incomplete" + bashunit::reports::__md_count_row "Snapshot" "$snapshot" + bashunit::reports::__md_count_row "Risky" "$risky" + bashunit::reports::__md_count_row "Flaky" "$flaky" + echo "" + + bashunit::reports::__md_failures + bashunit::reports::__md_coverage + bashunit::reports::__md_profile +} + +## +# The section that saves a click into the raw log: name, location and the +# failure message verbatim inside a fence. +## +function bashunit::reports::__md_failures() { + local i any=false + for i in "${!_BASHUNIT_REPORTS_TEST_NAMES[@]}"; do + case "${_BASHUNIT_REPORTS_TEST_STATUSES[$i]:-}" in + failed) ;; + *) continue ;; + esac + + if [ "$any" = false ]; then + echo "## Failures" + echo "" + any=true + fi + + local name file line message + name=$(bashunit::reports::__md_escape "${_BASHUNIT_REPORTS_TEST_NAMES[$i]:-}") + file="${_BASHUNIT_REPORTS_TEST_FILES[$i]:-}" + line="${_BASHUNIT_REPORTS_TEST_LINES[$i]:-}" + message=$(bashunit::reports::__strip_ansi "${_BASHUNIT_REPORTS_TEST_FAILURES[$i]:-}") + + printf '### %s\n\n' "$name" + if [ -n "$line" ]; then + printf '`%s:%s`\n\n' "$file" "$line" + else + printf '`%s`\n\n' "$file" + fi + # Not escaped: a fence renders its contents literally, which is the point. + echo '```' + printf '%s\n' "$message" + echo '```' + echo "" + done +} + +function bashunit::reports::__md_coverage() { + if [ "${_BASHUNIT_COVERAGE_ON:-0}" != 1 ]; then + return 0 + fi + + local pct + pct=$(bashunit::coverage::get_percentage 2>/dev/null) || return 0 + [ -n "$pct" ] || return 0 + + echo "## Coverage" + echo "" + printf '%s%% of tracked lines\n' "$pct" + echo "" +} + +## +# Reads the same tab-separated records the console profile renders. Must run +# before print_profile_and_reset, which removes the file. +## +function bashunit::reports::__md_profile() { + if ! bashunit::env::is_profile_enabled; then + return 0 + fi + [ -s "${PROFILE_OUTPUT_PATH:-}" ] || return 0 + + echo "## Slowest tests" + echo "" + echo "| Duration | Test | File |" + echo "|----------|------|------|" + + local duration name file formatted + while IFS=$'\t' read -r duration name file; do + formatted=$(bashunit::console_results::format_duration "$duration") + printf '| %s | %s | %s |\n' \ + "$formatted" \ + "$(bashunit::reports::__md_escape "$name")" \ + "$(bashunit::reports::__md_escape "$file")" + done < <(sort -t"$(printf '\t')" -k1 -rn "$PROFILE_OUTPUT_PATH" | head -n "${BASHUNIT_PROFILE_COUNT:-10}") + echo "" +} + +function bashunit::reports::generate_report_md() { + local output_file="$1" + + bashunit::reports::print_report_md >"$output_file" +} + +## +# Appends to $GITHUB_STEP_SUMMARY. Appending, never writing: the file is shared +# with every other step in the job, so truncating it would discard their output. +## +function bashunit::reports::append_step_summary() { + [ -n "${GITHUB_STEP_SUMMARY:-}" ] || return 0 + + bashunit::reports::print_report_md >>"$GITHUB_STEP_SUMMARY" +} diff --git a/tests/acceptance/bashunit_report_md_test.sh b/tests/acceptance/bashunit_report_md_test.sh new file mode 100644 index 00000000..861dac1d --- /dev/null +++ b/tests/acceptance/bashunit_report_md_test.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash + +# The Markdown summary targets the pull request page rather than a machine, so +# these cases assert what a reader would see, end to end. +# +# GITHUB_STEP_SUMMARY is inherited by every child process, so a nested run would +# append its own fixtures' results to the parent's job page. Clearing the +# outermost-run claim marker is how a test says "pretend I am the top-level run". + +function set_up_before_script() { + TEST_ENV_FILE="tests/acceptance/fixtures/.env.default" + FIXTURE="./tests/acceptance/fixtures/test_bashunit_when_log_junit.sh" + PASSING_FIXTURE="./tests/acceptance/fixtures/test_bashunit_when_a_test_passes.sh" +} + +function set_up() { + REPORT_DIR="$(bashunit::temp_dir report_md)" +} + +function test_report_md_writes_a_markdown_summary() { + ./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --report-md "$REPORT_DIR/summary.md" "$FIXTURE" >/dev/null 2>&1 || true + + local content + content="$(cat "$REPORT_DIR/summary.md")" + + assert_contains "## bashunit" "$content" + assert_contains "| Result | Count |" "$content" +} + +function test_a_failing_run_lists_the_failure_with_its_location() { + ./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --report-md "$REPORT_DIR/summary.md" "$FIXTURE" >/dev/null 2>&1 || true + + local content + content="$(cat "$REPORT_DIR/summary.md")" + + assert_contains "## Failures" "$content" + assert_contains "$FIXTURE:" "$content" + assert_contains "but got" "$content" +} + +function test_a_green_run_has_no_failures_section() { + ./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --report-md "$REPORT_DIR/summary.md" "$PASSING_FIXTURE" >/dev/null 2>&1 + + local content + content="$(cat "$REPORT_DIR/summary.md")" + + assert_contains "passed" "$content" + assert_not_contains "## Failures" "$content" +} + +function test_the_step_summary_is_appended_when_no_path_is_given() { + local summary="$REPORT_DIR/step.md" + printf 'previous step output\n' >"$summary" + + _BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_STEP_SUMMARY="$summary" \ + ./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" "$FIXTURE" >/dev/null 2>&1 || true + + local content + content="$(cat "$summary")" + + assert_contains "previous step output" "$content" + assert_contains "## bashunit" "$content" +} + +function test_an_explicit_path_wins_over_the_step_summary() { + local summary="$REPORT_DIR/step.md" + : >"$summary" + + _BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_STEP_SUMMARY="$summary" \ + ./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --report-md "$REPORT_DIR/summary.md" "$FIXTURE" >/dev/null 2>&1 || true + + assert_contains "## bashunit" "$(cat "$REPORT_DIR/summary.md")" + assert_empty "$(cat "$summary")" +} + +# A nested run must not append its own results to the parent's job page. +function test_a_nested_run_leaves_the_step_summary_alone() { + local summary="$REPORT_DIR/step.md" + : >"$summary" + + GITHUB_STEP_SUMMARY="$summary" ./bashunit --no-parallel --no-color \ + --env "$TEST_ENV_FILE" "$FIXTURE" >/dev/null 2>&1 || true + + assert_empty "$(cat "$summary")" +} + +function test_the_report_is_populated_under_parallel() { + ./bashunit --parallel --no-color --env "$TEST_ENV_FILE" \ + --report-md "$REPORT_DIR/summary.md" "$FIXTURE" >/dev/null 2>&1 || true + + assert_contains "## Failures" "$(cat "$REPORT_DIR/summary.md")" +} + +function test_the_slowest_tests_appear_only_with_profile() { + ./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --report-md "$REPORT_DIR/plain.md" "$PASSING_FIXTURE" >/dev/null 2>&1 + ./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" --profile \ + --report-md "$REPORT_DIR/profiled.md" "$PASSING_FIXTURE" >/dev/null 2>&1 + + assert_not_contains "## Slowest tests" "$(cat "$REPORT_DIR/plain.md")" + assert_contains "## Slowest tests" "$(cat "$REPORT_DIR/profiled.md")" +} diff --git a/tests/unit/reports/report_md_test.sh b/tests/unit/reports/report_md_test.sh new file mode 100644 index 00000000..4c7af3e8 --- /dev/null +++ b/tests/unit/reports/report_md_test.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2034,SC2329 # arrays and mocks are read indirectly + +function set_up() { + _BASHUNIT_REPORTS_TEST_FILES=() + _BASHUNIT_REPORTS_TEST_NAMES=() + _BASHUNIT_REPORTS_TEST_STATUSES=() + _BASHUNIT_REPORTS_TEST_DURATIONS=() + _BASHUNIT_REPORTS_TEST_ASSERTIONS=() + _BASHUNIT_REPORTS_TEST_FAILURES=() + _BASHUNIT_REPORTS_TEST_LINES=() + _BASHUNIT_REPORTS_TEST_RETRIES=() + _MD_OUTPUT_FILE=$(mktemp) +} + +function tear_down() { + rm -f "$_MD_OUTPUT_FILE" + unset BASHUNIT_REPORT_MD GITHUB_STEP_SUMMARY +} + +function _mock_counters() { + function bashunit::state::get_tests_passed() { echo "${_MOCK_PASSED:-0}"; } + function bashunit::state::get_tests_failed() { echo "${_MOCK_FAILED:-0}"; } + function bashunit::state::get_tests_skipped() { echo "0"; } + function bashunit::state::get_tests_incomplete() { echo "0"; } + function bashunit::state::get_tests_snapshot() { echo "0"; } + function bashunit::state::get_tests_risky() { echo "0"; } + function bashunit::state::get_tests_flaky() { echo "0"; } + function bashunit::clock::total_runtime_in_milliseconds() { echo "1234"; } +} + +function _green_run_fixture() { + _mock_counters + _MOCK_PASSED=2 + _MOCK_FAILED=0 + _BASHUNIT_REPORTS_TEST_FILES=("tests/math_test.sh" "tests/math_test.sh") + _BASHUNIT_REPORTS_TEST_NAMES=("it adds" "it divides") + _BASHUNIT_REPORTS_TEST_STATUSES=("passed" "passed") + _BASHUNIT_REPORTS_TEST_DURATIONS=("5" "3") + _BASHUNIT_REPORTS_TEST_ASSERTIONS=("1" "1") + _BASHUNIT_REPORTS_TEST_FAILURES=("" "") + _BASHUNIT_REPORTS_TEST_LINES=("10" "20") + _BASHUNIT_REPORTS_TEST_RETRIES=("0" "0") +} + +function _red_run_fixture() { + _mock_counters + _MOCK_PASSED=1 + _MOCK_FAILED=1 + _BASHUNIT_REPORTS_TEST_FILES=("tests/math_test.sh" "tests/math_test.sh") + _BASHUNIT_REPORTS_TEST_NAMES=("it adds" "it divides") + _BASHUNIT_REPORTS_TEST_STATUSES=("passed" "failed") + _BASHUNIT_REPORTS_TEST_DURATIONS=("5" "3") + _BASHUNIT_REPORTS_TEST_ASSERTIONS=("1" "1") + _BASHUNIT_REPORTS_TEST_FAILURES=("" "$(printf "\033[31mExpected '4'\033[0m\nbut got '5'")") + _BASHUNIT_REPORTS_TEST_LINES=("10" "20") + _BASHUNIT_REPORTS_TEST_RETRIES=("0" "0") +} + +function test_a_green_run_leads_with_a_passing_verdict() { + _green_run_fixture + bashunit::reports::generate_report_md "$_MD_OUTPUT_FILE" + + assert_contains "2 passed" "$(cat "$_MD_OUTPUT_FILE")" +} + +function test_a_green_run_has_no_failures_section() { + _green_run_fixture + bashunit::reports::generate_report_md "$_MD_OUTPUT_FILE" + + assert_not_contains "## Failures" "$(cat "$_MD_OUTPUT_FILE")" +} + +function test_a_red_run_leads_with_a_failing_verdict() { + _red_run_fixture + bashunit::reports::generate_report_md "$_MD_OUTPUT_FILE" + + assert_contains "1 failed" "$(cat "$_MD_OUTPUT_FILE")" +} + +function test_the_failure_message_is_fenced_and_ansi_free() { + _red_run_fixture + bashunit::reports::generate_report_md "$_MD_OUTPUT_FILE" + + local content + content=$(cat "$_MD_OUTPUT_FILE") + + assert_contains '```' "$content" + assert_contains "but got '5'" "$content" + assert_not_contains $'\e[' "$content" +} + +function test_the_failure_names_its_file_and_line() { + _red_run_fixture + bashunit::reports::generate_report_md "$_MD_OUTPUT_FILE" + + assert_contains "tests/math_test.sh:20" "$(cat "$_MD_OUTPUT_FILE")" +} + +# A pipe would split a table cell, and the rest would render as emphasis or code. +function test_markdown_metacharacters_in_a_test_name_are_escaped() { + _red_run_fixture + _BASHUNIT_REPORTS_TEST_NAMES=("it adds" 'a|b *c* _d_ `e`') + + bashunit::reports::generate_report_md "$_MD_OUTPUT_FILE" + + local content + content=$(cat "$_MD_OUTPUT_FILE") + + assert_contains 'a\|b \*c\* \_d\_ \`e\`' "$content" +} + +function test_the_counts_table_reports_the_totals() { + _green_run_fixture + bashunit::reports::generate_report_md "$_MD_OUTPUT_FILE" + + local content + content=$(cat "$_MD_OUTPUT_FILE") + + assert_contains "| Passed | 2 |" "$content" + assert_contains "| Failed | 0 |" "$content" +} + +# The step summary is shared with every other step in the job, so writing it +# would discard whatever ran before. +function test_the_step_summary_is_appended_not_truncated() { + _green_run_fixture + printf 'existing content\n' >"$_MD_OUTPUT_FILE" + GITHUB_STEP_SUMMARY="$_MD_OUTPUT_FILE" + + bashunit::reports::append_step_summary + + local content + content=$(cat "$_MD_OUTPUT_FILE") + + assert_contains "existing content" "$content" + assert_contains "2 passed" "$content" +} From 80160040dc3df3e5873dbb6e7ab7250271b65316 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Mon, 10 Aug 2026 20:45:23 +0200 Subject: [PATCH 2/3] test(reports): isolate is_enabled from the ambient step summary GITHUB_STEP_SUMMARY is set on every GitHub Actions runner, which makes the Markdown summary a third reason to collect rows. These tests ask whether a file report was configured, so the ambient CI environment has to be pinned off for the answer to mean anything. --- tests/unit/reports/reports_test.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/unit/reports/reports_test.sh b/tests/unit/reports/reports_test.sh index 1f10d738..94212ddc 100644 --- a/tests/unit/reports/reports_test.sh +++ b/tests/unit/reports/reports_test.sh @@ -25,10 +25,12 @@ function set_up() { unset BASHUNIT_REPORT_TAP # These tests ask whether a *file* report was configured. Stdout annotations - # are a second reason to collect rows, and they switch themselves on inside - # GitHub Actions, so pin them off to keep the answer independent of the - # ambient CI environment. + # and the Markdown step summary are two further reasons to collect rows, and + # both switch themselves on inside GitHub Actions, so pin them off to keep the + # answer independent of the ambient CI environment. export BASHUNIT_GHA_ANNOTATIONS=never + unset GITHUB_STEP_SUMMARY + unset BASHUNIT_REPORT_MD # Create temp file for output tests _TEMP_OUTPUT_FILE=$(mktemp) @@ -43,6 +45,7 @@ function tear_down() { unset BASHUNIT_REPORT_HTML unset BASHUNIT_LOG_GHA unset BASHUNIT_GHA_ANNOTATIONS + unset BASHUNIT_REPORT_MD } function _reports_is_enabled_state() { From 341da2a42952c09fc8bb0eec26aaf91fb0dd81da Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Mon, 10 Aug 2026 21:42:42 +0200 Subject: [PATCH 3/3] fix(reports): the Markdown coverage percentage matched neither the console nor parallel The summary read the coverage percentage before the hit records were finalized, and before a parallel run's per-worker data was aggregated, so it reported 0% for a covered run under --parallel and a low figure sequentially. Compute the coverage numbers before the summary is written; the reports still render where they did. --- src/main/run.sh | 27 ++++++++++++--------- tests/acceptance/bashunit_report_md_test.sh | 15 ++++++++++++ 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/main/run.sh b/src/main/run.sh index 95b9dd75..782db494 100644 --- a/src/main/run.sh +++ b/src/main/run.sh @@ -152,6 +152,21 @@ function bashunit::main::exec_tests() { # removes the profile records that summary reads. bashunit::reports::load_spooled + # Coverage data is turned into numbers here rather than with the reports + # below, because the Markdown summary quotes the same percentage and would + # otherwise read it before the hit records exist — 0% for a covered run. + if bashunit::env::is_coverage_enabled; then + # Turn captured xtrace output into hit records (no-op for the trap engine) + bashunit::coverage::finalize + + # Aggregate per-process coverage data from parallel runs + if bashunit::parallel::is_enabled; then + bashunit::coverage::aggregate_parallel + fi + + bashunit::coverage::precompute_file_stats + fi + if [ -n "$BASHUNIT_REPORT_MD" ]; then bashunit::reports::generate_report_md "$BASHUNIT_REPORT_MD" elif bashunit::env::should_append_step_summary; then @@ -195,18 +210,8 @@ function bashunit::main::exec_tests() { bashunit::reports::generate_report_json "$BASHUNIT_REPORT_JSON" fi - # Generate coverage report if enabled + # Render the coverage reports; the data behind them was computed above. if bashunit::env::is_coverage_enabled; then - # Turn captured xtrace output into hit records (no-op for the trap engine) - bashunit::coverage::finalize - - # Aggregate per-process coverage data from parallel runs - if bashunit::parallel::is_enabled; then - bashunit::coverage::aggregate_parallel - fi - - bashunit::coverage::precompute_file_stats - if bashunit::coverage::is_diff_enabled; then bashunit::coverage::report_diff else diff --git a/tests/acceptance/bashunit_report_md_test.sh b/tests/acceptance/bashunit_report_md_test.sh index 861dac1d..454467a0 100644 --- a/tests/acceptance/bashunit_report_md_test.sh +++ b/tests/acceptance/bashunit_report_md_test.sh @@ -95,6 +95,21 @@ function test_the_report_is_populated_under_parallel() { assert_contains "## Failures" "$(cat "$REPORT_DIR/summary.md")" } +# The summary's percentage is the console report's percentage. It used to be +# read before the hit records were finalized and, under --parallel, before the +# workers' data was aggregated, which reported 0% for a covered run. +function test_the_coverage_percentage_matches_the_console_report() { + local output + output=$(./bashunit --parallel --no-color --env "$TEST_ENV_FILE" --coverage \ + --report-md "$REPORT_DIR/summary.md" "$PASSING_FIXTURE" 2>/dev/null) + + local console_pct + console_pct=$(echo "$output" | grep "^Total:" | sed 's/.*(\([0-9]*\)%).*/\1/') + + assert_not_empty "$console_pct" + assert_contains "$console_pct% of tracked lines" "$(cat "$REPORT_DIR/summary.md")" +} + function test_the_slowest_tests_appear_only_with_profile() { ./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ --report-md "$REPORT_DIR/plain.md" "$PASSING_FIXTURE" >/dev/null 2>&1