diff --git a/.env.example b/.env.example index d2ea2eea..c1bbbf5e 100644 --- a/.env.example +++ b/.env.example @@ -62,6 +62,7 @@ BASHUNIT_LOGIN_SHELL= # Default: false (source login shell profile BASHUNIT_LOG_JUNIT= # JUnit XML report path (e.g., report.xml) BASHUNIT_LOG_GHA= # GitHub Actions workflow-commands log path (e.g., gha.log) BASHUNIT_REPORT_HTML= # HTML test report path (e.g., report.html) +BASHUNIT_REPORT_MD= # Markdown summary path (e.g., summary.md); unset: appends to $GITHUB_STEP_SUMMARY #─────────────────────────────────────────────────────────────────────────────── # Code Coverage diff --git a/CHANGELOG.md b/CHANGELOG.md index d027617e..20828e77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased ### Added +- `--report-md ` writes a human-readable Markdown summary: verdict, counts table, failures with fenced messages, coverage percentage when `--coverage` ran and the slowest tests under `--profile`. Inside GitHub Actions the summary is appended to `$GITHUB_STEP_SUMMARY` automatically when no explicit path is given (#1015) - `--coverage-diff ` restricts the coverage console report to lines changed since a base ref, and `--coverage-min` then gates on that diff percentage (#1032) - `--gha-annotations ` controls GitHub Actions annotations on stdout; `auto` turns them on inside GitHub Actions and stays quiet everywhere else (#1014) - `--repeat ` runs each selected test n times so flakiness can be hunted before it reaches CI. The test is reported once with the aggregate outcome, a failure names the iteration it happened on, and repeat wraps `--retry` rather than the other way round (#1013) diff --git a/adrs/adr-011-source-layout-and-build-pipeline.md b/adrs/adr-011-source-layout-and-build-pipeline.md index 9fb34ee5..598f2e43 100644 --- a/adrs/adr-011-source-layout-and-build-pipeline.md +++ b/adrs/adr-011-source-layout-and-build-pipeline.md @@ -60,13 +60,13 @@ Seventeen, in load order. The order is the dependency layering: leaves first. | 10 | `cli/` | 5 | 445 | the `doc`/`init`/`upgrade`/`watch` subcommand implementations | | 11 | `assert/` | 11 | 2326 | every assertion | | 12 | `doubles/` | 4 | 505 | spies and mocks — **sourced by `assert/index.sh`, not the entrypoint** | -| 13 | `reports/` | 7 | 465 | JUnit, TAP, JSON, GHA and HTML writers | +| 13 | `reports/` | 8 | 693 | JUnit, TAP, JSON, GHA, HTML and Markdown writers | | 14 | `runner/` | 12 | 2434 | the file loop, per-test execution, retry, result parsing, `--list` | | 15 | `benchmark/` | 4 | 219 | the bench implementation (`runner/bench.sh` is its loop) | | 16 | `learn/` | 14 | 1296 | the interactive tutorial (9 of those files are `learn/lessons/`) | | 17 | `main/` | 8 | 1473 | flag parsing per subcommand and the run lifecycle | -The file counts sum to 118, which is every `.sh` file in `src/` — that is the check to re-run +The file counts sum to 121, which is every `.sh` file in `src/` — that is the check to re-run when editing this table, because both of its previous errors came from counting the wrong way. `doubles/` was absent entirely: the table was generated from the entrypoint's `source` lines, and `doubles/` is the one module the entrypoint does not source, so it fell through while the diff --git a/completions/_bashunit b/completions/_bashunit index cfb8519d..23e980da 100644 --- a/completions/_bashunit +++ b/completions/_bashunit @@ -78,6 +78,7 @@ _bashunit() { '(-r --report-html)'{-r,--report-html}'[Write HTML report]:file:_files' \ '--report-tap[Write TAP version 13 report]:file:_files' \ '--report-json[Write machine-readable JSON report]:file:_files' \ + '--report-md[Write Markdown summary report]:file:_files' \ '(-s --simple)'{-s,--simple}'[Simple output with dots]' \ '--detailed[Detailed output, the default]' \ '--output[Output format]:format:(tap)' \ diff --git a/completions/bashunit.bash b/completions/bashunit.bash index 3e000baa..c52cc3af 100644 --- a/completions/bashunit.bash +++ b/completions/bashunit.bash @@ -21,7 +21,7 @@ _BASHUNIT_COMPLETIONS_TEST_OPTS="--assert --boot --changed --coverage --coverage --no-coverage-report --no-output --no-output-on-failure --no-parallel \ --no-progress --no-snapshot-create --order-by --output --parallel --profile \ --random-order --repeat --report-html \ ---report-json --report-junit --report-tap --rerun-failed --retry --run-all \ +--report-json --report-junit --report-md --report-tap --rerun-failed --retry --run-all \ --seed --shard --show-incomplete --show-output --show-skipped --simple \ --skip-env-file --snapshot-report-unused --snapshot-update \ --stop-on-failure --strict --tag \ diff --git a/docs/command-line.md b/docs/command-line.md index 6b378a28..55270945 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -75,6 +75,7 @@ bashunit test tests/ --parallel --simple | `-r, --report-html ` | Write HTML report | | `--report-tap ` | Write TAP version 13 report to a file | | `--report-json ` | Write machine-readable JSON report to a file | +| `--report-md ` | Write Markdown summary to a file (automatic on GitHub Actions) | | `-R, --run-all` | Run all assertions (don't stop on first failure) | | `-s, --simple` | Simple output (dots) | | `--detailed` | Detailed output (default) | @@ -456,6 +457,9 @@ bashunit test tests/ ```bash [JSON] bashunit test tests/ --report-json report.json ``` +```bash [Markdown] +bashunit test tests/ --report-md summary.md +``` ::: ### GitHub Actions annotations @@ -507,6 +511,21 @@ The `--report-json` flag writes machine-readable results for scripts, dashboards `status` is one of `passed`, `failed`, `skipped`, `incomplete` (`snapshot` and `risky` are also emitted per test and counted as passed in the summary). Like the other file reporters, per-test rows come from a sequential run; under `--parallel` the file is still valid JSON. +### Markdown summary + +> `bashunit test --report-md ` + +Every other report format targets a machine; `--report-md` targets the place a developer looks first. It writes a human-readable Markdown summary: a one-line verdict with the total duration, a counts table, and each failure's name, `file:line` and message in a fenced code block. The coverage percentage is included when `--coverage` ran, and the slowest tests when `--profile` is on (`BASHUNIT_PROFILE_COUNT` entries, default 10). + +Inside GitHub Actions there is nothing to configure: when `$GITHUB_STEP_SUMMARY` is set and no explicit `--report-md` path is given, the summary is appended to the job's step summary automatically, so it renders directly on the workflow run page. Only the outermost bashunit process writes it — nested runs (a script under test that itself calls bashunit) stay quiet. + +```bash +bashunit test tests/ --report-md summary.md + +# In a workflow: no flag needed, the step summary is filled automatically +bashunit test tests/ +``` + ### Show Output on Failure > `bashunit test --show-output` diff --git a/src/config/env.sh b/src/config/env.sh index 030a02d2..8d7c0d7d 100644 --- a/src/config/env.sh +++ b/src/config/env.sh @@ -164,6 +164,7 @@ _BASHUNIT_DEFAULT_LOG_GHA="" _BASHUNIT_DEFAULT_REPORT_HTML="" _BASHUNIT_DEFAULT_REPORT_TAP="" _BASHUNIT_DEFAULT_REPORT_JSON="" +_BASHUNIT_DEFAULT_REPORT_MD="" # Coverage defaults (following kcov, bashcov, SimpleCov conventions) _BASHUNIT_DEFAULT_COVERAGE="false" @@ -191,6 +192,9 @@ _BASHUNIT_DEFAULT_COVERAGE_DIFF="" : "${BASHUNIT_REPORT_HTML:=${REPORT_HTML:=$_BASHUNIT_DEFAULT_REPORT_HTML}}" : "${BASHUNIT_REPORT_TAP:=${REPORT_TAP:=$_BASHUNIT_DEFAULT_REPORT_TAP}}" : "${BASHUNIT_REPORT_JSON:=${REPORT_JSON:=$_BASHUNIT_DEFAULT_REPORT_JSON}}" +# No bare REPORT_MD alias: the unprefixed forms are deprecated, so a new +# setting only ever ships under the BASHUNIT_ prefix. +: "${BASHUNIT_REPORT_MD:=$_BASHUNIT_DEFAULT_REPORT_MD}" # Watch mode polling interval (seconds) used by the pure-shell fallback _BASHUNIT_DEFAULT_WATCH_INTERVAL="2" @@ -672,6 +676,18 @@ function bashunit::env::should_print_gha_annotations() { ! bashunit::env::is_tap_output_enabled } +## +# Whether the run should append its Markdown summary to $GITHUB_STEP_SUMMARY. +# Same ownership rule as the annotations above: GITHUB_STEP_SUMMARY is +# inherited by every child process, so only the outermost run may write to it. +# An explicit --report-md path takes over as the destination instead. +## +function bashunit::env::should_write_github_step_summary() { + [ -z "${BASHUNIT_REPORT_MD:-}" ] && + [ "${_BASHUNIT_IS_OUTERMOST_RUN:-true}" = true ] && + [ -n "${GITHUB_STEP_SUMMARY:-}" ] +} + function bashunit::env::is_fail_on_flaky_enabled() { [ "${BASHUNIT_FAIL_ON_FLAKY:-false}" = "true" ] } diff --git a/src/console/header.sh b/src/console/header.sh index d7f5b060..2e9ed918 100644 --- a/src/console/header.sh +++ b/src/console/header.sh @@ -133,6 +133,7 @@ Options: -r, --report-html Write HTML report --report-tap Write TAP version 13 report --report-json Write machine-readable JSON report + --report-md Write Markdown summary (auto-appends to \$GITHUB_STEP_SUMMARY without the flag) -s, --simple Simple output (dots) --detailed Detailed output (default) --output Output format: tap (TAP version 13) diff --git a/src/main/run.sh b/src/main/run.sh index a12ada9b..5512ecf5 100644 --- a/src/main/run.sh +++ b/src/main/run.sh @@ -221,6 +221,14 @@ function bashunit::main::exec_tests() { bashunit::coverage::cleanup fi + # After the coverage block on purpose: the Markdown summary embeds the + # coverage percentage, which exists only once precompute_file_stats ran. + if [ -n "$BASHUNIT_REPORT_MD" ]; then + bashunit::reports::generate_report_md "$BASHUNIT_REPORT_MD" + elif bashunit::env::should_write_github_step_summary; then + bashunit::reports::append_github_step_summary "$GITHUB_STEP_SUMMARY" + fi + if bashunit::parallel::is_enabled; then bashunit::parallel::cleanup fi diff --git a/src/main/test.sh b/src/main/test.sh index 38c15e4a..79f6f072 100644 --- a/src/main/test.sh +++ b/src/main/test.sh @@ -260,6 +260,11 @@ function bashunit::main::cmd_test() { export -n BASHUNIT_REPORT_JSON shift ;; + --report-md) + BASHUNIT_REPORT_MD="$2" + export -n BASHUNIT_REPORT_MD + shift + ;; --no-output) BASHUNIT_NO_OUTPUT=true export -n BASHUNIT_NO_OUTPUT diff --git a/src/main/validate.sh b/src/main/validate.sh index 4dc3845a..ba4df117 100644 --- a/src/main/validate.sh +++ b/src/main/validate.sh @@ -135,7 +135,7 @@ function bashunit::main::validate_config_or_exit() { local _report_var _report_path for _report_var in BASHUNIT_LOG_JUNIT BASHUNIT_LOG_GHA BASHUNIT_REPORT_HTML \ - BASHUNIT_REPORT_TAP BASHUNIT_REPORT_JSON; do + BASHUNIT_REPORT_TAP BASHUNIT_REPORT_JSON BASHUNIT_REPORT_MD; do _report_path=${!_report_var:-} if [ -n "$_report_path" ]; then bashunit::main::require_writable_path_or_exit "$_report_path" "$_report_var" diff --git a/src/reports/collect.sh b/src/reports/collect.sh index 2d542684..2c9f7582 100644 --- a/src/reports/collect.sh +++ b/src/reports/collect.sh @@ -61,7 +61,9 @@ function bashunit::reports::is_enabled() { [ -n "${BASHUNIT_LOG_GHA:-}" ] || [ -n "${BASHUNIT_REPORT_TAP:-}" ] || [ -n "${BASHUNIT_REPORT_JSON:-}" ] || - bashunit::env::should_print_gha_annotations + [ -n "${BASHUNIT_REPORT_MD:-}" ] || + bashunit::env::should_print_gha_annotations || + bashunit::env::should_write_github_step_summary } function bashunit::reports::add_test() { diff --git a/src/reports/index.sh b/src/reports/index.sh index d1a72ab4..21207129 100644 --- a/src/reports/index.sh +++ b/src/reports/index.sh @@ -11,3 +11,4 @@ source "$BASHUNIT_ROOT_DIR/src/reports/tap.sh" source "$BASHUNIT_ROOT_DIR/src/reports/json.sh" source "$BASHUNIT_ROOT_DIR/src/reports/gha.sh" source "$BASHUNIT_ROOT_DIR/src/reports/html.sh" +source "$BASHUNIT_ROOT_DIR/src/reports/markdown.sh" diff --git a/src/reports/markdown.sh b/src/reports/markdown.sh new file mode 100644 index 00000000..c3bf6890 --- /dev/null +++ b/src/reports/markdown.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash + +# Markdown summary report writer. Aimed at humans on the pull-request page: +# GitHub renders anything appended to $GITHUB_STEP_SUMMARY, so the same +# document serves --report-md and the automatic step-summary append. + +# Escapes a string for Markdown prose and table cells: ANSI stripped first, +# then backslash before the others so the escapes added below are not doubled. +# Pipes must be escaped or a test name would end its table cell early. +function bashunit::reports::__md_escape() { + local text + text=$(bashunit::reports::__strip_ansi "$1") + text="${text//\\/\\\\}" + text="${text//\`/\\\`}" + text="${text//\*/\\*}" + text="${text//_/\\_}" + text="${text//|/\\|}" + printf '%s' "$text" +} + +## +# Prints the whole Markdown summary to stdout. The file and step-summary +# entry points below only differ in redirection, so the document itself is +# rendered in exactly one place. +## +function bashunit::reports::print_report_md() { + local passed failed skipped incomplete risky snapshot flaky + passed=$(bashunit::state::get_tests_passed) + failed=$(bashunit::state::get_tests_failed) + skipped=$(bashunit::state::get_tests_skipped) + incomplete=$(bashunit::state::get_tests_incomplete) + risky=$(bashunit::state::get_tests_risky) + snapshot=$(bashunit::state::get_tests_snapshot) + flaky=$(bashunit::state::get_tests_flaky) + + local time_ms time_s + time_ms=$(bashunit::clock::total_runtime_in_milliseconds) + # `env` rather than a bare `LC_ALL=C` prefix: C keeps awk's radix a dot, and + # that prefix form segfaults inside `$()` on Bash 5.3 macOS (#912). + time_s=$(env LC_ALL=C awk -v ms="$time_ms" 'BEGIN {printf "%.3f", ms/1000}') + + if [ "$failed" -gt 0 ]; then + printf '### ❌ %s failed, %s passed in %ss\n\n' "$failed" "$passed" "$time_s" + else + printf '### ✅ %s passed in %ss\n\n' "$passed" "$time_s" + fi + + printf '| Passed | Failed | Skipped | Incomplete | Risky | Snapshot | Flaky |\n' + printf '|---:|---:|---:|---:|---:|---:|---:|\n' + printf '| %s | %s | %s | %s | %s | %s | %s |\n\n' \ + "$passed" "$failed" "$skipped" "$incomplete" "$risky" "$snapshot" "$flaky" + + if [ "$failed" -gt 0 ]; then + bashunit::reports::__print_md_failures + fi + + if bashunit::env::is_coverage_enabled; then + printf '**Coverage:** %s%%\n\n' "$(bashunit::coverage::get_percentage)" + fi + + if bashunit::env::is_profile_enabled; then + bashunit::reports::__print_md_slowest + fi +} + +# The failures section: name, file:line and the message in a fenced block -- +# the part that saves a click into the raw job log. +function bashunit::reports::__print_md_failures() { + printf '#### Failures\n\n' + + local i + for i in "${!_BASHUNIT_REPORTS_TEST_NAMES[@]}"; do + [ "${_BASHUNIT_REPORTS_TEST_STATUSES[$i]:-}" = "failed" ] || continue + + local name location message + name=$(bashunit::reports::__md_escape "${_BASHUNIT_REPORTS_TEST_NAMES[$i]:-}") + location="${_BASHUNIT_REPORTS_TEST_FILES[$i]:-}" + if [ -n "${_BASHUNIT_REPORTS_TEST_LINES[$i]:-}" ]; then + location="$location:${_BASHUNIT_REPORTS_TEST_LINES[$i]}" + fi + message=$(bashunit::reports::__strip_ansi "${_BASHUNIT_REPORTS_TEST_FAILURES[$i]:-}") + + printf -- '- **%s** — %s\n\n' "$name" "$location" + printf '```\n%s\n```\n\n' "$message" + done +} + +# The slowest tests recorded in the report rows, duration descending, limited +# to BASHUNIT_PROFILE_COUNT. Reads the same rows every writer reads instead of +# PROFILE_OUTPUT_PATH, which print_profile_and_reset has already consumed. +function bashunit::reports::__print_md_slowest() { + [ "${#_BASHUNIT_REPORTS_TEST_NAMES[@]}" -gt 0 ] || return 0 + + printf '#### Slowest tests\n\n' + printf '| Duration (ms) | Test | File |\n' + printf '|---:|---|---|\n' + + local i + local duration name file + for i in "${!_BASHUNIT_REPORTS_TEST_NAMES[@]}"; do + printf '%s\t%s\t%s\n' \ + "${_BASHUNIT_REPORTS_TEST_DURATIONS[$i]:-0}" \ + "${_BASHUNIT_REPORTS_TEST_NAMES[$i]:-}" \ + "${_BASHUNIT_REPORTS_TEST_FILES[$i]:-}" + done \ + | sort -rn \ + | head -n "${BASHUNIT_PROFILE_COUNT:-10}" \ + | while IFS="$(printf '\t')" read -r duration name file; do + printf '| %s | %s | %s |\n' \ + "$duration" "$(bashunit::reports::__md_escape "$name")" "$file" + done + + printf '\n' +} + +function bashunit::reports::generate_report_md() { + local output_file="$1" + + bashunit::reports::print_report_md >"$output_file" +} + +# $GITHUB_STEP_SUMMARY aggregates every step of a job, so the summary is +# appended -- truncating would erase what earlier steps wrote. +function bashunit::reports::append_github_step_summary() { + local output_file="$1" + + bashunit::reports::print_report_md >>"$output_file" +} diff --git a/tests/acceptance/bashunit_gha_annotations_test.sh b/tests/acceptance/bashunit_gha_annotations_test.sh index 07bc265a..f0ae6ec2 100644 --- a/tests/acceptance/bashunit_gha_annotations_test.sh +++ b/tests/acceptance/bashunit_gha_annotations_test.sh @@ -7,7 +7,9 @@ # Clearing _BASHUNIT_GHA_ANNOTATIONS_CLAIMED below is how a nested run says # "pretend I am the top-level one": this suite is itself a bashunit run and has # already claimed the job log for its process tree, which is the very pollution -# the marker exists to prevent. +# the marker exists to prevent. A run that claims top-level also claims +# $GITHUB_STEP_SUMMARY, so it is pinned empty alongside or the fixtures' +# summaries would land on the real job page under CI. function set_up_before_script() { TEST_ENV_FILE="tests/acceptance/fixtures/.env.default" @@ -16,7 +18,8 @@ function set_up_before_script() { function test_annotations_reach_stdout_inside_github_actions() { local output - output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \ + output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_STEP_SUMMARY='' GITHUB_ACTIONS=true \ + ./bashunit --no-parallel --no-color \ --env "$TEST_ENV_FILE" "$FIXTURE")" || true assert_contains "::error file=$FIXTURE" "$output" @@ -25,7 +28,8 @@ function test_annotations_reach_stdout_inside_github_actions() { function test_the_annotation_carries_the_failing_line() { local output - output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \ + output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_STEP_SUMMARY='' GITHUB_ACTIONS=true \ + ./bashunit --no-parallel --no-color \ --env "$TEST_ENV_FILE" "$FIXTURE")" || true assert_matches "::error file=[^,]*,line=[0-9]+,title=" "$output" @@ -52,7 +56,8 @@ function test_a_nested_run_never_annotates_the_parents_log() { function test_never_suppresses_annotations_inside_github_actions() { local output - output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \ + output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_STEP_SUMMARY='' GITHUB_ACTIONS=true \ + ./bashunit --no-parallel --no-color \ --env "$TEST_ENV_FILE" --gha-annotations never "$FIXTURE")" || true assert_not_contains "::error" "$output" @@ -68,7 +73,8 @@ function test_always_emits_annotations_outside_github_actions() { function test_a_multi_line_message_stays_one_annotation() { local output - output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \ + output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_STEP_SUMMARY='' GITHUB_ACTIONS=true \ + ./bashunit --no-parallel --no-color \ --env "$TEST_ENV_FILE" "$FIXTURE")" || true # One failing test, so one ::error line, with the newlines percent-encoded. @@ -81,7 +87,8 @@ function test_log_gha_still_writes_the_file_without_duplicating_stdout() { log_file="$(bashunit::temp_file)" local output - output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \ + output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_STEP_SUMMARY='' GITHUB_ACTIONS=true \ + ./bashunit --no-parallel --no-color \ --env "$TEST_ENV_FILE" --log-gha "$log_file" "$FIXTURE")" || true assert_contains "::error" "$(cat "$log_file")" @@ -90,7 +97,8 @@ function test_log_gha_still_writes_the_file_without_duplicating_stdout() { function test_annotations_survive_parallel_aggregation() { local output - output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_ACTIONS=true ./bashunit --parallel --no-color \ + output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_STEP_SUMMARY='' GITHUB_ACTIONS=true \ + ./bashunit --parallel --no-color \ --env "$TEST_ENV_FILE" "$FIXTURE")" || true assert_contains "::error file=$FIXTURE" "$output" diff --git a/tests/acceptance/bashunit_report_md_test.sh b/tests/acceptance/bashunit_report_md_test.sh new file mode 100644 index 00000000..4d540d96 --- /dev/null +++ b/tests/acceptance/bashunit_report_md_test.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash + +# Clearing _BASHUNIT_GHA_ANNOTATIONS_CLAIMED below is how a nested run says +# "pretend I am the top-level one": this suite is itself a bashunit run and has +# already claimed the job-owned outputs for its process tree, which is the very +# pollution the marker exists to prevent. + +function set_up_before_script() { + TEST_ENV_FILE="tests/acceptance/fixtures/.env.default" + FIXTURE="tests/acceptance/fixtures/test_bashunit_report_md.sh" +} + +function test_report_md_writes_verdict_counts_and_fenced_failure() { + local report + report="$(bashunit::temp_file)" + ./bashunit --no-parallel --env "$TEST_ENV_FILE" --report-md "$report" "$FIXTURE" >/dev/null 2>&1 || true + + local content + content="$(cat "$report")" + assert_contains "1 failed, 1 passed in" "$content" + assert_contains "| Passed | Failed | Skipped | Incomplete | Risky | Snapshot | Flaky |" "$content" + assert_contains "| 1 | 1 | 0 | 0 | 0 | 0 | 0 |" "$content" + assert_contains '```' "$content" + assert_contains "expected value" "$content" +} + +function test_report_md_escapes_markdown_specials_in_test_names() { + local report + report="$(bashunit::temp_file)" + ./bashunit --no-parallel --env "$TEST_ENV_FILE" --report-md "$report" "$FIXTURE" >/dev/null 2>&1 || true + + assert_contains 'md \|name\* \_with\_ \`specials\`' "$(cat "$report")" +} + +function test_report_md_strips_ansi_from_failure_messages() { + local report + report="$(bashunit::temp_file)" + ./bashunit --no-parallel --env "$TEST_ENV_FILE" --report-md "$report" "$FIXTURE" >/dev/null 2>&1 || true + + assert_not_contains "$(printf '\033')" "$(cat "$report")" +} + +function test_report_md_is_populated_under_parallel() { + local report + report="$(bashunit::temp_file)" + ./bashunit --parallel --env "$TEST_ENV_FILE" --report-md "$report" "$FIXTURE" >/dev/null 2>&1 || true + + local content + content="$(cat "$report")" + assert_contains "1 failed, 1 passed in" "$content" + assert_contains "expected value" "$content" +} + +function test_github_step_summary_is_appended_not_truncated() { + local summary + summary="$(bashunit::temp_file)" + echo "earlier step content" >"$summary" + + _BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_STEP_SUMMARY="$summary" \ + ./bashunit --no-parallel --env "$TEST_ENV_FILE" "$FIXTURE" >/dev/null 2>&1 || true + + local content + content="$(cat "$summary")" + assert_contains "earlier step content" "$content" + assert_contains "1 failed, 1 passed in" "$content" +} + +function test_explicit_report_md_wins_over_github_step_summary() { + local report summary + report="$(bashunit::temp_file)" + summary="$(bashunit::temp_file)" + echo "earlier step content" >"$summary" + + _BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_STEP_SUMMARY="$summary" \ + ./bashunit --no-parallel --env "$TEST_ENV_FILE" --report-md "$report" "$FIXTURE" >/dev/null 2>&1 || true + + assert_contains "1 failed, 1 passed in" "$(cat "$report")" + assert_same "earlier step content" "$(cat "$summary")" +} + +# The claim marker is left alone here, so this is a genuinely nested run. Under +# CI it inherits GITHUB_STEP_SUMMARY and must stay quiet, or every nested run +# in a suite would append its fixtures' results to the job summary. +function test_a_nested_run_never_writes_the_step_summary() { + local summary + summary="$(bashunit::temp_file)" + echo "earlier step content" >"$summary" + + GITHUB_STEP_SUMMARY="$summary" \ + ./bashunit --no-parallel --env "$TEST_ENV_FILE" "$FIXTURE" >/dev/null 2>&1 || true + + assert_same "earlier step content" "$(cat "$summary")" +} + +function test_report_md_is_not_written_without_the_flag() { + local report + report="$(bashunit::temp_file)" + rm -f "$report" + ./bashunit --no-parallel --env "$TEST_ENV_FILE" "$FIXTURE" >/dev/null 2>&1 || true + + assert_file_not_exists "$report" +} + +function test_an_unwritable_report_md_path_fails_fast() { + local ec=0 + local output + output="$(./bashunit --no-parallel --env "$TEST_ENV_FILE" \ + --report-md /nonexistent-dir/report.md "$FIXTURE" 2>&1)" || ec=$? + + assert_general_error "" "" "$ec" + assert_contains "BASHUNIT_REPORT_MD" "$output" + assert_contains "cannot be written" "$output" +} + +function test_report_md_appears_in_the_help() { + assert_contains "--report-md" "$(./bashunit test --help)" +} diff --git a/tests/acceptance/fixtures/test_bashunit_report_md.sh b/tests/acceptance/fixtures/test_bashunit_report_md.sh new file mode 100644 index 00000000..3afde6f3 --- /dev/null +++ b/tests/acceptance/fixtures/test_bashunit_report_md.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +function test_md_pass() { + assert_same "ok" "ok" +} + +# The rendered name carries every Markdown-hostile character the table and +# headings must survive; the message lands inside the fenced block. +function test_md_fail_with_specials() { + bashunit::set_test_title 'md |name* _with_ `specials`' + assert_same 'expected value' 'actual value' +} diff --git a/tests/unit/reports/report_md_test.sh b/tests/unit/reports/report_md_test.sh new file mode 100644 index 00000000..18ac662a --- /dev/null +++ b/tests/unit/reports/report_md_test.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2329,SC2034 # Mock functions are invoked indirectly + +function set_up() { + _BASHUNIT_REPORTS_TEST_FILES=() + _BASHUNIT_REPORTS_TEST_NAMES=() + _BASHUNIT_REPORTS_TEST_STATUSES=() + _BASHUNIT_REPORTS_TEST_DURATIONS=() + _BASHUNIT_REPORTS_TEST_ASSERTIONS=() + _BASHUNIT_REPORTS_TEST_FAILURES=() + _BASHUNIT_REPORTS_TEST_LINES=() + _BASHUNIT_REPORTS_TEST_RETRIES=() + + BASHUNIT_PROFILE=false + BASHUNIT_COVERAGE=false +} + +function test_md_escape_escapes_markdown_specials() { + assert_same '\|a\* \_b\_ \`c\`' "$(bashunit::reports::__md_escape '|a* _b_ `c`')" +} + +function test_md_escape_escapes_backslash_first() { + assert_same '\\\*' "$(bashunit::reports::__md_escape '\*')" +} + +function test_md_escape_strips_ansi() { + assert_same 'red' "$(bashunit::reports::__md_escape "$(printf '\033[31mred\033[0m')")" +} + +function test_reports_is_enabled_when_report_md_configured() { + local out + out="$(mktemp)" + export BASHUNIT_REPORT_MD="$out" + assert_successful_code "$(bashunit::reports::is_enabled && echo ok)" + unset BASHUNIT_REPORT_MD + rm -f "$out" +} + +function test_generate_report_md_writes_failed_verdict_and_counts() { + local out + out="$(mktemp)" + set_up_md_fixture + mock_md_state + + bashunit::reports::generate_report_md "$out" + + local content + content="$(cat "$out")" + assert_contains "❌ 1 failed, 1 passed in 1.234s" "$content" + assert_contains "| Passed | Failed | Skipped | Incomplete | Risky | Snapshot | Flaky |" "$content" + assert_contains "| 1 | 1 | 0 | 0 | 0 | 0 | 0 |" "$content" + rm -f "$out" +} + +function test_generate_report_md_passing_run_omits_failures_section() { + local out + out="$(mktemp)" + _BASHUNIT_REPORTS_TEST_FILES=("tests/math_test.sh") + _BASHUNIT_REPORTS_TEST_NAMES=("it adds") + _BASHUNIT_REPORTS_TEST_STATUSES=("passed") + _BASHUNIT_REPORTS_TEST_DURATIONS=("5") + _BASHUNIT_REPORTS_TEST_ASSERTIONS=("1") + _BASHUNIT_REPORTS_TEST_FAILURES=("") + _BASHUNIT_REPORTS_TEST_LINES=("10") + _BASHUNIT_REPORTS_TEST_RETRIES=("0") + mock_md_state_all_passing + + bashunit::reports::generate_report_md "$out" + + local content + content="$(cat "$out")" + assert_contains "✅ 1 passed in 1.234s" "$content" + assert_not_contains "Failures" "$content" + rm -f "$out" +} + +function test_generate_report_md_strips_ansi_and_fences_failure() { + local out + out="$(mktemp)" + set_up_md_fixture + mock_md_state + + bashunit::reports::generate_report_md "$out" + + local content + content="$(cat "$out")" + assert_contains '```' "$content" + assert_contains 'Expected "a" but got "b"' "$content" + assert_not_contains "$(printf '\033')" "$content" + rm -f "$out" +} + +function test_generate_report_md_escapes_test_name_in_failures() { + local out + out="$(mktemp)" + set_up_md_fixture + mock_md_state + + bashunit::reports::generate_report_md "$out" + + local content + content="$(cat "$out")" + assert_contains 'it \|divides\* \`fast\`' "$content" + assert_contains "tests/math_test.sh:20" "$content" + rm -f "$out" +} + +function test_append_github_step_summary_appends_not_truncates() { + local out + out="$(mktemp)" + echo "previous step content" >"$out" + set_up_md_fixture + mock_md_state + + bashunit::reports::append_github_step_summary "$out" + + local content + content="$(cat "$out")" + assert_contains "previous step content" "$content" + assert_contains "❌ 1 failed, 1 passed" "$content" + rm -f "$out" +} + +function test_slowest_tests_section_only_with_profile() { + local out + out="$(mktemp)" + set_up_md_fixture + mock_md_state + + bashunit::reports::generate_report_md "$out" + assert_not_contains "Slowest tests" "$(cat "$out")" + + BASHUNIT_PROFILE=true + bashunit::reports::generate_report_md "$out" + + local content + content="$(cat "$out")" + assert_contains "Slowest tests" "$content" + # Sorted by duration descending: the 5ms test outranks the 3ms one. + assert_matches "5 .*it adds" "$content" + rm -f "$out" +} + +function test_coverage_line_only_when_coverage_ran() { + local out + out="$(mktemp)" + set_up_md_fixture + mock_md_state + function bashunit::coverage::get_percentage() { echo "85"; } + + bashunit::reports::generate_report_md "$out" + assert_not_contains "Coverage" "$(cat "$out")" + + BASHUNIT_COVERAGE=true + bashunit::reports::generate_report_md "$out" + assert_contains "**Coverage:** 85%" "$(cat "$out")" + rm -f "$out" +} + +# One passed and one failed test; the failed one carries an ANSI-colored, +# multi-line message and a Markdown-hostile name. +function set_up_md_fixture() { + _BASHUNIT_REPORTS_TEST_FILES=("tests/math_test.sh" "tests/math_test.sh") + _BASHUNIT_REPORTS_TEST_NAMES=("it adds" 'it |divides* `fast`') + _BASHUNIT_REPORTS_TEST_STATUSES=("passed" "failed") + _BASHUNIT_REPORTS_TEST_DURATIONS=("5" "3") + _BASHUNIT_REPORTS_TEST_ASSERTIONS=("1" "1") + _BASHUNIT_REPORTS_TEST_FAILURES=("" "$(printf '\033[31mExpected "a" but got "b"\033[0m\nsecond line')") + _BASHUNIT_REPORTS_TEST_LINES=("10" "20") + _BASHUNIT_REPORTS_TEST_RETRIES=("0" "0") +} + +function mock_md_state() { + function bashunit::state::get_tests_passed() { echo "1"; } + function bashunit::state::get_tests_failed() { echo "1"; } + function bashunit::state::get_tests_skipped() { echo "0"; } + function bashunit::state::get_tests_incomplete() { echo "0"; } + function bashunit::state::get_tests_risky() { echo "0"; } + function bashunit::state::get_tests_snapshot() { echo "0"; } + function bashunit::state::get_tests_flaky() { echo "0"; } + function bashunit::clock::total_runtime_in_milliseconds() { echo "1234"; } +} + +function mock_md_state_all_passing() { + mock_md_state + function bashunit::state::get_tests_passed() { echo "1"; } + function bashunit::state::get_tests_failed() { echo "0"; } +} diff --git a/tests/unit/reports/reports_test.sh b/tests/unit/reports/reports_test.sh index 1f10d738..305b8fd8 100644 --- a/tests/unit/reports/reports_test.sh +++ b/tests/unit/reports/reports_test.sh @@ -25,10 +25,11 @@ function set_up() { unset BASHUNIT_REPORT_TAP # These tests ask whether a *file* report was configured. Stdout annotations - # are a second reason to collect rows, and they switch themselves on inside - # GitHub Actions, so pin them off to keep the answer independent of the - # ambient CI environment. + # and the step-summary append are a second reason to collect rows, and both + # switch themselves on inside GitHub Actions, so pin them off to keep the + # answer independent of the ambient CI environment. export BASHUNIT_GHA_ANNOTATIONS=never + unset GITHUB_STEP_SUMMARY # Create temp file for output tests _TEMP_OUTPUT_FILE=$(mktemp)