diff --git a/.claude/rules/architecture-map.md b/.claude/rules/architecture-map.md index e5517246..29bbd416 100644 --- a/.claude/rules/architecture-map.md +++ b/.claude/rules/architecture-map.md @@ -61,7 +61,8 @@ shell (or, in parallel, in per-test `.result` files aggregated at the end). | `runner/parallel.sh` | job-slot waiting (`wait -n` or poll), running-job count, spinner | | `runner/hooks.sh` | set_up/tear_down (test + script scope), hook failure records, mock clearing, EXIT cleanup | | `runner/provider.sh` | `@data_provider` argument parsing | -| `runner/exec.sh` | `run_test`, the capture-subshell body, retry, timeout watchdog, per-file dispatch | +| `runner/exec.sh` | `run_test`, the capture-subshell body, retry, timeout watchdog, per-file dispatch, `order_functions_for_script` (the run order a seed produces) | +| `runner/list.sh` | `--list`/`--dry-run`: emits selected tests instead of running them; hooks in after all filtering, before any hook | | `runner/discovery.sh` | `load_test_files` (the per-file loop), `functions_for_script` | | `runner/bench.sh` | benchmark file loop and bench function dispatch | | `helper/index.sh` | aggregator only — sources the `src/helper/` module below | diff --git a/.env.example b/.env.example index 8c51ab19..f030d4fb 100644 --- a/.env.example +++ b/.env.example @@ -42,6 +42,8 @@ BASHUNIT_NO_DIFF= # Default: false (disable unified diff on mu BASHUNIT_PARALLEL_RUN= # Default: false BASHUNIT_STOP_ON_FAILURE= # Default: false (stop suite on first failure) BASHUNIT_RERUN_FAILED= # Default: false (replay only last run's failing tests) +BASHUNIT_LIST_TESTS= # Default: false (print the tests that would run, run none) +BASHUNIT_LIST_FORMAT= # Default: text (--list rendering: text or json) BASHUNIT_STOP_ON_ASSERTION_FAILURE= # Default: true (stop test on first assertion fail) BASHUNIT_STRICT_MODE= # Default: false (enable set -euo pipefail) BASHUNIT_LOGIN_SHELL= # Default: false (source login shell profiles) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd2fece1..4a2211cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased ### Added +- `--list` (alias `--dry-run`) prints the tests a run would execute, without running them; `--list-format json` emits file, function, name, line and tags. Honours every selection flag, including `--shard` and `--random-order --seed` ordering (#1007) - The coverage engine in use is reported by `--verbose`, and an explicit `BASHUNIT_COVERAGE_ENGINE=xtrace` that the running Bash cannot honour now warns instead of being silently ignored (#1005) ### Changed diff --git a/adrs/adr-011-source-layout-and-build-pipeline.md b/adrs/adr-011-source-layout-and-build-pipeline.md index faf3f130..7cdfdad5 100644 --- a/adrs/adr-011-source-layout-and-build-pipeline.md +++ b/adrs/adr-011-source-layout-and-build-pipeline.md @@ -61,7 +61,7 @@ Seventeen, in load order. The order is the dependency layering: leaves first. | 11 | `assert/` | 11 | 2326 | every assertion | | 12 | `doubles/` | 4 | 505 | spies and mocks — **sourced by `assert/index.sh`, not the entrypoint** | | 13 | `reports/` | 7 | 465 | JUnit, TAP, JSON, GHA and HTML writers | -| 14 | `runner/` | 11 | 2190 | the file loop, per-test execution, retry, result parsing | +| 14 | `runner/` | 12 | 2434 | the file loop, per-test execution, retry, result parsing, `--list` | | 15 | `benchmark/` | 4 | 219 | the bench implementation (`runner/bench.sh` is its loop) | | 16 | `learn/` | 14 | 1296 | the interactive tutorial (9 of those files are `learn/lessons/`) | | 17 | `main/` | 8 | 1473 | flag parsing per subcommand and the run lifecycle | diff --git a/completions/_bashunit b/completions/_bashunit index a3cf0205..6b989877 100644 --- a/completions/_bashunit +++ b/completions/_bashunit @@ -87,6 +87,8 @@ _bashunit() { '--seed[Seed for random order]:seed:' \ '--shard[Run shard i of n]:shard:' \ '--rerun-failed[Replay only the tests that failed on the last run]' \ + '(--list --dry-run)'{--list,--dry-run}'[Print the tests that would run, then exit]' \ + '--list-format[Rendering for --list]:format:(text json)' \ '--snapshot-update[Rewrite existing snapshots from the actual value]' \ '--no-snapshot-create[Fail instead of recording a missing snapshot]' \ '--snapshot-report-unused[List snapshot files no test resolved]' \ diff --git a/completions/bashunit.bash b/completions/bashunit.bash index 4d5d4704..2836bb4f 100644 --- a/completions/bashunit.bash +++ b/completions/bashunit.bash @@ -16,8 +16,8 @@ _BASHUNIT_COMPLETIONS_DOC_OPTS="--custom -e --env --boot -h --help" _BASHUNIT_COMPLETIONS_TEST_OPTS="--assert --boot --coverage --coverage-exclude \ --coverage-min --coverage-paths --coverage-report --coverage-report-html \ ---debug --detailed --env --exclude-tag --fail-on-risky --failures-only \ ---filter --help --jobs --log-gha --log-junit --login --no-color \ +--debug --detailed --dry-run --env --exclude-tag --fail-on-risky --failures-only \ +--filter --help --jobs --list --list-format --log-gha --log-junit --login --no-color \ --no-coverage-report --no-output --no-output-on-failure --no-parallel \ --no-progress --no-snapshot-create --output --parallel --profile \ --random-order --report-html \ @@ -76,6 +76,10 @@ _bashunit_completions() { COMPREPLY=($(compgen -f -- "$cur")) return 0 ;; + --list-format) + COMPREPLY=($(compgen -W "text json" -- "$cur")) + return 0 + ;; -f | --filter | --tag | --exclude-tag | --retry | --seed | --shard | --test-timeout) return 0 ;; diff --git a/docs/command-line.md b/docs/command-line.md index 4756d422..d6ab0f1e 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -83,6 +83,8 @@ bashunit test tests/ --parallel --simple | `--seed ` | Seed for `--random-order` (reproducible shuffle) | | `--shard /` | Run shard i of n (split suite across runners) | | `--rerun-failed` | Replay only the tests that failed on the last run | +| `--list`, `--dry-run` | Print the tests that would run, then exit | +| `--list-format ` | Rendering for `--list`: `text` (default) or `json` | | `--snapshot-update` | Rewrite existing snapshots from the actual value | | `--no-snapshot-create` | Fail on a missing snapshot instead of recording it | | `--snapshot-report-unused` | List snapshot files no test resolved (deletes nothing) | @@ -647,6 +649,73 @@ locally, commit the file, and let CI run with this flag. The two snapshot flags are opposites and pair up: `--snapshot-update` records deliberately, `--no-snapshot-create` forbids recording by accident. +### List + +> `bashunit test --list` · `bashunit test --dry-run` + +Print the tests that *would* run, then exit without running any of them. +`--dry-run` is an alias, for anyone arriving from shellspec. + +```bash +./bashunit --list tests/ +# tests/unit/assert_test.sh::test_assert_equals +# tests/unit/assert_test.sh::test_assert_contains +# ... +# 412 tests +``` + +Test ids go to **stdout**, one `path::function` per line; the count goes to +**stderr**, so the list pipes cleanly into `grep`, `fzf` or a CI matrix. + +Every selection mechanism applies exactly as it would in a real run — +`--filter`, `--tag`, `--exclude-tag`, `--shard`, `--rerun-failed`, +`--random-order --seed`, and `file::fn` / `file:LINE`. That makes it the way to +answer questions that previously needed a full run per answer: + +```bash +# Are the shards balanced? +for i in 1 2 3 4; do + printf '%s: ' "$i"; ./bashunit --list --shard "$i/4" tests/ | wc -l +done + +# Which tests does this filter actually select? +./bashunit --list --filter "snapshot" tests/ + +# What order will seed 42 use? +./bashunit --list --random-order --seed 42 tests/ +``` + +An empty selection prints nothing and exits **0** — this is a query, not a run, +so a filter matching nothing is an empty answer rather than the "No tests found" +error a real run reports. + +Test files are still *sourced* (that is how their functions are discovered), but +no test body and no lifecycle hook runs, and no report file is written. + +A test using a `@data_provider` is listed **once**, by function: the id is the +thing you can pass back to `--filter`, while the number of executions it expands +to is a property of the run. + +#### JSON output + +> `bashunit test --list --list-format json` + +```bash +./bashunit --list --list-format json tests/ | jq '.tests[] | select(.tags[]? == "slow")' +``` + +```json +{ + "count": 2, + "tests": [ + { "file": "tests/unit/example_test.sh", "function": "test_slow_path", + "name": "Slow path", "line": 12, "tags": ["slow"] } + ] +} +``` + +An unsupported format is rejected rather than silently falling back to `text`. + ### Rerun failed > `bashunit test --rerun-failed` diff --git a/src/config/env.sh b/src/config/env.sh index 21bc4964..765362e0 100644 --- a/src/config/env.sh +++ b/src/config/env.sh @@ -255,6 +255,10 @@ _BASHUNIT_DEFAULT_SHARD_INDEX="" _BASHUNIT_DEFAULT_SHARD_TOTAL="" # Replay only the tests recorded as failing by the previous run _BASHUNIT_DEFAULT_RERUN_FAILED="false" +# Print the tests that would run and exit, without running any of them +_BASHUNIT_DEFAULT_LIST_TESTS="false" +# Rendering for --list: text (one id per line) or json +_BASHUNIT_DEFAULT_LIST_FORMAT="text" # Rewrite existing snapshots from the actual value instead of comparing _BASHUNIT_DEFAULT_SNAPSHOT_UPDATE="false" # Record a snapshot the first time it is asserted (false = a missing one fails) @@ -301,6 +305,10 @@ _BASHUNIT_DEFAULT_SNAPSHOT_REPORT_UNUSED="false" # lives here rather than inline in rerun.sh so every BASHUNIT_* default has one # home; bashunit::rerun::is_enabled keeps its :- guard for callers that unset it. : "${BASHUNIT_RERUN_FAILED:=$_BASHUNIT_DEFAULT_RERUN_FAILED}" +# No bare LIST/LIST_FORMAT aliases: `LIST` is far too generic a name to let the +# environment turn a real run into a no-op query. +: "${BASHUNIT_LIST_TESTS:=$_BASHUNIT_DEFAULT_LIST_TESTS}" +: "${BASHUNIT_LIST_FORMAT:=$_BASHUNIT_DEFAULT_LIST_FORMAT}" # No bare SNAPSHOT_UPDATE alias either: rewriting files on disk is the last # setting that should be reachable by a generic name from the environment. : "${BASHUNIT_SNAPSHOT_UPDATE:=$_BASHUNIT_DEFAULT_SNAPSHOT_UPDATE}" @@ -565,6 +573,10 @@ function bashunit::env::is_snapshot_update_enabled() { [ "$BASHUNIT_SNAPSHOT_UPDATE" = "true" ] } +function bashunit::env::is_list_enabled() { + [ "$BASHUNIT_LIST_TESTS" = "true" ] +} + function bashunit::env::is_fail_on_risky_enabled() { [ "$BASHUNIT_FAIL_ON_RISKY" = "true" ] } diff --git a/src/console/header.sh b/src/console/header.sh index 6050397e..ac28bf2a 100644 --- a/src/console/header.sh +++ b/src/console/header.sh @@ -140,6 +140,8 @@ Options: --seed Seed for --random-order (reproducible shuffle) --shard / Run shard i of n (split the suite across runners) --rerun-failed Replay only the tests that failed on the last run (.bashunit/last-failed) + --list, --dry-run Print the tests that would run, then exit without running them + --list-format Rendering for --list: text (default) or json --snapshot-update Rewrite existing snapshots from the actual value (combine with --filter) --no-snapshot-create Fail on a missing snapshot instead of recording it (for CI) --snapshot-report-unused List snapshot files no test resolved (full runs only, deletes nothing) diff --git a/src/main/run.sh b/src/main/run.sh index 153c97ab..a7469164 100644 --- a/src/main/run.sh +++ b/src/main/run.sh @@ -66,7 +66,11 @@ function bashunit::main::exec_tests() { bashunit::parallel::init fi - if bashunit::env::is_tap_output_enabled; then + # --list is a query: stdout must be nothing but test ids, so the banner and + # the seed line are suppressed and the run header never prints (#1007). + if bashunit::env::is_list_enabled; then + : + elif bashunit::env::is_tap_output_enabled; then printf "TAP version 13\n" else bashunit::console_header::print_version_with_env "$filter" "${test_files[@]}" @@ -79,7 +83,7 @@ function bashunit::main::exec_tests() { BASHUNIT_SEED=$RANDOM export -n BASHUNIT_SEED fi - if ! bashunit::env::is_tap_output_enabled; then + if ! bashunit::env::is_tap_output_enabled && ! bashunit::env::is_list_enabled; then bashunit::console_header::print_random_order_seed "$BASHUNIT_SEED" fi fi @@ -100,6 +104,15 @@ function bashunit::main::exec_tests() { bashunit::runner::load_test_files "$filter" "$tag_filter" "$exclude_tag_filter" "${test_files[@]}" + # Nothing ran, so there are no results to render, no reports to write and no + # rerun cache to update. An empty selection is a valid answer to a query, so + # this exits 0 where a real run would exit 1 with "No tests found". + if bashunit::env::is_list_enabled; then + bashunit::runner::list_render_summary + bashunit::env::cleanup_run_output_dir + exit 0 + fi + if bashunit::parallel::is_enabled; then wait fi diff --git a/src/main/test.sh b/src/main/test.sh index 876cdb7f..8621bead 100644 --- a/src/main/test.sh +++ b/src/main/test.sh @@ -140,6 +140,15 @@ function bashunit::main::cmd_test() { BASHUNIT_RERUN_FAILED=true export -n BASHUNIT_RERUN_FAILED ;; + --list | --dry-run) + BASHUNIT_LIST_TESTS=true + export -n BASHUNIT_LIST_TESTS + ;; + --list-format) + BASHUNIT_LIST_FORMAT="$2" + export -n BASHUNIT_LIST_FORMAT + shift + ;; --snapshot-update) BASHUNIT_SNAPSHOT_UPDATE=true export -n BASHUNIT_SNAPSHOT_UPDATE diff --git a/src/main/validate.sh b/src/main/validate.sh index 897c8bc9..df45f158 100644 --- a/src/main/validate.sh +++ b/src/main/validate.sh @@ -125,6 +125,17 @@ function bashunit::main::validate_config_or_exit() { exit 1 ;; esac + + # Same shape as --output above: an unrecognised name would otherwise fall + # through to the default renderer and look like it worked. + case "${BASHUNIT_LIST_FORMAT:-}" in + text | json) ;; + *) + printf "%sError: unsupported list format '%s' for --list-format. Supported: text, json.%s\n" \ + "${_BASHUNIT_COLOR_FAILED}" "${BASHUNIT_LIST_FORMAT}" "${_BASHUNIT_COLOR_DEFAULT}" >&2 + exit 1 + ;; + esac } ## diff --git a/src/runner/discovery.sh b/src/runner/discovery.sh index 15c76650..a47984a9 100644 --- a/src/runner/discovery.sh +++ b/src/runner/discovery.sh @@ -120,6 +120,15 @@ function bashunit::runner::load_test_files() { bashunit::runner::restore_workdir continue fi + # --list stops here: every selection step has been applied, and nothing + # below this point can run without producing side effects (#1007). + if bashunit::env::is_list_enabled; then + bashunit::runner::list_functions "$test_file" "$functions_for_script" + bashunit::runner::clean_script_test_functions "$_script_fns_to_clean" + bashunit::runner::clean_set_up_and_tear_down_after_script + bashunit::runner::restore_workdir + continue + fi # Render header BEFORE set_up_before_script so user sees activity immediately bashunit::runner::render_running_file_header "$test_file" # Call hook directly (not with `if !`) to preserve errexit behavior inside the hook diff --git a/src/runner/exec.sh b/src/runner/exec.sh index 36bc81ec..af7f7a75 100644 --- a/src/runner/exec.sh +++ b/src/runner/exec.sh @@ -1,30 +1,31 @@ #!/usr/bin/env bash +_BASHUNIT_RUNNER_ORDERED_FNS_OUT="" + ## -# Runs the given test functions of a script (sequentially, or one background -# worker per test under --parallel). -# Arguments: $1 script path, $2 space-separated test function names, already -# filter/tag/rerun-filtered by load_test_files (never empty: the caller skips -# the file when no function survives filtering). +# Puts a file's already-filtered test functions in the order they will run, +# into _BASHUNIT_RUNNER_ORDERED_FNS_OUT. +# +# Under --random-order the seed is mixed with a stable per-file value (cksum of +# the path) so different files get different orders while staying reproducible +# for the resolved seed. Extracted from call_test_functions so `--list` reports +# the order a run would actually use instead of re-deriving it (#1007). +# +# Arguments: $1 script path, $2 space-separated test function names ## -function bashunit::runner::call_test_functions() { +function bashunit::runner::order_functions_for_script() { local script="$1" - local cached_functions="${2:-}" + local fns="${2:-}" local IFS=$' \t\n' - local -a functions_to_run=() - local functions_to_run_count=0 - local _fn - for _fn in $cached_functions; do - [ -z "$_fn" ] && continue - functions_to_run[functions_to_run_count]="$_fn" - functions_to_run_count=$((functions_to_run_count + 1)) + local -a ordered=() + local fn + for fn in $fns; do + [ -z "$fn" ] && continue + ordered[${#ordered[@]}]="$fn" done - # Randomize function order within this file. The seed is mixed with a stable - # per-file value (cksum of the path) so different files get different orders - # while staying reproducible for the resolved seed. - if bashunit::env::is_random_order_enabled && [ "$functions_to_run_count" -gt 1 ]; then + if bashunit::env::is_random_order_enabled && [ "${#ordered[@]}" -gt 1 ]; then local _base _crc _fn_seed _base=$(bashunit::env::seed) _crc=$(printf '%s' "$script" | cksum | cut -d' ' -f1) @@ -33,11 +34,34 @@ function bashunit::runner::call_test_functions() { local _sfn while IFS= read -r _sfn; do [ -n "$_sfn" ] && _shuffled_fns[${#_shuffled_fns[@]}]=$_sfn - done < <(printf '%s\n' "${functions_to_run[@]+"${functions_to_run[@]}"}" | bashunit::math::shuffle "$_fn_seed") - functions_to_run=("${_shuffled_fns[@]+"${_shuffled_fns[@]}"}") - functions_to_run_count=${#functions_to_run[@]} + done < <(printf '%s\n' "${ordered[@]+"${ordered[@]}"}" | bashunit::math::shuffle "$_fn_seed") + ordered=("${_shuffled_fns[@]+"${_shuffled_fns[@]}"}") fi + _BASHUNIT_RUNNER_ORDERED_FNS_OUT="${ordered[*]+${ordered[*]}}" +} + +## +# Runs the given test functions of a script (sequentially, or one background +# worker per test under --parallel). +# Arguments: $1 script path, $2 space-separated test function names, already +# filter/tag/rerun-filtered by load_test_files (never empty: the caller skips +# the file when no function survives filtering). +## +function bashunit::runner::call_test_functions() { + local script="$1" + local cached_functions="${2:-}" + local IFS=$' \t\n' + local -a functions_to_run=() + local functions_to_run_count=0 + + bashunit::runner::order_functions_for_script "$script" "$cached_functions" + local _ofn + for _ofn in $_BASHUNIT_RUNNER_ORDERED_FNS_OUT; do + functions_to_run[functions_to_run_count]="$_ofn" + functions_to_run_count=$((functions_to_run_count + 1)) + done + if [ "$functions_to_run_count" -le 0 ]; then return fi diff --git a/src/runner/index.sh b/src/runner/index.sh index 598d4592..93445d20 100755 --- a/src/runner/index.sh +++ b/src/runner/index.sh @@ -4,7 +4,8 @@ # for the reason recorded in adrs/adr-011-source-layout-and-build-pipeline.md. # # Sourced in dependency layers, leaves first: -# context · payload · diagnostics → parallel · hooks · result → provider · exec → discovery · bench +# context · payload · diagnostics → parallel · hooks · result → provider · exec +# → list → discovery · bench source "$BASHUNIT_ROOT_DIR/src/runner/context.sh" source "$BASHUNIT_ROOT_DIR/src/runner/payload.sh" source "$BASHUNIT_ROOT_DIR/src/runner/diagnostics.sh" @@ -13,5 +14,6 @@ source "$BASHUNIT_ROOT_DIR/src/runner/hooks.sh" source "$BASHUNIT_ROOT_DIR/src/runner/result.sh" source "$BASHUNIT_ROOT_DIR/src/runner/provider.sh" source "$BASHUNIT_ROOT_DIR/src/runner/exec.sh" +source "$BASHUNIT_ROOT_DIR/src/runner/list.sh" source "$BASHUNIT_ROOT_DIR/src/runner/discovery.sh" source "$BASHUNIT_ROOT_DIR/src/runner/bench.sh" diff --git a/src/runner/list.sh b/src/runner/list.sh new file mode 100644 index 00000000..9d69ed8f --- /dev/null +++ b/src/runner/list.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +# `--list` / `--dry-run`: report which tests a run would execute, without +# executing any of them (#1007). +# +# The listing hooks into load_test_files *after* every selection step (file +# shuffle, --filter, definition-line ordering, --tag/--exclude-tag, +# --rerun-failed, --shard) and *before* render_running_file_header and +# set_up_before_script, so a listed run has no side effects beyond sourcing the +# test files. Ordering comes from runner::order_functions_for_script, the same +# helper the runner uses, so the two cannot disagree about what a seed means. + +# JSON items are buffered because the document wraps them; text is streamed. +_BASHUNIT_LIST_JSON_ITEMS="" +_BASHUNIT_LIST_COUNT=0 + +function bashunit::runner::list_reset() { + _BASHUNIT_LIST_JSON_ITEMS="" + _BASHUNIT_LIST_COUNT=0 +} + +## +# Emits one file's selected test functions in run order. +# Arguments: $1 script path, $2 space-separated test function names +## +function bashunit::runner::list_functions() { + local script="$1" + local fns="${2:-}" + local IFS=$' \t\n' + + bashunit::runner::order_functions_for_script "$script" "$fns" + + local wants_json=false + if [ "$BASHUNIT_LIST_FORMAT" = "json" ]; then + wants_json=true + # Tags are only needed for the JSON columns; building the map per file is + # one awk scan, so skip it entirely for text output. + bashunit::helper::build_tags_map "$script" + fi + + local fn + for fn in $_BASHUNIT_RUNNER_ORDERED_FNS_OUT; do + [ -z "$fn" ] && continue + _BASHUNIT_LIST_COUNT=$((_BASHUNIT_LIST_COUNT + 1)) + + if [ "$wants_json" = false ]; then + printf '%s::%s\n' "$script" "$fn" + continue + fi + + local name line tags tags_json tag + name=$(bashunit::helper::normalize_test_function_name "$fn") + line=$(bashunit::helper::get_function_line_number "$fn") + bashunit::helper::tags_for_function "$fn" + tags="$_BASHUNIT_TAGS_OUT" + + tags_json="" + for tag in $tags; do + [ -z "$tag" ] && continue + [ -n "$tags_json" ] && tags_json="$tags_json," + tags_json="$tags_json\"$(bashunit::reports::__json_escape "$tag")\"" + done + + local item + item="{\"file\":\"$(bashunit::reports::__json_escape "$script")\"" + item="$item,\"function\":\"$(bashunit::reports::__json_escape "$fn")\"" + item="$item,\"name\":\"$(bashunit::reports::__json_escape "$name")\"" + item="$item,\"line\":${line:-0}" + item="$item,\"tags\":[$tags_json]}" + + [ -n "$_BASHUNIT_LIST_JSON_ITEMS" ] && _BASHUNIT_LIST_JSON_ITEMS="$_BASHUNIT_LIST_JSON_ITEMS," + _BASHUNIT_LIST_JSON_ITEMS="$_BASHUNIT_LIST_JSON_ITEMS$item" + done +} + +## +# Closes the listing: the JSON document, or the count on stderr so that stdout +# stays a clean list of ids for piping. +## +function bashunit::runner::list_render_summary() { + if [ "$BASHUNIT_LIST_FORMAT" = "json" ]; then + printf '{"count":%s,"tests":[%s]}\n' \ + "$_BASHUNIT_LIST_COUNT" "$_BASHUNIT_LIST_JSON_ITEMS" + return 0 + fi + + local unit="tests" + [ "$_BASHUNIT_LIST_COUNT" -eq 1 ] && unit="test" + printf '%s %s\n' "$_BASHUNIT_LIST_COUNT" "$unit" >&2 +} diff --git a/tests/acceptance/bashunit_list_test.sh b/tests/acceptance/bashunit_list_test.sh new file mode 100644 index 00000000..f999d613 --- /dev/null +++ b/tests/acceptance/bashunit_list_test.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash +set -euo pipefail + +# `--list` / `--dry-run` answer "which tests would run" without running them. +# Every selection mechanism (--filter, --tag, --shard, --random-order, ...) is +# otherwise only observable by executing the suite (#1007). +# +# The fixtures deliberately do not end in *test.sh — anything that does under +# tests/ is picked up by the real suite — so every run below passes explicit +# file paths rather than the fixture directory. + +FIXTURES_PATH="./tests/acceptance/fixtures/list" +ALPHA="$FIXTURES_PATH/alpha.sh" +BETA="$FIXTURES_PATH/beta.sh" +TAGGED="$FIXTURES_PATH/tagged.sh" +ORDER="$FIXTURES_PATH/order.sh" + +function test_list_prints_every_test_as_file_and_function() { + local output + output="$(./bashunit --list "$ALPHA" 2>/dev/null)" + + assert_same "\ +$ALPHA::test_alpha_one +$ALPHA::test_alpha_two" "$output" +} + +function test_list_exits_zero() { + ./bashunit --list "$ALPHA" >/dev/null 2>&1 + + assert_successful_code $? +} + +function test_list_reports_the_count_on_stderr() { + local stderr + stderr="$(./bashunit --list "$ALPHA" 2>&1 >/dev/null)" + + assert_same "2 tests" "$stderr" +} + +# A query, not a run: an empty selection is not an error the way "No tests +# found" is for a real run. +function test_list_exits_zero_and_prints_nothing_for_an_empty_selection() { + local output + output="$(./bashunit --list --filter "no_such_test_anywhere" "$ALPHA" "$BETA" 2>/dev/null)" + + assert_empty "$output" +} + +function test_an_empty_selection_still_exits_zero() { + local exit_code=0 + ./bashunit --list --filter "no_such_test_anywhere" "$ALPHA" >/dev/null 2>&1 || exit_code=$? + + assert_equals 0 "$exit_code" +} + +function test_dry_run_is_an_alias_of_list() { + local listed dry + listed="$(./bashunit --list "$ALPHA" 2>/dev/null)" + dry="$(./bashunit --dry-run "$ALPHA" 2>/dev/null)" + + # Without this both sides are empty when the flag is unknown, and the + # comparison passes while proving nothing. + assert_not_empty "$dry" + assert_same "$listed" "$dry" +} + +function test_list_respects_the_filter() { + local output + output="$(./bashunit --list --filter "alpha_two" "$ALPHA" "$BETA" 2>/dev/null)" + + assert_same "$ALPHA::test_alpha_two" "$output" +} + +function test_list_respects_tag_filtering() { + local output + output="$(./bashunit --list --tag slow "$TAGGED" 2>/dev/null)" + + assert_same "$TAGGED::test_tagged_slow" "$output" +} + +function test_list_respects_exclude_tag_filtering() { + local output + output="$(./bashunit --list --exclude-tag slow "$TAGGED" 2>/dev/null)" + + assert_same "$TAGGED::test_tagged_fast" "$output" +} + +# Sharding is the case the issue called out: checking balance used to need one +# full run per shard. +function test_list_shards_cover_every_test_exactly_once() { + local all shard1 shard2 combined + all="$(./bashunit --list "$ALPHA" "$BETA" 2>/dev/null | sort)" + shard1="$(./bashunit --list --shard 1/2 "$ALPHA" "$BETA" 2>/dev/null)" + shard2="$(./bashunit --list --shard 2/2 "$ALPHA" "$BETA" 2>/dev/null)" + combined="$(printf '%s\n%s\n' "$shard1" "$shard2" | grep -v '^$' | sort)" + + # Two empty sides also compare equal; require the union to be the real suite. + assert_not_empty "$all" + assert_same "$all" "$combined" +} + +# The ordering a seed produces must be the ordering that seed actually runs, +# or --list cannot reproduce a random-order failure. +function test_list_matches_the_execution_order_for_a_seed() { + local dir log listed executed + dir="$(bashunit::temp_dir)" + log="$dir/order.log" + + # The fixture records its own execution order, so this compares orders rather + # than parsing rendered result lines — those carry ANSI codes and a multi-byte + # status glyph whose width depends on the locale. + listed="$(./bashunit --list --random-order --seed 42 "$ORDER" 2>/dev/null | sed 's|.*::||')" + LIST_ORDER_LOG="$log" ./bashunit --no-parallel --random-order --seed 42 "$ORDER" >/dev/null 2>&1 + executed="$(cat "$log")" + + assert_not_empty "$listed" + assert_same "$listed" "$executed" +} + +# Paired with the test above: if the seed did not actually reorder anything, +# comparing list order to run order would hold trivially. +function test_a_seed_actually_changes_the_listed_order() { + local natural seeded + natural="$(./bashunit --list "$ORDER" 2>/dev/null)" + seeded="$(./bashunit --list --random-order --seed 42 "$ORDER" 2>/dev/null)" + + assert_not_equals "$natural" "$seeded" +} + +function test_list_runs_no_test_body_and_no_script_hook() { + local marker="$FIXTURES_PATH/.marker" + rm -f "$marker" + + local output + output="$(./bashunit --list "$FIXTURES_PATH/side_effect.sh" 2>/dev/null)" + + # The listing must have actually happened, or "no marker" only means the + # command failed before reaching the fixture. + assert_same "$FIXTURES_PATH/side_effect.sh::test_writes_a_marker" "$output" + assert_file_not_exists "$marker" +} + +function test_list_writes_no_report_file() { + local dir report output + dir="$(bashunit::temp_dir)" + report="$dir/report.tap" + + output="$(./bashunit --list --report-tap "$report" "$ALPHA" 2>/dev/null)" + + assert_not_empty "$output" + assert_file_not_exists "$report" +} + +function test_list_format_json_emits_valid_json_with_the_documented_fields() { + if ! command -v jq >/dev/null 2>&1; then + bashunit::skip "jq is required to validate the JSON shape" && return + fi + + local output + output="$(./bashunit --list --list-format json "$ALPHA" 2>/dev/null)" + + assert_same "2" "$(printf '%s' "$output" | jq -r '.count')" + assert_same "$ALPHA" "$(printf '%s' "$output" | jq -r '.tests[0].file')" + assert_same "test_alpha_one" "$(printf '%s' "$output" | jq -r '.tests[0].function')" + assert_same "Alpha one" "$(printf '%s' "$output" | jq -r '.tests[0].name')" + assert_not_empty "$(printf '%s' "$output" | jq -r '.tests[0].line')" +} + +function test_list_format_json_reports_tags() { + if ! command -v jq >/dev/null 2>&1; then + bashunit::skip "jq is required to validate the JSON shape" && return + fi + + local output + output="$(./bashunit --list --list-format json --tag slow "$TAGGED" 2>/dev/null)" + + assert_same "slow" "$(printf '%s' "$output" | jq -r '.tests[0].tags[0]')" +} + +function test_list_format_json_is_valid_for_an_empty_selection() { + if ! command -v jq >/dev/null 2>&1; then + bashunit::skip "jq is required to validate the JSON shape" && return + fi + + local output + output="$(./bashunit --list --list-format json --filter "no_such_test" "$ALPHA" 2>/dev/null)" + + assert_same "0" "$(printf '%s' "$output" | jq -r '.count')" + assert_same "0" "$(printf '%s' "$output" | jq -r '.tests | length')" +} + +function test_an_unsupported_list_format_is_rejected() { + local output + output="$(./bashunit --list --list-format yaml "$ALPHA" 2>&1)" || true + + assert_contains "yaml" "$output" +} + +function test_an_unsupported_list_format_exits_non_zero() { + local exit_code=0 + ./bashunit --list --list-format yaml "$ALPHA" >/dev/null 2>&1 || exit_code=$? + + # Paired with the supported-value case: an unknown *flag* would also exit 1, + # so on its own this proves nothing. + local ok_code=0 + ./bashunit --list --list-format text "$ALPHA" >/dev/null 2>&1 || ok_code=$? + + assert_equals 1 "$exit_code" + assert_equals 0 "$ok_code" +} + +# A data provider expands to N executions but is one test function; listing it +# once keeps the output a stable identifier list. +function test_a_data_provider_test_is_listed_once() { + local output + output="$(./bashunit --list "$FIXTURES_PATH/provider.sh" 2>/dev/null)" + + assert_same "$FIXTURES_PATH/provider.sh::test_provided" "$output" +} diff --git a/tests/acceptance/fixtures/list/alpha.sh b/tests/acceptance/fixtures/list/alpha.sh new file mode 100644 index 00000000..14b59824 --- /dev/null +++ b/tests/acceptance/fixtures/list/alpha.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +function test_alpha_one() { + assert_true true +} + +function test_alpha_two() { + assert_true true +} diff --git a/tests/acceptance/fixtures/list/beta.sh b/tests/acceptance/fixtures/list/beta.sh new file mode 100644 index 00000000..4527318a --- /dev/null +++ b/tests/acceptance/fixtures/list/beta.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +function test_beta_one() { + assert_true true +} diff --git a/tests/acceptance/fixtures/list/order.sh b/tests/acceptance/fixtures/list/order.sh new file mode 100644 index 00000000..fe573869 --- /dev/null +++ b/tests/acceptance/fixtures/list/order.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Each test appends its own name to $LIST_ORDER_LOG, so the execution order can +# be compared against --list without parsing rendered output (which varies with +# locale and sed dialect). + +function _record() { + printf '%s\n' "$1" >>"${LIST_ORDER_LOG:?}" +} + +function test_order_one() { + _record test_order_one + assert_true true +} + +function test_order_two() { + _record test_order_two + assert_true true +} + +function test_order_three() { + _record test_order_three + assert_true true +} + +function test_order_four() { + _record test_order_four + assert_true true +} diff --git a/tests/acceptance/fixtures/list/provider.sh b/tests/acceptance/fixtures/list/provider.sh new file mode 100644 index 00000000..3055faf7 --- /dev/null +++ b/tests/acceptance/fixtures/list/provider.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +function provider_values() { + echo "1" + echo "2" + echo "3" +} + +# @data_provider provider_values +function test_provided() { + assert_not_empty "$1" +} diff --git a/tests/acceptance/fixtures/list/side_effect.sh b/tests/acceptance/fixtures/list/side_effect.sh new file mode 100644 index 00000000..db83f9f7 --- /dev/null +++ b/tests/acceptance/fixtures/list/side_effect.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +# Both the script hook and the test body write the marker: --list must run +# neither, so its absence proves more than a test-body-only fixture would. +function set_up_before_script() { + touch "$(dirname "${BASH_SOURCE[0]}")/.marker" +} + +function test_writes_a_marker() { + touch "$(dirname "${BASH_SOURCE[0]}")/.marker" + assert_true true +} diff --git a/tests/acceptance/fixtures/list/tagged.sh b/tests/acceptance/fixtures/list/tagged.sh new file mode 100644 index 00000000..d636ef94 --- /dev/null +++ b/tests/acceptance/fixtures/list/tagged.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +# @tag slow +function test_tagged_slow() { + assert_true true +} + +# @tag fast +function test_tagged_fast() { + assert_true true +}