diff --git a/.env.example b/.env.example index 7be7d05f..211a077d 100644 --- a/.env.example +++ b/.env.example @@ -42,6 +42,7 @@ BASHUNIT_NO_DIFF= # Default: false (disable unified diff on mu BASHUNIT_PARALLEL_RUN= # Default: false BASHUNIT_STOP_ON_FAILURE= # Default: false (stop suite on first failure) BASHUNIT_RERUN_FAILED= # Default: false (replay only last run's failing tests) +BASHUNIT_ORDER_BY= # Default: defined (or defects, random) BASHUNIT_CHANGED= # Default: false (run only test files changed since a git ref) BASHUNIT_CHANGED_REF= # Default: empty (--changed ref: origin/HEAD, then HEAD) BASHUNIT_EXCLUDE_FILTER= # Default: empty (skip tests whose name matches) diff --git a/CHANGELOG.md b/CHANGELOG.md index 857af8d5..7a9514d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased ### Added +- `--order-by ` picks the execution order: `defined` (default), `defects` (last run's failures first, whole suite still runs) or `random`. `--random-order` and `--seed` keep working unchanged (#1011) - `--changed []` runs only the test files git reports as touched since `` (default `origin/HEAD`, then `HEAD`), covering committed, staged, unstaged and untracked changes. Deletions are dropped, a rename selects its new path, and a missing work tree or unresolvable ref fails the run instead of selecting nothing (#1010) - `--list` (alias `--dry-run`) prints the tests a run would execute, without running them; `--list-format json` emits file, function, name, line and tags. Honours every selection flag, including `--shard` and `--random-order --seed` ordering (#1007) - `--exclude-filter ` skips tests by name, the counterpart of `--exclude-tag`. Repeatable, OR'd, and wins over `--filter` (#1009) diff --git a/completions/_bashunit b/completions/_bashunit index cf792846..2a16d648 100644 --- a/completions/_bashunit +++ b/completions/_bashunit @@ -85,6 +85,7 @@ _bashunit() { '--test-timeout[Fail a test running longer than N seconds]:seconds:' \ '--retry[Rerun a failed test up to N extra times]:count:' \ '--random-order[Randomize test execution order]' \ + '--order-by[Execution order]:mode:(defined defects random)' \ '--seed[Seed for random order]:seed:' \ '--shard[Run shard i of n]:shard:' \ '--rerun-failed[Replay only the tests that failed on the last run]' \ diff --git a/completions/bashunit.bash b/completions/bashunit.bash index 0fc2e0fe..af4c3f11 100644 --- a/completions/bashunit.bash +++ b/completions/bashunit.bash @@ -19,7 +19,7 @@ _BASHUNIT_COMPLETIONS_TEST_OPTS="--assert --boot --changed --coverage --coverage --debug --detailed --dry-run --env --exclude-filter --exclude-tag --fail-on-risky --failures-only \ --filter --help --jobs --list --list-format --log-gha --log-junit --login --no-color \ --no-coverage-report --no-output --no-output-on-failure --no-parallel \ ---no-progress --no-snapshot-create --output --parallel --profile \ +--no-progress --no-snapshot-create --order-by --output --parallel --profile \ --random-order --report-html \ --report-json --report-junit --report-tap --rerun-failed --retry --run-all \ --seed --shard --show-incomplete --show-output --show-skipped --simple \ @@ -80,6 +80,10 @@ _bashunit_completions() { COMPREPLY=($(compgen -W "text json" -- "$cur")) return 0 ;; + --order-by) + COMPREPLY=($(compgen -W "defined defects random" -- "$cur")) + return 0 + ;; -f | --filter | --exclude-filter | --tag | --exclude-tag | --retry | --seed | --shard | --test-timeout) return 0 ;; diff --git a/docs/command-line.md b/docs/command-line.md index 6ed3cc75..b1a32e16 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -81,6 +81,7 @@ bashunit test tests/ --parallel --simple | `--test-timeout ` | Fail a test if it runs longer than N seconds | | `--retry ` | Re-run a failed test up to N extra times | | `--random-order` | Randomize test execution order | +| `--order-by ` | Execution order: `defined` (default), `defects` or `random` | | `--seed ` | Seed for `--random-order` (reproducible shuffle) | | `--shard /` | Run shard i of n (split suite across runners) | | `--rerun-failed` | Replay only the tests that failed on the last run | @@ -821,6 +822,46 @@ BASHUNIT_RERUN_FAILED=true bashunit test tests/ ``` ::: +### Order by + +> `bashunit test --order-by ` + +Choose the execution order. Three modes: + +| Mode | Order | +|------|-------| +| `defined` | Definition order. The default. | +| `defects` | Tests that failed on the last recorded run first, then everything else. | +| `random` | Shuffled, the same mode `--random-order` selects. | + +`defects` reads the same `.bashunit/last-failed` cache +[`--rerun-failed`](#rerun-failed) writes, but it **reorders instead of +narrowing**: the whole suite still runs. Paired with `--stop-on-failure`, that +turns the pre-push check from minutes into seconds, because the tests most +likely to fail run first. + +```bash +bashunit test tests/ --order-by defects --stop-on-failure +``` + +Notes: + +- With no cache file the order falls back to `defined`, silently. +- `--order-by random` and `--random-order` are the same mode, and `--seed` + applies to both. +- Combining it with `--rerun-failed` is allowed: `--rerun-failed` still narrows + the selection, `--order-by` only orders what survives. +- Under `--parallel` the recorded failures are dispatched first. + +::: code-group +```bash [Fail fast on known-bad tests] +bashunit test --order-by defects --stop-on-failure +``` +```bash [Env variable] +BASHUNIT_ORDER_BY=defects bashunit test tests/ +``` +::: + ### Changed > `bashunit test --changed []` diff --git a/src/config/env.sh b/src/config/env.sh index fbb7d529..ea7c4ffa 100644 --- a/src/config/env.sh +++ b/src/config/env.sh @@ -255,6 +255,9 @@ _BASHUNIT_DEFAULT_SHARD_INDEX="" _BASHUNIT_DEFAULT_SHARD_TOTAL="" # Replay only the tests recorded as failing by the previous run _BASHUNIT_DEFAULT_RERUN_FAILED="false" +# Execution order: defined (definition order), defects (last run's failures +# first) or random (equivalent to --random-order) +_BASHUNIT_DEFAULT_ORDER_BY="defined" # Run only the test files git reports as changed since a ref _BASHUNIT_DEFAULT_CHANGED="false" # The ref --changed diffs against (empty = origin/HEAD, then HEAD) @@ -305,6 +308,8 @@ _BASHUNIT_DEFAULT_SNAPSHOT_REPORT_UNUSED="false" # up unrelated environment values. : "${BASHUNIT_RANDOM_ORDER:=$_BASHUNIT_DEFAULT_RANDOM_ORDER}" : "${BASHUNIT_SEED:=$_BASHUNIT_DEFAULT_SEED}" +# No bare ORDER_BY alias, same reasoning as RETRY/SEED above. +: "${BASHUNIT_ORDER_BY:=$_BASHUNIT_DEFAULT_ORDER_BY}" : "${BASHUNIT_SHARD_INDEX:=$_BASHUNIT_DEFAULT_SHARD_INDEX}" : "${BASHUNIT_SHARD_TOTAL:=$_BASHUNIT_DEFAULT_SHARD_TOTAL}" # No bare RERUN_FAILED alias, same reasoning as RETRY/SEED above. The default @@ -377,8 +382,17 @@ function bashunit::env::retry_count() { printf '%s' "$_BASHUNIT_RETRY_VALIDATED" } +## +# --random-order and `--order-by random` are the same mode under two names, and +# either may arrive through the environment rather than the parser, so the +# predicate accepts both rather than one arm normalising into the other. +## function bashunit::env::is_random_order_enabled() { - [ "$BASHUNIT_RANDOM_ORDER" = "true" ] + [ "$BASHUNIT_RANDOM_ORDER" = "true" ] || [ "${BASHUNIT_ORDER_BY:-defined}" = "random" ] +} + +function bashunit::env::is_defects_order_enabled() { + [ "${BASHUNIT_ORDER_BY:-defined}" = "defects" ] } ## diff --git a/src/config/rerun.sh b/src/config/rerun.sh index 57a64f07..e0eaced5 100644 --- a/src/config/rerun.sh +++ b/src/config/rerun.sh @@ -106,6 +106,84 @@ $file:$fn return 1 } +## +# Echoes the given test files with the recorded ones first, in the order the +# cache holds them, then the rest in their given order. +# +# Reordering, not filtering: --order-by defects still runs the whole suite, it +# just puts the known-bad files where --stop-on-failure trips on them first. +# Arguments: $@ test file paths. +## +function bashunit::rerun::order_files() { + local recorded_files + recorded_files="$(bashunit::rerun::files)" + if [ -z "$recorded_files" ]; then + [ "$#" -gt 0 ] && printf '%s\n' "$@" + return 0 + fi + + local file recorded + # A recorded file the current selection does not contain is skipped, so a + # deleted or filtered-out file cannot resurrect itself through the cache. + while IFS= read -r recorded; do + [ -z "$recorded" ] && continue + for file in "$@"; do + if [ "$file" = "$recorded" ]; then + printf '%s\n' "$file" + break + fi + done + done < Fail a test if it runs longer than N seconds (0 = off) --retry Re-run a failed test up to N extra times (0 = off) --random-order Randomize test execution order + --order-by Execution order: defined (default), defects (last run's failures first) or random --seed Seed for --random-order (reproducible shuffle) --shard / Run shard i of n (split the suite across runners) --rerun-failed Replay only the tests that failed on the last run (.bashunit/last-failed) diff --git a/src/main/test.sh b/src/main/test.sh index 8c70cfce..3fd88ab3 100644 --- a/src/main/test.sh +++ b/src/main/test.sh @@ -140,6 +140,11 @@ function bashunit::main::cmd_test() { BASHUNIT_RANDOM_ORDER=true export -n BASHUNIT_RANDOM_ORDER ;; + --order-by) + BASHUNIT_ORDER_BY="$2" + export -n BASHUNIT_ORDER_BY + shift + ;; --seed) BASHUNIT_SEED="$2" export -n BASHUNIT_SEED diff --git a/src/main/validate.sh b/src/main/validate.sh index 2d1e9543..2b92e186 100644 --- a/src/main/validate.sh +++ b/src/main/validate.sh @@ -144,6 +144,17 @@ function bashunit::main::validate_config_or_exit() { ;; esac + # Same shape as --output above: an unrecognised mode would otherwise leave the + # suite in definition order and look like it was honoured. + case "${BASHUNIT_ORDER_BY:-defined}" in + defined | defects | random) ;; + *) + printf "%sError: unsupported order '%s' for --order-by. Supported: defined, defects, random.%s\n" \ + "${_BASHUNIT_COLOR_FAILED}" "${BASHUNIT_ORDER_BY}" "${_BASHUNIT_COLOR_DEFAULT}" >&2 + exit 1 + ;; + esac + # Same shape as --output above: an unrecognised name would otherwise fall # through to the default renderer and look like it worked. case "${BASHUNIT_LIST_FORMAT:-}" in diff --git a/src/runner/discovery.sh b/src/runner/discovery.sh index a47984a9..e649e254 100644 --- a/src/runner/discovery.sh +++ b/src/runner/discovery.sh @@ -14,6 +14,18 @@ function bashunit::runner::load_test_files() { local -a worker_stderr_owners=() local worker_stderr_count=0 + # --order-by defects: files holding last run's failures go first. The cache is + # only read here, never used to drop a file, so the full suite still runs. + if bashunit::env::is_defects_order_enabled; then + bashunit::rerun::load + local -a _defect_files=() + local _defect_file + while IFS= read -r _defect_file; do + [ -n "$_defect_file" ] && _defect_files[${#_defect_files[@]}]=$_defect_file + done < <(bashunit::rerun::order_files "${files[@]+"${files[@]}"}") + files=("${_defect_files[@]+"${_defect_files[@]}"}") + fi + # Randomize file execution order (deterministic for the resolved seed). if bashunit::env::is_random_order_enabled; then local -a _shuffled_files=() diff --git a/src/runner/exec.sh b/src/runner/exec.sh index af7f7a75..365965b7 100644 --- a/src/runner/exec.sh +++ b/src/runner/exec.sh @@ -25,6 +25,15 @@ function bashunit::runner::order_functions_for_script() { ordered[${#ordered[@]}]="$fn" done + if bashunit::env::is_defects_order_enabled && [ "${#ordered[@]}" -gt 1 ]; then + local -a _defect_fns=() + local _defect_fn + for _defect_fn in $(bashunit::rerun::order_functions "$script" "${ordered[*]+${ordered[*]}}"); do + _defect_fns[${#_defect_fns[@]}]=$_defect_fn + done + ordered=("${_defect_fns[@]+"${_defect_fns[@]}"}") + fi + if bashunit::env::is_random_order_enabled && [ "${#ordered[@]}" -gt 1 ]; then local _base _crc _fn_seed _base=$(bashunit::env::seed) diff --git a/tests/acceptance/bashunit_order_by_test.sh b/tests/acceptance/bashunit_order_by_test.sh new file mode 100644 index 00000000..2331dfa0 --- /dev/null +++ b/tests/acceptance/bashunit_order_by_test.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +set -euo pipefail + +# --order-by reorders the suite, it never narrows it, so every case asserts the +# full selection through --list: same pipeline as a run, exact comparison. + +function set_up_before_script() { + D="./tests/acceptance/fixtures/order_by" + FIXTURE_A="$D/alpha_fixture.sh" + FIXTURE_B="$D/beta_fixture.sh" +} + +# Runs the two fixtures once so beta's failing test lands in a private cache, +# and echoes the directory holding it. The run is expected to fail. +function _seeded_cache_dir() { + local dir + dir="$(bashunit::temp_dir order_by)" + BASHUNIT_RERUN_CACHE_DIR="$dir/.bashunit" \ + ./bashunit --skip-env-file --no-color --no-parallel "$FIXTURE_A" "$FIXTURE_B" >/dev/null 2>&1 || true + echo "$dir" +} + +function test_order_by_defects_runs_the_recorded_failure_first() { + local dir + dir="$(_seeded_cache_dir)" + + local output + output=$(BASHUNIT_RERUN_CACHE_DIR="$dir/.bashunit" \ + ./bashunit --skip-env-file --list --order-by defects "$FIXTURE_A" "$FIXTURE_B" 2>/dev/null) + + assert_same "$FIXTURE_B::test_beta_fails +$FIXTURE_B::test_beta_one +$FIXTURE_A::test_alpha_one +$FIXTURE_A::test_alpha_two" "$output" +} + +function test_order_by_defects_keeps_the_whole_suite() { + local dir + dir="$(_seeded_cache_dir)" + + local count + count=$(BASHUNIT_RERUN_CACHE_DIR="$dir/.bashunit" \ + ./bashunit --skip-env-file --list --order-by defects "$FIXTURE_A" "$FIXTURE_B" 2>/dev/null | wc -l) + + assert_same "4" "$(printf '%s' "$count" | tr -d ' ')" +} + +function test_rerun_failed_still_narrows_while_order_by_only_orders() { + local dir + dir="$(_seeded_cache_dir)" + + local output + output=$(BASHUNIT_RERUN_CACHE_DIR="$dir/.bashunit" \ + ./bashunit --skip-env-file --list --order-by defects --rerun-failed "$FIXTURE_A" "$FIXTURE_B" 2>/dev/null) + + assert_same "$FIXTURE_B::test_beta_fails" "$output" +} + +function test_order_by_defects_stops_on_the_known_bad_test_first() { + local dir + dir="$(_seeded_cache_dir)" + + local ec=0 + local output + output=$(BASHUNIT_RERUN_CACHE_DIR="$dir/.bashunit" \ + ./bashunit --skip-env-file --no-color --no-parallel --order-by defects --stop-on-failure \ + "$FIXTURE_A" "$FIXTURE_B" 2>&1) || ec=$? + + assert_general_error "" "" "$ec" + # Only the recorded failure ran: without the reordering it would be test 4 of 4. + assert_contains "Tests: 1 failed, 1 total" "$(printf '%s' "$output" | tr -s ' ')" +} + +function test_order_by_defects_without_a_cache_keeps_the_defined_order() { + local dir + dir="$(bashunit::temp_dir order_by_empty)" + + local output + output=$(BASHUNIT_RERUN_CACHE_DIR="$dir/.bashunit" \ + ./bashunit --skip-env-file --list --order-by defects "$FIXTURE_A" "$FIXTURE_B" 2>/dev/null) + + assert_same "$FIXTURE_A::test_alpha_one +$FIXTURE_A::test_alpha_two +$FIXTURE_B::test_beta_one +$FIXTURE_B::test_beta_fails" "$output" +} + +function test_order_by_defined_is_the_default_order() { + local plain ordered + plain=$(./bashunit --skip-env-file --list "$FIXTURE_A" "$FIXTURE_B" 2>/dev/null) + ordered=$(./bashunit --skip-env-file --list --order-by defined "$FIXTURE_A" "$FIXTURE_B" 2>/dev/null) + + assert_same "$plain" "$ordered" +} + +function test_order_by_random_matches_random_order_for_the_same_seed() { + local legacy modern + legacy=$(./bashunit --skip-env-file --list --random-order --seed 42 "$FIXTURE_A" "$FIXTURE_B" 2>/dev/null) + modern=$(./bashunit --skip-env-file --list --order-by random --seed 42 "$FIXTURE_A" "$FIXTURE_B" 2>/dev/null) + + assert_same "$legacy" "$modern" +} + +function test_order_by_rejects_an_unknown_mode() { + local ec=0 + local output + output=$(./bashunit --skip-env-file --list --order-by sideways "$FIXTURE_A" 2>&1) || ec=$? + + assert_general_error "" "" "$ec" + assert_contains "sideways" "$output" + assert_contains "defined, defects, random" "$output" +} diff --git a/tests/acceptance/fixtures/order_by/alpha_fixture.sh b/tests/acceptance/fixtures/order_by/alpha_fixture.sh new file mode 100644 index 00000000..1d007c51 --- /dev/null +++ b/tests/acceptance/fixtures/order_by/alpha_fixture.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +function test_alpha_one() { + assert_same 1 1 +} + +function test_alpha_two() { + assert_same 2 2 +} diff --git a/tests/acceptance/fixtures/order_by/beta_fixture.sh b/tests/acceptance/fixtures/order_by/beta_fixture.sh new file mode 100644 index 00000000..9fefc9ee --- /dev/null +++ b/tests/acceptance/fixtures/order_by/beta_fixture.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +function test_beta_one() { + assert_same 3 3 +} + +# Defined last on purpose: --order-by defects has to move it to the front on the +# next run, which is what the ordering assertions measure. +function test_beta_fails() { + assert_same "expected" "actual" +} diff --git a/tests/unit/config/rerun_test.sh b/tests/unit/config/rerun_test.sh index 3ce18c4f..04acdf37 100644 --- a/tests/unit/config/rerun_test.sh +++ b/tests/unit/config/rerun_test.sh @@ -125,3 +125,72 @@ function test_filter_functions_keeps_only_allowed() { assert_same "test_one test_three" \ "$(bashunit::rerun::filter_functions "tests/a_test.sh" "test_one test_two test_three")" } + +# Writes a cache and loads it, so the ordering tests read as data -> expectation. +function _rerun_cache_with() { + local cache + cache="$(bashunit::rerun::cache_file)" + mkdir -p "$(dirname "$cache")" + printf '%s' "$1" >"$cache" + bashunit::rerun::load +} + +function test_order_files_puts_recorded_files_first() { + _rerun_cache_with 'tests/c_test.sh:test_three +tests/a_test.sh:test_one +' + + assert_same "tests/c_test.sh +tests/a_test.sh +tests/b_test.sh" \ + "$(bashunit::rerun::order_files tests/a_test.sh tests/b_test.sh tests/c_test.sh)" +} + +function test_order_files_keeps_every_file_exactly_once() { + _rerun_cache_with 'tests/a_test.sh:test_one +tests/a_test.sh:test_two +' + + assert_same "tests/a_test.sh +tests/b_test.sh" "$(bashunit::rerun::order_files tests/a_test.sh tests/b_test.sh)" +} + +function test_order_files_ignores_recorded_files_this_run_did_not_discover() { + _rerun_cache_with 'tests/gone_test.sh:test_gone +tests/b_test.sh:test_two +' + + assert_same "tests/b_test.sh +tests/a_test.sh" "$(bashunit::rerun::order_files tests/a_test.sh tests/b_test.sh)" +} + +function test_order_files_is_a_no_op_without_a_cache() { + _rerun_cache_with '' + + assert_same "tests/a_test.sh +tests/b_test.sh" "$(bashunit::rerun::order_files tests/a_test.sh tests/b_test.sh)" +} + +function test_order_functions_puts_recorded_functions_first_in_recorded_order() { + _rerun_cache_with 'tests/a_test.sh:test_three +tests/a_test.sh:test_one +' + + assert_same "test_three test_one test_two" \ + "$(bashunit::rerun::order_functions "tests/a_test.sh" "test_one test_two test_three")" +} + +function test_order_functions_ignores_entries_recorded_for_another_file() { + _rerun_cache_with 'tests/b_test.sh:test_two +' + + assert_same "test_one test_two" \ + "$(bashunit::rerun::order_functions "tests/a_test.sh" "test_one test_two")" +} + +function test_order_functions_is_a_no_op_without_a_cache() { + _rerun_cache_with '' + + assert_same "test_one test_two" \ + "$(bashunit::rerun::order_functions "tests/a_test.sh" "test_one test_two")" +}