From 43aa460ccca9b208135f45c6014770b893d3f556 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Mon, 10 Aug 2026 15:15:03 +0200 Subject: [PATCH] feat(cli): --order-by to run the last run's failures first --rerun-failed replays the recorded failures and drops everything else, which is the wrong tool for CI and for the final check before pushing. There you want the whole suite, just with the known-bad tests first, so --stop-on-failure trips in seconds instead of minutes. --order-by defects reads the same .bashunit/last-failed cache but reorders instead of narrowing: recorded files lead the file list and recorded functions lead their file, both in the order the cache holds them, and everything else follows unchanged. With no cache the order falls back to defined. defined names the current behaviour so it can be set explicitly, and random is the existing --random-order under the new flag. Both spellings stay live because either can arrive through the environment rather than the parser, so is_random_order_enabled accepts each. Closes #1011 --- .env.example | 1 + CHANGELOG.md | 1 + completions/_bashunit | 1 + completions/bashunit.bash | 6 +- docs/command-line.md | 41 +++++++ src/config/env.sh | 16 ++- src/config/rerun.sh | 78 ++++++++++++ src/console/header.sh | 1 + src/main/test.sh | 5 + src/main/validate.sh | 11 ++ src/runner/discovery.sh | 12 ++ src/runner/exec.sh | 9 ++ tests/acceptance/bashunit_order_by_test.sh | 112 ++++++++++++++++++ .../fixtures/order_by/alpha_fixture.sh | 9 ++ .../fixtures/order_by/beta_fixture.sh | 11 ++ tests/unit/config/rerun_test.sh | 69 +++++++++++ 16 files changed, 381 insertions(+), 2 deletions(-) create mode 100644 tests/acceptance/bashunit_order_by_test.sh create mode 100644 tests/acceptance/fixtures/order_by/alpha_fixture.sh create mode 100644 tests/acceptance/fixtures/order_by/beta_fixture.sh diff --git a/.env.example b/.env.example index 7be7d05f..211a077d 100644 --- a/.env.example +++ b/.env.example @@ -42,6 +42,7 @@ BASHUNIT_NO_DIFF= # Default: false (disable unified diff on mu BASHUNIT_PARALLEL_RUN= # Default: false BASHUNIT_STOP_ON_FAILURE= # Default: false (stop suite on first failure) BASHUNIT_RERUN_FAILED= # Default: false (replay only last run's failing tests) +BASHUNIT_ORDER_BY= # Default: defined (or defects, random) BASHUNIT_CHANGED= # Default: false (run only test files changed since a git ref) BASHUNIT_CHANGED_REF= # Default: empty (--changed ref: origin/HEAD, then HEAD) BASHUNIT_EXCLUDE_FILTER= # Default: empty (skip tests whose name matches) diff --git a/CHANGELOG.md b/CHANGELOG.md index 857af8d5..7a9514d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased ### Added +- `--order-by ` picks the execution order: `defined` (default), `defects` (last run's failures first, whole suite still runs) or `random`. `--random-order` and `--seed` keep working unchanged (#1011) - `--changed []` runs only the test files git reports as touched since `` (default `origin/HEAD`, then `HEAD`), covering committed, staged, unstaged and untracked changes. Deletions are dropped, a rename selects its new path, and a missing work tree or unresolvable ref fails the run instead of selecting nothing (#1010) - `--list` (alias `--dry-run`) prints the tests a run would execute, without running them; `--list-format json` emits file, function, name, line and tags. Honours every selection flag, including `--shard` and `--random-order --seed` ordering (#1007) - `--exclude-filter ` skips tests by name, the counterpart of `--exclude-tag`. Repeatable, OR'd, and wins over `--filter` (#1009) diff --git a/completions/_bashunit b/completions/_bashunit index cf792846..2a16d648 100644 --- a/completions/_bashunit +++ b/completions/_bashunit @@ -85,6 +85,7 @@ _bashunit() { '--test-timeout[Fail a test running longer than N seconds]:seconds:' \ '--retry[Rerun a failed test up to N extra times]:count:' \ '--random-order[Randomize test execution order]' \ + '--order-by[Execution order]:mode:(defined defects random)' \ '--seed[Seed for random order]:seed:' \ '--shard[Run shard i of n]:shard:' \ '--rerun-failed[Replay only the tests that failed on the last run]' \ diff --git a/completions/bashunit.bash b/completions/bashunit.bash index 0fc2e0fe..af4c3f11 100644 --- a/completions/bashunit.bash +++ b/completions/bashunit.bash @@ -19,7 +19,7 @@ _BASHUNIT_COMPLETIONS_TEST_OPTS="--assert --boot --changed --coverage --coverage --debug --detailed --dry-run --env --exclude-filter --exclude-tag --fail-on-risky --failures-only \ --filter --help --jobs --list --list-format --log-gha --log-junit --login --no-color \ --no-coverage-report --no-output --no-output-on-failure --no-parallel \ ---no-progress --no-snapshot-create --output --parallel --profile \ +--no-progress --no-snapshot-create --order-by --output --parallel --profile \ --random-order --report-html \ --report-json --report-junit --report-tap --rerun-failed --retry --run-all \ --seed --shard --show-incomplete --show-output --show-skipped --simple \ @@ -80,6 +80,10 @@ _bashunit_completions() { COMPREPLY=($(compgen -W "text json" -- "$cur")) return 0 ;; + --order-by) + COMPREPLY=($(compgen -W "defined defects random" -- "$cur")) + return 0 + ;; -f | --filter | --exclude-filter | --tag | --exclude-tag | --retry | --seed | --shard | --test-timeout) return 0 ;; diff --git a/docs/command-line.md b/docs/command-line.md index 6ed3cc75..b1a32e16 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -81,6 +81,7 @@ bashunit test tests/ --parallel --simple | `--test-timeout ` | Fail a test if it runs longer than N seconds | | `--retry ` | Re-run a failed test up to N extra times | | `--random-order` | Randomize test execution order | +| `--order-by ` | Execution order: `defined` (default), `defects` or `random` | | `--seed ` | Seed for `--random-order` (reproducible shuffle) | | `--shard /` | Run shard i of n (split suite across runners) | | `--rerun-failed` | Replay only the tests that failed on the last run | @@ -821,6 +822,46 @@ BASHUNIT_RERUN_FAILED=true bashunit test tests/ ``` ::: +### Order by + +> `bashunit test --order-by ` + +Choose the execution order. Three modes: + +| Mode | Order | +|------|-------| +| `defined` | Definition order. The default. | +| `defects` | Tests that failed on the last recorded run first, then everything else. | +| `random` | Shuffled, the same mode `--random-order` selects. | + +`defects` reads the same `.bashunit/last-failed` cache +[`--rerun-failed`](#rerun-failed) writes, but it **reorders instead of +narrowing**: the whole suite still runs. Paired with `--stop-on-failure`, that +turns the pre-push check from minutes into seconds, because the tests most +likely to fail run first. + +```bash +bashunit test tests/ --order-by defects --stop-on-failure +``` + +Notes: + +- With no cache file the order falls back to `defined`, silently. +- `--order-by random` and `--random-order` are the same mode, and `--seed` + applies to both. +- Combining it with `--rerun-failed` is allowed: `--rerun-failed` still narrows + the selection, `--order-by` only orders what survives. +- Under `--parallel` the recorded failures are dispatched first. + +::: code-group +```bash [Fail fast on known-bad tests] +bashunit test --order-by defects --stop-on-failure +``` +```bash [Env variable] +BASHUNIT_ORDER_BY=defects bashunit test tests/ +``` +::: + ### Changed > `bashunit test --changed []` diff --git a/src/config/env.sh b/src/config/env.sh index fbb7d529..ea7c4ffa 100644 --- a/src/config/env.sh +++ b/src/config/env.sh @@ -255,6 +255,9 @@ _BASHUNIT_DEFAULT_SHARD_INDEX="" _BASHUNIT_DEFAULT_SHARD_TOTAL="" # Replay only the tests recorded as failing by the previous run _BASHUNIT_DEFAULT_RERUN_FAILED="false" +# Execution order: defined (definition order), defects (last run's failures +# first) or random (equivalent to --random-order) +_BASHUNIT_DEFAULT_ORDER_BY="defined" # Run only the test files git reports as changed since a ref _BASHUNIT_DEFAULT_CHANGED="false" # The ref --changed diffs against (empty = origin/HEAD, then HEAD) @@ -305,6 +308,8 @@ _BASHUNIT_DEFAULT_SNAPSHOT_REPORT_UNUSED="false" # up unrelated environment values. : "${BASHUNIT_RANDOM_ORDER:=$_BASHUNIT_DEFAULT_RANDOM_ORDER}" : "${BASHUNIT_SEED:=$_BASHUNIT_DEFAULT_SEED}" +# No bare ORDER_BY alias, same reasoning as RETRY/SEED above. +: "${BASHUNIT_ORDER_BY:=$_BASHUNIT_DEFAULT_ORDER_BY}" : "${BASHUNIT_SHARD_INDEX:=$_BASHUNIT_DEFAULT_SHARD_INDEX}" : "${BASHUNIT_SHARD_TOTAL:=$_BASHUNIT_DEFAULT_SHARD_TOTAL}" # No bare RERUN_FAILED alias, same reasoning as RETRY/SEED above. The default @@ -377,8 +382,17 @@ function bashunit::env::retry_count() { printf '%s' "$_BASHUNIT_RETRY_VALIDATED" } +## +# --random-order and `--order-by random` are the same mode under two names, and +# either may arrive through the environment rather than the parser, so the +# predicate accepts both rather than one arm normalising into the other. +## function bashunit::env::is_random_order_enabled() { - [ "$BASHUNIT_RANDOM_ORDER" = "true" ] + [ "$BASHUNIT_RANDOM_ORDER" = "true" ] || [ "${BASHUNIT_ORDER_BY:-defined}" = "random" ] +} + +function bashunit::env::is_defects_order_enabled() { + [ "${BASHUNIT_ORDER_BY:-defined}" = "defects" ] } ## diff --git a/src/config/rerun.sh b/src/config/rerun.sh index 57a64f07..e0eaced5 100644 --- a/src/config/rerun.sh +++ b/src/config/rerun.sh @@ -106,6 +106,84 @@ $file:$fn return 1 } +## +# Echoes the given test files with the recorded ones first, in the order the +# cache holds them, then the rest in their given order. +# +# Reordering, not filtering: --order-by defects still runs the whole suite, it +# just puts the known-bad files where --stop-on-failure trips on them first. +# Arguments: $@ test file paths. +## +function bashunit::rerun::order_files() { + local recorded_files + recorded_files="$(bashunit::rerun::files)" + if [ -z "$recorded_files" ]; then + [ "$#" -gt 0 ] && printf '%s\n' "$@" + return 0 + fi + + local file recorded + # A recorded file the current selection does not contain is skipped, so a + # deleted or filtered-out file cannot resurrect itself through the cache. + while IFS= read -r recorded; do + [ -z "$recorded" ] && continue + for file in "$@"; do + if [ "$file" = "$recorded" ]; then + printf '%s\n' "$file" + break + fi + done + done < Fail a test if it runs longer than N seconds (0 = off) --retry Re-run a failed test up to N extra times (0 = off) --random-order Randomize test execution order + --order-by Execution order: defined (default), defects (last run's failures first) or random --seed Seed for --random-order (reproducible shuffle) --shard / Run shard i of n (split the suite across runners) --rerun-failed Replay only the tests that failed on the last run (.bashunit/last-failed) diff --git a/src/main/test.sh b/src/main/test.sh index 8c70cfce..3fd88ab3 100644 --- a/src/main/test.sh +++ b/src/main/test.sh @@ -140,6 +140,11 @@ function bashunit::main::cmd_test() { BASHUNIT_RANDOM_ORDER=true export -n BASHUNIT_RANDOM_ORDER ;; + --order-by) + BASHUNIT_ORDER_BY="$2" + export -n BASHUNIT_ORDER_BY + shift + ;; --seed) BASHUNIT_SEED="$2" export -n BASHUNIT_SEED diff --git a/src/main/validate.sh b/src/main/validate.sh index 2d1e9543..2b92e186 100644 --- a/src/main/validate.sh +++ b/src/main/validate.sh @@ -144,6 +144,17 @@ function bashunit::main::validate_config_or_exit() { ;; esac + # Same shape as --output above: an unrecognised mode would otherwise leave the + # suite in definition order and look like it was honoured. + case "${BASHUNIT_ORDER_BY:-defined}" in + defined | defects | random) ;; + *) + printf "%sError: unsupported order '%s' for --order-by. Supported: defined, defects, random.%s\n" \ + "${_BASHUNIT_COLOR_FAILED}" "${BASHUNIT_ORDER_BY}" "${_BASHUNIT_COLOR_DEFAULT}" >&2 + exit 1 + ;; + esac + # Same shape as --output above: an unrecognised name would otherwise fall # through to the default renderer and look like it worked. case "${BASHUNIT_LIST_FORMAT:-}" in diff --git a/src/runner/discovery.sh b/src/runner/discovery.sh index a47984a9..e649e254 100644 --- a/src/runner/discovery.sh +++ b/src/runner/discovery.sh @@ -14,6 +14,18 @@ function bashunit::runner::load_test_files() { local -a worker_stderr_owners=() local worker_stderr_count=0 + # --order-by defects: files holding last run's failures go first. The cache is + # only read here, never used to drop a file, so the full suite still runs. + if bashunit::env::is_defects_order_enabled; then + bashunit::rerun::load + local -a _defect_files=() + local _defect_file + while IFS= read -r _defect_file; do + [ -n "$_defect_file" ] && _defect_files[${#_defect_files[@]}]=$_defect_file + done < <(bashunit::rerun::order_files "${files[@]+"${files[@]}"}") + files=("${_defect_files[@]+"${_defect_files[@]}"}") + fi + # Randomize file execution order (deterministic for the resolved seed). if bashunit::env::is_random_order_enabled; then local -a _shuffled_files=() diff --git a/src/runner/exec.sh b/src/runner/exec.sh index af7f7a75..365965b7 100644 --- a/src/runner/exec.sh +++ b/src/runner/exec.sh @@ -25,6 +25,15 @@ function bashunit::runner::order_functions_for_script() { ordered[${#ordered[@]}]="$fn" done + if bashunit::env::is_defects_order_enabled && [ "${#ordered[@]}" -gt 1 ]; then + local -a _defect_fns=() + local _defect_fn + for _defect_fn in $(bashunit::rerun::order_functions "$script" "${ordered[*]+${ordered[*]}}"); do + _defect_fns[${#_defect_fns[@]}]=$_defect_fn + done + ordered=("${_defect_fns[@]+"${_defect_fns[@]}"}") + fi + if bashunit::env::is_random_order_enabled && [ "${#ordered[@]}" -gt 1 ]; then local _base _crc _fn_seed _base=$(bashunit::env::seed) diff --git a/tests/acceptance/bashunit_order_by_test.sh b/tests/acceptance/bashunit_order_by_test.sh new file mode 100644 index 00000000..2331dfa0 --- /dev/null +++ b/tests/acceptance/bashunit_order_by_test.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +set -euo pipefail + +# --order-by reorders the suite, it never narrows it, so every case asserts the +# full selection through --list: same pipeline as a run, exact comparison. + +function set_up_before_script() { + D="./tests/acceptance/fixtures/order_by" + FIXTURE_A="$D/alpha_fixture.sh" + FIXTURE_B="$D/beta_fixture.sh" +} + +# Runs the two fixtures once so beta's failing test lands in a private cache, +# and echoes the directory holding it. The run is expected to fail. +function _seeded_cache_dir() { + local dir + dir="$(bashunit::temp_dir order_by)" + BASHUNIT_RERUN_CACHE_DIR="$dir/.bashunit" \ + ./bashunit --skip-env-file --no-color --no-parallel "$FIXTURE_A" "$FIXTURE_B" >/dev/null 2>&1 || true + echo "$dir" +} + +function test_order_by_defects_runs_the_recorded_failure_first() { + local dir + dir="$(_seeded_cache_dir)" + + local output + output=$(BASHUNIT_RERUN_CACHE_DIR="$dir/.bashunit" \ + ./bashunit --skip-env-file --list --order-by defects "$FIXTURE_A" "$FIXTURE_B" 2>/dev/null) + + assert_same "$FIXTURE_B::test_beta_fails +$FIXTURE_B::test_beta_one +$FIXTURE_A::test_alpha_one +$FIXTURE_A::test_alpha_two" "$output" +} + +function test_order_by_defects_keeps_the_whole_suite() { + local dir + dir="$(_seeded_cache_dir)" + + local count + count=$(BASHUNIT_RERUN_CACHE_DIR="$dir/.bashunit" \ + ./bashunit --skip-env-file --list --order-by defects "$FIXTURE_A" "$FIXTURE_B" 2>/dev/null | wc -l) + + assert_same "4" "$(printf '%s' "$count" | tr -d ' ')" +} + +function test_rerun_failed_still_narrows_while_order_by_only_orders() { + local dir + dir="$(_seeded_cache_dir)" + + local output + output=$(BASHUNIT_RERUN_CACHE_DIR="$dir/.bashunit" \ + ./bashunit --skip-env-file --list --order-by defects --rerun-failed "$FIXTURE_A" "$FIXTURE_B" 2>/dev/null) + + assert_same "$FIXTURE_B::test_beta_fails" "$output" +} + +function test_order_by_defects_stops_on_the_known_bad_test_first() { + local dir + dir="$(_seeded_cache_dir)" + + local ec=0 + local output + output=$(BASHUNIT_RERUN_CACHE_DIR="$dir/.bashunit" \ + ./bashunit --skip-env-file --no-color --no-parallel --order-by defects --stop-on-failure \ + "$FIXTURE_A" "$FIXTURE_B" 2>&1) || ec=$? + + assert_general_error "" "" "$ec" + # Only the recorded failure ran: without the reordering it would be test 4 of 4. + assert_contains "Tests: 1 failed, 1 total" "$(printf '%s' "$output" | tr -s ' ')" +} + +function test_order_by_defects_without_a_cache_keeps_the_defined_order() { + local dir + dir="$(bashunit::temp_dir order_by_empty)" + + local output + output=$(BASHUNIT_RERUN_CACHE_DIR="$dir/.bashunit" \ + ./bashunit --skip-env-file --list --order-by defects "$FIXTURE_A" "$FIXTURE_B" 2>/dev/null) + + assert_same "$FIXTURE_A::test_alpha_one +$FIXTURE_A::test_alpha_two +$FIXTURE_B::test_beta_one +$FIXTURE_B::test_beta_fails" "$output" +} + +function test_order_by_defined_is_the_default_order() { + local plain ordered + plain=$(./bashunit --skip-env-file --list "$FIXTURE_A" "$FIXTURE_B" 2>/dev/null) + ordered=$(./bashunit --skip-env-file --list --order-by defined "$FIXTURE_A" "$FIXTURE_B" 2>/dev/null) + + assert_same "$plain" "$ordered" +} + +function test_order_by_random_matches_random_order_for_the_same_seed() { + local legacy modern + legacy=$(./bashunit --skip-env-file --list --random-order --seed 42 "$FIXTURE_A" "$FIXTURE_B" 2>/dev/null) + modern=$(./bashunit --skip-env-file --list --order-by random --seed 42 "$FIXTURE_A" "$FIXTURE_B" 2>/dev/null) + + assert_same "$legacy" "$modern" +} + +function test_order_by_rejects_an_unknown_mode() { + local ec=0 + local output + output=$(./bashunit --skip-env-file --list --order-by sideways "$FIXTURE_A" 2>&1) || ec=$? + + assert_general_error "" "" "$ec" + assert_contains "sideways" "$output" + assert_contains "defined, defects, random" "$output" +} diff --git a/tests/acceptance/fixtures/order_by/alpha_fixture.sh b/tests/acceptance/fixtures/order_by/alpha_fixture.sh new file mode 100644 index 00000000..1d007c51 --- /dev/null +++ b/tests/acceptance/fixtures/order_by/alpha_fixture.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +function test_alpha_one() { + assert_same 1 1 +} + +function test_alpha_two() { + assert_same 2 2 +} diff --git a/tests/acceptance/fixtures/order_by/beta_fixture.sh b/tests/acceptance/fixtures/order_by/beta_fixture.sh new file mode 100644 index 00000000..9fefc9ee --- /dev/null +++ b/tests/acceptance/fixtures/order_by/beta_fixture.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +function test_beta_one() { + assert_same 3 3 +} + +# Defined last on purpose: --order-by defects has to move it to the front on the +# next run, which is what the ordering assertions measure. +function test_beta_fails() { + assert_same "expected" "actual" +} diff --git a/tests/unit/config/rerun_test.sh b/tests/unit/config/rerun_test.sh index 3ce18c4f..04acdf37 100644 --- a/tests/unit/config/rerun_test.sh +++ b/tests/unit/config/rerun_test.sh @@ -125,3 +125,72 @@ function test_filter_functions_keeps_only_allowed() { assert_same "test_one test_three" \ "$(bashunit::rerun::filter_functions "tests/a_test.sh" "test_one test_two test_three")" } + +# Writes a cache and loads it, so the ordering tests read as data -> expectation. +function _rerun_cache_with() { + local cache + cache="$(bashunit::rerun::cache_file)" + mkdir -p "$(dirname "$cache")" + printf '%s' "$1" >"$cache" + bashunit::rerun::load +} + +function test_order_files_puts_recorded_files_first() { + _rerun_cache_with 'tests/c_test.sh:test_three +tests/a_test.sh:test_one +' + + assert_same "tests/c_test.sh +tests/a_test.sh +tests/b_test.sh" \ + "$(bashunit::rerun::order_files tests/a_test.sh tests/b_test.sh tests/c_test.sh)" +} + +function test_order_files_keeps_every_file_exactly_once() { + _rerun_cache_with 'tests/a_test.sh:test_one +tests/a_test.sh:test_two +' + + assert_same "tests/a_test.sh +tests/b_test.sh" "$(bashunit::rerun::order_files tests/a_test.sh tests/b_test.sh)" +} + +function test_order_files_ignores_recorded_files_this_run_did_not_discover() { + _rerun_cache_with 'tests/gone_test.sh:test_gone +tests/b_test.sh:test_two +' + + assert_same "tests/b_test.sh +tests/a_test.sh" "$(bashunit::rerun::order_files tests/a_test.sh tests/b_test.sh)" +} + +function test_order_files_is_a_no_op_without_a_cache() { + _rerun_cache_with '' + + assert_same "tests/a_test.sh +tests/b_test.sh" "$(bashunit::rerun::order_files tests/a_test.sh tests/b_test.sh)" +} + +function test_order_functions_puts_recorded_functions_first_in_recorded_order() { + _rerun_cache_with 'tests/a_test.sh:test_three +tests/a_test.sh:test_one +' + + assert_same "test_three test_one test_two" \ + "$(bashunit::rerun::order_functions "tests/a_test.sh" "test_one test_two test_three")" +} + +function test_order_functions_ignores_entries_recorded_for_another_file() { + _rerun_cache_with 'tests/b_test.sh:test_two +' + + assert_same "test_one test_two" \ + "$(bashunit::rerun::order_functions "tests/a_test.sh" "test_one test_two")" +} + +function test_order_functions_is_a_no_op_without_a_cache() { + _rerun_cache_with '' + + assert_same "test_one test_two" \ + "$(bashunit::rerun::order_functions "tests/a_test.sh" "test_one test_two")" +}