feat(cli): --repeat to run each test n times

.env.example

@@ -44,6 +44,7 @@ BASHUNIT_STOP_ON_FAILURE=           # Default: false (stop suite on first failur
 BASHUNIT_RERUN_FAILED=              # Default: false (replay only last run's failing tests)
 BASHUNIT_ORDER_BY=                  # Default: defined (or defects, random)
 BASHUNIT_FAIL_ON_FLAKY=             # Default: false (treat retry-passed tests as failed)
+BASHUNIT_REPEAT=                    # Default: 1 (run each test N times)
 BASHUNIT_CHANGED=                   # Default: false (run only test files changed since a git ref)
 BASHUNIT_CHANGED_REF=               # Default: empty (--changed ref: origin/HEAD, then HEAD)
 BASHUNIT_EXCLUDE_FILTER=            # Default: empty (skip tests whose name matches)

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Unreleased
 
 ### Added
+- `--repeat <n>` runs each selected test n times so flakiness can be hunted before it reaches CI. The test is reported once with the aggregate outcome, a failure names the iteration it happened on, and repeat wraps `--retry` rather than the other way round (#1013)
 - Flaky is a first-class outcome: a test that only passed after a retry is counted separately, kept inside the pass total so the exit code is unchanged, and carried into JUnit (`<flakyFailure>`), TAP, JSON, HTML and GitHub Actions along with the first attempt's failure message. `--fail-on-flaky` turns such a run red (#1012)
 - `--order-by <mode>` picks the execution order: `defined` (default), `defects` (last run's failures first, whole suite still runs) or `random`. `--random-order` and `--seed` keep working unchanged (#1011)
 - `--changed [<ref>]` runs only the test files git reports as touched since `<ref>` (default `origin/HEAD`, then `HEAD`), covering committed, staged, unstaged and untracked changes. Deletions are dropped, a rename selects its new path, and a missing work tree or unresolvable ref fails the run instead of selecting nothing (#1010)

completions/_bashunit

@@ -84,6 +84,7 @@ _bashunit() {
     '(-S --stop-on-failure)'{-S,--stop-on-failure}'[Stop on first failure]' \
     '--test-timeout[Fail a test running longer than N seconds]:seconds:' \
     '--retry[Rerun a failed test up to N extra times]:count:' \
+    '--repeat[Run each selected test N times]:count:' \
     '--random-order[Randomize test execution order]' \
     '--fail-on-flaky[Treat tests that only passed after a retry as failures]' \
     '--order-by[Execution order]:mode:(defined defects random)' \

completions/bashunit.bash

@@ -20,7 +20,7 @@ _BASHUNIT_COMPLETIONS_TEST_OPTS="--assert --boot --changed --coverage --coverage
 --filter --help --jobs --list --list-format --log-gha --log-junit --login --no-color \
 --no-coverage-report --no-output --no-output-on-failure --no-parallel \
 --no-progress --no-snapshot-create --order-by --output --parallel --profile \
---random-order --report-html \
+--random-order --repeat --report-html \
 --report-json --report-junit --report-tap --rerun-failed --retry --run-all \
 --seed --shard --show-incomplete --show-output --show-skipped --simple \
 --skip-env-file --snapshot-report-unused --snapshot-update \
@@ -84,7 +84,7 @@ _bashunit_completions() {
     COMPREPLY=($(compgen -W "defined defects random" -- "$cur"))
     return 0
     ;;
-  -f | --filter | --exclude-filter | --tag | --exclude-tag | --retry | --seed | --shard | --test-timeout)
+  -f | --filter | --exclude-filter | --tag | --exclude-tag | --repeat | --retry | --seed | --shard | --test-timeout)
     return 0
     ;;
   esac

docs/command-line.md

@@ -80,6 +80,7 @@ bashunit test tests/ --parallel --simple
 | `-S, --stop-on-failure`        | Stop on first failure                            |
 | `--test-timeout <seconds>`     | Fail a test if it runs longer than N seconds     |
 | `--retry <n>`                  | Re-run a failed test up to N extra times         |
+| `--repeat <n>`                 | Run each selected test N times; it fails if any iteration fails |
 | `--random-order`               | Randomize test execution order                   |
 | `--order-by <mode>`            | Execution order: `defined` (default), `defects` or `random` |
 | `--seed <n>`                   | Seed for `--random-order` (reproducible shuffle) |
@@ -823,6 +824,52 @@ BASHUNIT_RERUN_FAILED=true bashunit test tests/
 ```
 :::
 
+### Repeat
+
+> `bashunit test --repeat <n>`
+
+Run each selected test n times. Where [`--retry`](#test-options) mitigates
+flakiness after it has already burned a CI run, `--repeat` goes looking for it:
+
+```bash
+bashunit test tests/ --repeat 50 --filter flaky_candidate
+```
+
+The test is reported **once**, with the aggregate outcome, and the assertion
+counts are those of the deciding iteration rather than the sum of all of them.
+A failure names the iteration it happened on:
+
+```
+✗ Failed: My test
+  (failed on iteration 7 of 50)
+```
+
+Iterating stops at the first failing iteration: the test is already going to be
+reported failed, and the remaining iterations cannot change that.
+
+**Interaction with `--retry`.** Repeat is the outer loop, retry the inner one.
+Each iteration gets its full retry budget before the next iteration starts, so
+`--repeat 2 --retry 1` runs the body at most four times, and an iteration that
+recovers on its retry lets the next iteration begin.
+
+Notes:
+
+- `--repeat 1` behaves exactly as if the flag were absent.
+- `--repeat 0`, a negative value and a non-numeric value are usage errors, not
+  silent no-ops.
+- Per-test `set_up` / `tear_down` run once per iteration; `set_up_before_script`
+  runs once for the file, as always.
+- Works under `--parallel`: each worker repeats its own test.
+
+::: code-group
+```bash [Hammer one suspect test]
+bashunit test --repeat 50 --filter flaky_candidate
+```
+```bash [Env variable]
+BASHUNIT_REPEAT=10 bashunit test tests/
+```
+:::
+
 ### Flaky tests
 
 > `bashunit test --retry 2 --fail-on-flaky`

src/config/env.sh

@@ -255,6 +255,8 @@ _BASHUNIT_DEFAULT_SHARD_INDEX=""
 _BASHUNIT_DEFAULT_SHARD_TOTAL=""
 # Replay only the tests recorded as failing by the previous run
 _BASHUNIT_DEFAULT_RERUN_FAILED="false"
+# Run each selected test n times; the test fails if any iteration fails
+_BASHUNIT_DEFAULT_REPEAT="1"
 # Treat a test that only passed after a retry as a failure for the exit code
 _BASHUNIT_DEFAULT_FAIL_ON_FLAKY="false"
 # Execution order: defined (definition order), defects (last run's failures
@@ -313,6 +315,7 @@ _BASHUNIT_DEFAULT_SNAPSHOT_REPORT_UNUSED="false"
 # No bare ORDER_BY/FAIL_ON_FLAKY aliases, same reasoning as RETRY/SEED above.
 : "${BASHUNIT_ORDER_BY:=$_BASHUNIT_DEFAULT_ORDER_BY}"
 : "${BASHUNIT_FAIL_ON_FLAKY:=$_BASHUNIT_DEFAULT_FAIL_ON_FLAKY}"
+: "${BASHUNIT_REPEAT:=$_BASHUNIT_DEFAULT_REPEAT}"
 : "${BASHUNIT_SHARD_INDEX:=$_BASHUNIT_DEFAULT_SHARD_INDEX}"
 : "${BASHUNIT_SHARD_TOTAL:=$_BASHUNIT_DEFAULT_SHARD_TOTAL}"
 # No bare RERUN_FAILED alias, same reasoning as RETRY/SEED above. The default
@@ -373,6 +376,19 @@ function bashunit::env::test_timeout_secs() {
 # In-shell (no fork) so the per-test hot path can read the global instead of
 # capturing retry_count in a $(...) subshell every test (#764).
 _BASHUNIT_RETRY_VALIDATED=0
+_BASHUNIT_REPEAT_VALIDATED=1
+##
+# Writes the validated repeat count into _BASHUNIT_REPEAT_VALIDATED. Mirrors
+# resolve_retry_count: fork-free, and a value the validator would have rejected
+# degrades to 1 rather than reaching the arithmetic in run_test.
+##
+function bashunit::env::resolve_repeat_count() {
+  case "${BASHUNIT_REPEAT:-1}" in
+  '' | *[!0-9]* | 0) _BASHUNIT_REPEAT_VALIDATED=1 ;;
+  *) _BASHUNIT_REPEAT_VALIDATED="${BASHUNIT_REPEAT:-1}" ;;
+  esac
+}
+
 function bashunit::env::resolve_retry_count() {
   case "${BASHUNIT_RETRY:-0}" in
   '' | *[!0-9]*) _BASHUNIT_RETRY_VALIDATED=0 ;;

src/console/header.sh

@@ -138,6 +138,7 @@ Options:
   -S, --stop-on-failure       Stop on first failure
   --test-timeout <seconds>    Fail a test if it runs longer than N seconds (0 = off)
   --retry <n>                 Re-run a failed test up to N extra times (0 = off)
+  --repeat <n>                Run each selected test N times; it fails if any iteration fails
   --random-order              Randomize test execution order
   --order-by <mode>           Execution order: defined (default), defects (last run's failures first) or random
   --seed <n>                  Seed for --random-order (reproducible shuffle)

src/main/test.sh

@@ -136,6 +136,11 @@ function bashunit::main::cmd_test() {
       export -n BASHUNIT_RETRY
       shift
       ;;
+    --repeat)
+      BASHUNIT_REPEAT="$2"
+      export -n BASHUNIT_REPEAT
+      shift
+      ;;
     --random-order)
       BASHUNIT_RANDOM_ORDER=true
       export -n BASHUNIT_RANDOM_ORDER

src/main/validate.sh

@@ -66,6 +66,15 @@ function bashunit::main::validate_config_or_exit() {
     "${BASHUNIT_RETRY:-0}" "BASHUNIT_RETRY (--retry)"
   bashunit::main::require_non_negative_int_or_exit \
     "${BASHUNIT_TEST_TIMEOUT:-0}" "BASHUNIT_TEST_TIMEOUT (--test-timeout)"
+  bashunit::main::require_non_negative_int_or_exit \
+    "${BASHUNIT_REPEAT:-1}" "BASHUNIT_REPEAT (--repeat)"
+  # 0 passes the non-negative check but means "run nothing", which would be a
+  # silent no-op rather than the usage error it is.
+  if [ "${BASHUNIT_REPEAT:-1}" -lt 1 ]; then
+    printf "%sError: BASHUNIT_REPEAT (--repeat) must be at least 1, got '%s'.%s\n" \
+      "${_BASHUNIT_COLOR_FAILED}" "${BASHUNIT_REPEAT}" "${_BASHUNIT_COLOR_DEFAULT}" >&2
+    exit 1
+  fi
   # Empty is the documented "no minimum" default, so only a set value is checked.
   if [ -n "${BASHUNIT_COVERAGE_MIN:-}" ]; then
     bashunit::main::require_non_negative_int_or_exit \

src/runner/exec.sh

@@ -341,47 +341,69 @@ function bashunit::runner::run_test() {
   # failure -- the only evidence of what the flakiness looks like -- is kept here
   # before it is lost.
   local first_attempt_result=""
+  bashunit::env::resolve_repeat_count
+  local repeat_max=$_BASHUNIT_REPEAT_VALIDATED
+  local iteration=0
+  local failed_iteration=0
   local measure_duration=false
   bashunit::runner::needs_test_duration && measure_duration=true
   # Retry wraps ONLY execution: a failed attempt is judged from its encoded
   # result without committing, so the parse/report/counter path below still runs
   # exactly once (on the final attempt) and nothing is double-counted. Each fork
   # in --parallel retries itself before writing its single .result file.
-  while :; do
-    if [ "$measure_duration" = true ]; then
-      bashunit::clock::now_to_slot
-      start_time=$_BASHUNIT_CLOCK_NOW_OUT
-    fi
-    if bashunit::env::is_test_timeout_enabled; then
-      bashunit::runner::run_with_timeout "$test_file" "$fn_name" "$@"
-      test_execution_result="$_BASHUNIT_RUNNER_EXEC_OUT"
-      timed_out="$_BASHUNIT_RUNNER_TIMED_OUT"
-    else
-      test_execution_result=$(bashunit::runner::execute_test_body "$test_file" "$fn_name" "$@")
-    fi
+  # --repeat is the OUTER loop and --retry the inner one: an iteration gets its
+  # full retry budget before the next iteration starts. Iterating stops at the
+  # first failure, since the test is already going to be reported failed and the
+  # remaining iterations cannot change that.
+  while [ "$iteration" -lt "$repeat_max" ]; do
+    iteration=$((iteration + 1))
+    retries_used=0
+    first_attempt_result=""
+    while :; do
+      if [ "$measure_duration" = true ]; then
+        bashunit::clock::now_to_slot
+        start_time=$_BASHUNIT_CLOCK_NOW_OUT
+      fi
+      if bashunit::env::is_test_timeout_enabled; then
+        bashunit::runner::run_with_timeout "$test_file" "$fn_name" "$@"
+        test_execution_result="$_BASHUNIT_RUNNER_EXEC_OUT"
+        timed_out="$_BASHUNIT_RUNNER_TIMED_OUT"
+      else
+        test_execution_result=$(bashunit::runner::execute_test_body "$test_file" "$fn_name" "$@")
+      fi
+
+      local attempt_runtime_output="${test_execution_result%%##ASSERTIONS_*}"
+      # Counts first: detect_runtime_error consults the exit code when the output
+      # text is translated and matches nothing. extract_result_counts is a pure
+      # read, so moving it ahead commits nothing.
+      bashunit::runner::extract_result_counts "$test_execution_result"
+      bashunit::runner::detect_runtime_error "$attempt_runtime_output" \
+        "$_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT"
+      local attempt_runtime_error=$_BASHUNIT_RUNNER_RUNTIME_ERROR_OUT
+      local attempt_display_output=$_BASHUNIT_RUNNER_RUNTIME_OUTPUT_OUT
+      # Mirror the commit-phase failure test exactly (runtime error, non-zero exit,
+      # or a failed assertion); snapshot/incomplete/skipped/risky are not failures.
+      if [ -z "$attempt_runtime_error" ] &&
+        [ "$_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT" -eq 0 ] &&
+        [ "$_BASHUNIT_RUNNER_COUNTS_FAILED_OUT" -eq 0 ]; then
+        break
+      fi
+      # Only reached when the attempt failed, so this is the first failure.
+      if [ -z "$first_attempt_result" ]; then
+        first_attempt_result="$test_execution_result"
+      fi
+      [ "$retries_used" -ge "$retry_max" ] && break
+      retries_used=$((retries_used + 1))
+    done
 
-    local attempt_runtime_output="${test_execution_result%%##ASSERTIONS_*}"
-    # Counts first: detect_runtime_error consults the exit code when the output
-    # text is translated and matches nothing. extract_result_counts is a pure
-    # read, so moving it ahead commits nothing.
-    bashunit::runner::extract_result_counts "$test_execution_result"
-    bashunit::runner::detect_runtime_error "$attempt_runtime_output" \
-      "$_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT"
-    local attempt_runtime_error=$_BASHUNIT_RUNNER_RUNTIME_ERROR_OUT
-    local attempt_display_output=$_BASHUNIT_RUNNER_RUNTIME_OUTPUT_OUT
-    # Mirror the commit-phase failure test exactly (runtime error, non-zero exit,
-    # or a failed assertion); snapshot/incomplete/skipped/risky are not failures.
-    if [ -z "$attempt_runtime_error" ] &&
-      [ "$_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT" -eq 0 ] &&
-      [ "$_BASHUNIT_RUNNER_COUNTS_FAILED_OUT" -eq 0 ]; then
+    # The inner loop only exits with a failure once the retries are exhausted,
+    # so reaching here with one means this iteration is the verdict.
+    if [ -n "$attempt_runtime_error" ] ||
+      [ "$_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT" -ne 0 ] ||
+      [ "$_BASHUNIT_RUNNER_COUNTS_FAILED_OUT" -ne 0 ]; then
+      failed_iteration=$iteration
       break
     fi
-    # Only reached when the attempt failed, so this is the first failure.
-    if [ -z "$first_attempt_result" ]; then
-      first_attempt_result="$test_execution_result"
-    fi
-    [ "$retries_used" -ge "$retry_max" ] && break
-    retries_used=$((retries_used + 1))
   done
 
   # The retry count lives in this shell, not in the test subshell that built the
@@ -457,6 +479,13 @@ function bashunit::runner::run_test() {
   bashunit::state::reset_test_title
   bashunit::state::reset_current_test_interpolated_function_name
 
+  # Under --repeat the test is reported once, so the message has to say which
+  # iteration produced the failure or the count is unactionable.
+  local repeat_note=""
+  if [ "$repeat_max" -gt 1 ] && [ "$failed_iteration" -gt 0 ]; then
+    repeat_note=" (failed on iteration $failed_iteration of $repeat_max)"
+  fi
+
   local failure_label="$label"
   local failure_function="$fn_name"
   if [ -n "$hook_failure" ]; then
@@ -492,6 +521,7 @@ function bashunit::runner::run_test() {
       error_message="Test timed out after $(bashunit::env::test_timeout_secs)s"
     fi
 
+    error_message="$error_message$repeat_note"
     bashunit::console_results::print_error_test "$failure_function" "$error_message" "$runtime_output"
     bashunit::reports::add_test_failed "$test_file" "$failure_label" "$duration" "$total_assertions" "$error_message"
     bashunit::runner::write_failure_result_output "$test_file" "$failure_function" "$error_message" "$runtime_output"
@@ -504,7 +534,11 @@ function bashunit::runner::run_test() {
   if [ "$current_assertions_failed" != "$_BASHUNIT_ASSERTIONS_FAILED" ]; then
     bashunit::state::add_tests_failed
     bashunit::rerun::record "$test_file" "$fn_name"
-    bashunit::reports::add_test_failed "$test_file" "$label" "$duration" "$total_assertions" "$subshell_output"
+    bashunit::reports::add_test_failed \
+      "$test_file" "$label" "$duration" "$total_assertions" "$subshell_output$repeat_note"
+    if [ -n "$repeat_note" ]; then
+      bashunit::console_results::print_line "failed" "${repeat_note# }"
+    fi
     local assertion_runtime_output
     assertion_runtime_output="$(
       bashunit::runner::extract_assertion_runtime_output "$runtime_output" "$subshell_output"