diff --git a/.env.example b/.env.example index 386d1e20..86f96141 100644 --- a/.env.example +++ b/.env.example @@ -44,6 +44,7 @@ BASHUNIT_STOP_ON_FAILURE= # Default: false (stop suite on first failur BASHUNIT_RERUN_FAILED= # Default: false (replay only last run's failing tests) BASHUNIT_ORDER_BY= # Default: defined (or defects, random) BASHUNIT_FAIL_ON_FLAKY= # Default: false (treat retry-passed tests as failed) +BASHUNIT_REPEAT= # Default: 1 (run each test N times) BASHUNIT_CHANGED= # Default: false (run only test files changed since a git ref) BASHUNIT_CHANGED_REF= # Default: empty (--changed ref: origin/HEAD, then HEAD) BASHUNIT_EXCLUDE_FILTER= # Default: empty (skip tests whose name matches) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3719366..f130f646 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased ### Added +- `--repeat ` runs each selected test n times so flakiness can be hunted before it reaches CI. The test is reported once with the aggregate outcome, a failure names the iteration it happened on, and repeat wraps `--retry` rather than the other way round (#1013) - Flaky is a first-class outcome: a test that only passed after a retry is counted separately, kept inside the pass total so the exit code is unchanged, and carried into JUnit (``), TAP, JSON, HTML and GitHub Actions along with the first attempt's failure message. `--fail-on-flaky` turns such a run red (#1012) - `--order-by ` picks the execution order: `defined` (default), `defects` (last run's failures first, whole suite still runs) or `random`. `--random-order` and `--seed` keep working unchanged (#1011) - `--changed []` runs only the test files git reports as touched since `` (default `origin/HEAD`, then `HEAD`), covering committed, staged, unstaged and untracked changes. Deletions are dropped, a rename selects its new path, and a missing work tree or unresolvable ref fails the run instead of selecting nothing (#1010) diff --git a/completions/_bashunit b/completions/_bashunit index f1cdbd4c..6e9c06fd 100644 --- a/completions/_bashunit +++ b/completions/_bashunit @@ -84,6 +84,7 @@ _bashunit() { '(-S --stop-on-failure)'{-S,--stop-on-failure}'[Stop on first failure]' \ '--test-timeout[Fail a test running longer than N seconds]:seconds:' \ '--retry[Rerun a failed test up to N extra times]:count:' \ + '--repeat[Run each selected test N times]:count:' \ '--random-order[Randomize test execution order]' \ '--fail-on-flaky[Treat tests that only passed after a retry as failures]' \ '--order-by[Execution order]:mode:(defined defects random)' \ diff --git a/completions/bashunit.bash b/completions/bashunit.bash index e6b3569f..4980b34a 100644 --- a/completions/bashunit.bash +++ b/completions/bashunit.bash @@ -20,7 +20,7 @@ _BASHUNIT_COMPLETIONS_TEST_OPTS="--assert --boot --changed --coverage --coverage --filter --help --jobs --list --list-format --log-gha --log-junit --login --no-color \ --no-coverage-report --no-output --no-output-on-failure --no-parallel \ --no-progress --no-snapshot-create --order-by --output --parallel --profile \ ---random-order --report-html \ +--random-order --repeat --report-html \ --report-json --report-junit --report-tap --rerun-failed --retry --run-all \ --seed --shard --show-incomplete --show-output --show-skipped --simple \ --skip-env-file --snapshot-report-unused --snapshot-update \ @@ -84,7 +84,7 @@ _bashunit_completions() { COMPREPLY=($(compgen -W "defined defects random" -- "$cur")) return 0 ;; - -f | --filter | --exclude-filter | --tag | --exclude-tag | --retry | --seed | --shard | --test-timeout) + -f | --filter | --exclude-filter | --tag | --exclude-tag | --repeat | --retry | --seed | --shard | --test-timeout) return 0 ;; esac diff --git a/docs/command-line.md b/docs/command-line.md index d35775c0..3d4481d9 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -80,6 +80,7 @@ bashunit test tests/ --parallel --simple | `-S, --stop-on-failure` | Stop on first failure | | `--test-timeout ` | Fail a test if it runs longer than N seconds | | `--retry ` | Re-run a failed test up to N extra times | +| `--repeat ` | Run each selected test N times; it fails if any iteration fails | | `--random-order` | Randomize test execution order | | `--order-by ` | Execution order: `defined` (default), `defects` or `random` | | `--seed ` | Seed for `--random-order` (reproducible shuffle) | @@ -823,6 +824,52 @@ BASHUNIT_RERUN_FAILED=true bashunit test tests/ ``` ::: +### Repeat + +> `bashunit test --repeat ` + +Run each selected test n times. Where [`--retry`](#test-options) mitigates +flakiness after it has already burned a CI run, `--repeat` goes looking for it: + +```bash +bashunit test tests/ --repeat 50 --filter flaky_candidate +``` + +The test is reported **once**, with the aggregate outcome, and the assertion +counts are those of the deciding iteration rather than the sum of all of them. +A failure names the iteration it happened on: + +``` +✗ Failed: My test + (failed on iteration 7 of 50) +``` + +Iterating stops at the first failing iteration: the test is already going to be +reported failed, and the remaining iterations cannot change that. + +**Interaction with `--retry`.** Repeat is the outer loop, retry the inner one. +Each iteration gets its full retry budget before the next iteration starts, so +`--repeat 2 --retry 1` runs the body at most four times, and an iteration that +recovers on its retry lets the next iteration begin. + +Notes: + +- `--repeat 1` behaves exactly as if the flag were absent. +- `--repeat 0`, a negative value and a non-numeric value are usage errors, not + silent no-ops. +- Per-test `set_up` / `tear_down` run once per iteration; `set_up_before_script` + runs once for the file, as always. +- Works under `--parallel`: each worker repeats its own test. + +::: code-group +```bash [Hammer one suspect test] +bashunit test --repeat 50 --filter flaky_candidate +``` +```bash [Env variable] +BASHUNIT_REPEAT=10 bashunit test tests/ +``` +::: + ### Flaky tests > `bashunit test --retry 2 --fail-on-flaky` diff --git a/src/config/env.sh b/src/config/env.sh index a32614fb..6e15455c 100644 --- a/src/config/env.sh +++ b/src/config/env.sh @@ -255,6 +255,8 @@ _BASHUNIT_DEFAULT_SHARD_INDEX="" _BASHUNIT_DEFAULT_SHARD_TOTAL="" # Replay only the tests recorded as failing by the previous run _BASHUNIT_DEFAULT_RERUN_FAILED="false" +# Run each selected test n times; the test fails if any iteration fails +_BASHUNIT_DEFAULT_REPEAT="1" # Treat a test that only passed after a retry as a failure for the exit code _BASHUNIT_DEFAULT_FAIL_ON_FLAKY="false" # Execution order: defined (definition order), defects (last run's failures @@ -313,6 +315,7 @@ _BASHUNIT_DEFAULT_SNAPSHOT_REPORT_UNUSED="false" # No bare ORDER_BY/FAIL_ON_FLAKY aliases, same reasoning as RETRY/SEED above. : "${BASHUNIT_ORDER_BY:=$_BASHUNIT_DEFAULT_ORDER_BY}" : "${BASHUNIT_FAIL_ON_FLAKY:=$_BASHUNIT_DEFAULT_FAIL_ON_FLAKY}" +: "${BASHUNIT_REPEAT:=$_BASHUNIT_DEFAULT_REPEAT}" : "${BASHUNIT_SHARD_INDEX:=$_BASHUNIT_DEFAULT_SHARD_INDEX}" : "${BASHUNIT_SHARD_TOTAL:=$_BASHUNIT_DEFAULT_SHARD_TOTAL}" # No bare RERUN_FAILED alias, same reasoning as RETRY/SEED above. The default @@ -373,6 +376,19 @@ function bashunit::env::test_timeout_secs() { # In-shell (no fork) so the per-test hot path can read the global instead of # capturing retry_count in a $(...) subshell every test (#764). _BASHUNIT_RETRY_VALIDATED=0 +_BASHUNIT_REPEAT_VALIDATED=1 +## +# Writes the validated repeat count into _BASHUNIT_REPEAT_VALIDATED. Mirrors +# resolve_retry_count: fork-free, and a value the validator would have rejected +# degrades to 1 rather than reaching the arithmetic in run_test. +## +function bashunit::env::resolve_repeat_count() { + case "${BASHUNIT_REPEAT:-1}" in + '' | *[!0-9]* | 0) _BASHUNIT_REPEAT_VALIDATED=1 ;; + *) _BASHUNIT_REPEAT_VALIDATED="${BASHUNIT_REPEAT:-1}" ;; + esac +} + function bashunit::env::resolve_retry_count() { case "${BASHUNIT_RETRY:-0}" in '' | *[!0-9]*) _BASHUNIT_RETRY_VALIDATED=0 ;; diff --git a/src/console/header.sh b/src/console/header.sh index b31660ba..0e2e23ae 100644 --- a/src/console/header.sh +++ b/src/console/header.sh @@ -138,6 +138,7 @@ Options: -S, --stop-on-failure Stop on first failure --test-timeout Fail a test if it runs longer than N seconds (0 = off) --retry Re-run a failed test up to N extra times (0 = off) + --repeat Run each selected test N times; it fails if any iteration fails --random-order Randomize test execution order --order-by Execution order: defined (default), defects (last run's failures first) or random --seed Seed for --random-order (reproducible shuffle) diff --git a/src/main/test.sh b/src/main/test.sh index 2f758fb4..a396b628 100644 --- a/src/main/test.sh +++ b/src/main/test.sh @@ -136,6 +136,11 @@ function bashunit::main::cmd_test() { export -n BASHUNIT_RETRY shift ;; + --repeat) + BASHUNIT_REPEAT="$2" + export -n BASHUNIT_REPEAT + shift + ;; --random-order) BASHUNIT_RANDOM_ORDER=true export -n BASHUNIT_RANDOM_ORDER diff --git a/src/main/validate.sh b/src/main/validate.sh index 2b92e186..609f4b2e 100644 --- a/src/main/validate.sh +++ b/src/main/validate.sh @@ -66,6 +66,15 @@ function bashunit::main::validate_config_or_exit() { "${BASHUNIT_RETRY:-0}" "BASHUNIT_RETRY (--retry)" bashunit::main::require_non_negative_int_or_exit \ "${BASHUNIT_TEST_TIMEOUT:-0}" "BASHUNIT_TEST_TIMEOUT (--test-timeout)" + bashunit::main::require_non_negative_int_or_exit \ + "${BASHUNIT_REPEAT:-1}" "BASHUNIT_REPEAT (--repeat)" + # 0 passes the non-negative check but means "run nothing", which would be a + # silent no-op rather than the usage error it is. + if [ "${BASHUNIT_REPEAT:-1}" -lt 1 ]; then + printf "%sError: BASHUNIT_REPEAT (--repeat) must be at least 1, got '%s'.%s\n" \ + "${_BASHUNIT_COLOR_FAILED}" "${BASHUNIT_REPEAT}" "${_BASHUNIT_COLOR_DEFAULT}" >&2 + exit 1 + fi # Empty is the documented "no minimum" default, so only a set value is checked. if [ -n "${BASHUNIT_COVERAGE_MIN:-}" ]; then bashunit::main::require_non_negative_int_or_exit \ diff --git a/src/runner/exec.sh b/src/runner/exec.sh index aa48b6dc..8497c59e 100644 --- a/src/runner/exec.sh +++ b/src/runner/exec.sh @@ -341,47 +341,69 @@ function bashunit::runner::run_test() { # failure -- the only evidence of what the flakiness looks like -- is kept here # before it is lost. local first_attempt_result="" + bashunit::env::resolve_repeat_count + local repeat_max=$_BASHUNIT_REPEAT_VALIDATED + local iteration=0 + local failed_iteration=0 local measure_duration=false bashunit::runner::needs_test_duration && measure_duration=true # Retry wraps ONLY execution: a failed attempt is judged from its encoded # result without committing, so the parse/report/counter path below still runs # exactly once (on the final attempt) and nothing is double-counted. Each fork # in --parallel retries itself before writing its single .result file. - while :; do - if [ "$measure_duration" = true ]; then - bashunit::clock::now_to_slot - start_time=$_BASHUNIT_CLOCK_NOW_OUT - fi - if bashunit::env::is_test_timeout_enabled; then - bashunit::runner::run_with_timeout "$test_file" "$fn_name" "$@" - test_execution_result="$_BASHUNIT_RUNNER_EXEC_OUT" - timed_out="$_BASHUNIT_RUNNER_TIMED_OUT" - else - test_execution_result=$(bashunit::runner::execute_test_body "$test_file" "$fn_name" "$@") - fi + # --repeat is the OUTER loop and --retry the inner one: an iteration gets its + # full retry budget before the next iteration starts. Iterating stops at the + # first failure, since the test is already going to be reported failed and the + # remaining iterations cannot change that. + while [ "$iteration" -lt "$repeat_max" ]; do + iteration=$((iteration + 1)) + retries_used=0 + first_attempt_result="" + while :; do + if [ "$measure_duration" = true ]; then + bashunit::clock::now_to_slot + start_time=$_BASHUNIT_CLOCK_NOW_OUT + fi + if bashunit::env::is_test_timeout_enabled; then + bashunit::runner::run_with_timeout "$test_file" "$fn_name" "$@" + test_execution_result="$_BASHUNIT_RUNNER_EXEC_OUT" + timed_out="$_BASHUNIT_RUNNER_TIMED_OUT" + else + test_execution_result=$(bashunit::runner::execute_test_body "$test_file" "$fn_name" "$@") + fi + + local attempt_runtime_output="${test_execution_result%%##ASSERTIONS_*}" + # Counts first: detect_runtime_error consults the exit code when the output + # text is translated and matches nothing. extract_result_counts is a pure + # read, so moving it ahead commits nothing. + bashunit::runner::extract_result_counts "$test_execution_result" + bashunit::runner::detect_runtime_error "$attempt_runtime_output" \ + "$_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT" + local attempt_runtime_error=$_BASHUNIT_RUNNER_RUNTIME_ERROR_OUT + local attempt_display_output=$_BASHUNIT_RUNNER_RUNTIME_OUTPUT_OUT + # Mirror the commit-phase failure test exactly (runtime error, non-zero exit, + # or a failed assertion); snapshot/incomplete/skipped/risky are not failures. + if [ -z "$attempt_runtime_error" ] && + [ "$_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT" -eq 0 ] && + [ "$_BASHUNIT_RUNNER_COUNTS_FAILED_OUT" -eq 0 ]; then + break + fi + # Only reached when the attempt failed, so this is the first failure. + if [ -z "$first_attempt_result" ]; then + first_attempt_result="$test_execution_result" + fi + [ "$retries_used" -ge "$retry_max" ] && break + retries_used=$((retries_used + 1)) + done - local attempt_runtime_output="${test_execution_result%%##ASSERTIONS_*}" - # Counts first: detect_runtime_error consults the exit code when the output - # text is translated and matches nothing. extract_result_counts is a pure - # read, so moving it ahead commits nothing. - bashunit::runner::extract_result_counts "$test_execution_result" - bashunit::runner::detect_runtime_error "$attempt_runtime_output" \ - "$_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT" - local attempt_runtime_error=$_BASHUNIT_RUNNER_RUNTIME_ERROR_OUT - local attempt_display_output=$_BASHUNIT_RUNNER_RUNTIME_OUTPUT_OUT - # Mirror the commit-phase failure test exactly (runtime error, non-zero exit, - # or a failed assertion); snapshot/incomplete/skipped/risky are not failures. - if [ -z "$attempt_runtime_error" ] && - [ "$_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT" -eq 0 ] && - [ "$_BASHUNIT_RUNNER_COUNTS_FAILED_OUT" -eq 0 ]; then + # The inner loop only exits with a failure once the retries are exhausted, + # so reaching here with one means this iteration is the verdict. + if [ -n "$attempt_runtime_error" ] || + [ "$_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT" -ne 0 ] || + [ "$_BASHUNIT_RUNNER_COUNTS_FAILED_OUT" -ne 0 ]; then + failed_iteration=$iteration break fi - # Only reached when the attempt failed, so this is the first failure. - if [ -z "$first_attempt_result" ]; then - first_attempt_result="$test_execution_result" - fi - [ "$retries_used" -ge "$retry_max" ] && break - retries_used=$((retries_used + 1)) done # The retry count lives in this shell, not in the test subshell that built the @@ -457,6 +479,13 @@ function bashunit::runner::run_test() { bashunit::state::reset_test_title bashunit::state::reset_current_test_interpolated_function_name + # Under --repeat the test is reported once, so the message has to say which + # iteration produced the failure or the count is unactionable. + local repeat_note="" + if [ "$repeat_max" -gt 1 ] && [ "$failed_iteration" -gt 0 ]; then + repeat_note=" (failed on iteration $failed_iteration of $repeat_max)" + fi + local failure_label="$label" local failure_function="$fn_name" if [ -n "$hook_failure" ]; then @@ -492,6 +521,7 @@ function bashunit::runner::run_test() { error_message="Test timed out after $(bashunit::env::test_timeout_secs)s" fi + error_message="$error_message$repeat_note" bashunit::console_results::print_error_test "$failure_function" "$error_message" "$runtime_output" bashunit::reports::add_test_failed "$test_file" "$failure_label" "$duration" "$total_assertions" "$error_message" bashunit::runner::write_failure_result_output "$test_file" "$failure_function" "$error_message" "$runtime_output" @@ -504,7 +534,11 @@ function bashunit::runner::run_test() { if [ "$current_assertions_failed" != "$_BASHUNIT_ASSERTIONS_FAILED" ]; then bashunit::state::add_tests_failed bashunit::rerun::record "$test_file" "$fn_name" - bashunit::reports::add_test_failed "$test_file" "$label" "$duration" "$total_assertions" "$subshell_output" + bashunit::reports::add_test_failed \ + "$test_file" "$label" "$duration" "$total_assertions" "$subshell_output$repeat_note" + if [ -n "$repeat_note" ]; then + bashunit::console_results::print_line "failed" "${repeat_note# }" + fi local assertion_runtime_output assertion_runtime_output="$( bashunit::runner::extract_assertion_runtime_output "$runtime_output" "$subshell_output" diff --git a/tests/acceptance/bashunit_repeat_test.sh b/tests/acceptance/bashunit_repeat_test.sh new file mode 100644 index 00000000..49609135 --- /dev/null +++ b/tests/acceptance/bashunit_repeat_test.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash + +# --repeat hammers each selected test n times to flush out flakiness before it +# reaches CI. The fixture tallies its body executions, so "did it really run n +# times" is asserted directly rather than inferred from the summary. + +function set_up_before_script() { + TEST_ENV_FILE="tests/acceptance/fixtures/.env.default" + FIXTURE="tests/acceptance/fixtures/test_bashunit_repeat.sh" +} + +function set_up() { + COUNTER_FILE="$(mktemp)" + export BASHUNIT_REPEAT_FIXTURE_COUNTER="$COUNTER_FILE" + printf '0' >"$COUNTER_FILE" +} + +function tear_down() { + rm -f "$COUNTER_FILE" + unset BASHUNIT_REPEAT_FIXTURE_COUNTER BASHUNIT_REPEAT_FIXTURE_FAIL_ON +} + +function test_repeat_runs_the_body_once_per_iteration() { + local output + output="$(./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --repeat 3 --filter test_repeat_counts "$FIXTURE")" + + assert_same "3" "$(cat "$COUNTER_FILE")" + assert_contains "Tests: 1 passed, 1 total" "$(printf '%s' "$output" | tr -s ' ')" +} + +function test_repeat_does_not_multiply_the_assertion_count() { + local output + output="$(./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --repeat 3 --filter test_repeat_counts "$FIXTURE")" + + assert_contains "Assertions: 1 passed, 1 total" "$(printf '%s' "$output" | tr -s ' ')" +} + +function test_repeat_reports_the_failing_iteration() { + export BASHUNIT_REPEAT_FIXTURE_FAIL_ON=2 + local output + output="$(./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --repeat 3 --filter test_repeat_fails "$FIXTURE")" || true + + assert_contains "Tests: 1 failed, 1 total" "$(printf '%s' "$output" | tr -s ' ')" + assert_contains "iteration 2 of 3" "$output" +} + +function test_repeat_stops_the_iterations_at_the_first_failure() { + export BASHUNIT_REPEAT_FIXTURE_FAIL_ON=2 + ./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --repeat 5 --filter test_repeat_fails "$FIXTURE" >/dev/null 2>&1 || true + + assert_same "2" "$(cat "$COUNTER_FILE")" +} + +function test_repeat_one_matches_no_flag_at_all() { + local with_flag without_flag + without_flag="$(./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --filter test_repeat_counts "$FIXTURE")" + printf '0' >"$COUNTER_FILE" + with_flag="$(./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --repeat 1 --filter test_repeat_counts "$FIXTURE")" + + assert_same "$without_flag" "$with_flag" +} + +# Repeat is the outer loop and retry the inner one: run 1 fails, the retry +# recovers it inside iteration 1, then iteration 2 runs the body once more. +function test_repeat_wraps_retry_rather_than_the_other_way_round() { + export BASHUNIT_REPEAT_FIXTURE_FAIL_ON=1 + local output + output="$(./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --repeat 2 --retry 1 --filter test_repeat_fails "$FIXTURE")" + + assert_contains "Tests: 1 passed" "$(printf '%s' "$output" | tr -s ' ')" + assert_same "3" "$(cat "$COUNTER_FILE")" +} + +function test_repeat_works_in_parallel() { + local output + output="$(./bashunit --parallel --no-color --env "$TEST_ENV_FILE" \ + --repeat 3 --filter test_repeat_counts "$FIXTURE")" + + assert_same "3" "$(cat "$COUNTER_FILE")" + assert_contains "Tests: 1 passed, 1 total" "$(printf '%s' "$output" | tr -s ' ')" +} + +function test_repeat_rejects_zero() { + local ec=0 + local output + output="$(./bashunit --no-parallel --env "$TEST_ENV_FILE" \ + --repeat 0 "$FIXTURE" 2>&1)" || ec=$? + + assert_general_error "" "" "$ec" + assert_contains "BASHUNIT_REPEAT" "$output" +} + +function test_repeat_rejects_a_non_numeric_value() { + local ec=0 + local output + output="$(./bashunit --no-parallel --env "$TEST_ENV_FILE" \ + --repeat abc "$FIXTURE" 2>&1)" || ec=$? + + assert_general_error "" "" "$ec" + assert_contains "BASHUNIT_REPEAT" "$output" +} + +function test_repeat_rejects_a_negative_value() { + local ec=0 + local output + output="$(./bashunit --no-parallel --env "$TEST_ENV_FILE" \ + --repeat -1 "$FIXTURE" 2>&1)" || ec=$? + + assert_general_error "" "" "$ec" + assert_contains "BASHUNIT_REPEAT" "$output" +} diff --git a/tests/acceptance/fixtures/test_bashunit_repeat.sh b/tests/acceptance/fixtures/test_bashunit_repeat.sh new file mode 100644 index 00000000..42fadb65 --- /dev/null +++ b/tests/acceptance/fixtures/test_bashunit_repeat.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# Body executions are tallied in a file that survives the iterations, so a test +# can assert how many times --repeat actually ran the body. +function _repeat_fixture_tick() { + local counter_file="${BASHUNIT_REPEAT_FIXTURE_COUNTER:?counter file required}" + local runs + runs=$(cat "$counter_file" 2>/dev/null || echo 0) + runs=$((runs + 1)) + printf '%s' "$runs" >"$counter_file" + echo "$runs" +} + +function test_repeat_counts_its_runs() { + _repeat_fixture_tick >/dev/null + assert_same "ok" "ok" +} + +# Fails only on the execution named by BASHUNIT_REPEAT_FIXTURE_FAIL_ON, so a +# test can place the failure on a chosen iteration. +function test_repeat_fails_on_a_chosen_run() { + local runs + runs=$(_repeat_fixture_tick) + + if [ "$runs" = "${BASHUNIT_REPEAT_FIXTURE_FAIL_ON:-0}" ]; then + assert_same "expected" "actual-on-run-$runs" + else + assert_same "ok" "ok" + fi +}