From 9a43a6d6a9508a30fec12d5726222f1821647b7e Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Mon, 10 Aug 2026 15:52:21 +0200 Subject: [PATCH] feat(cli): --repeat to run each test n times --retry mitigates flakiness after it has already burned a CI run. There was no way to go looking for it: proving a test deterministic meant running the suite N times by hand and diffing the output. --repeat wraps the existing retry loop rather than replacing it, so repeat is the outer loop and retry the inner one: each iteration gets its full retry budget before the next iteration starts. The test is reported once with the aggregate outcome, so the counters and every report see one entry, and the assertion counts are the deciding iteration's rather than the sum of all of them. Iterating stops at the first failing iteration, since the test is already going to be reported failed. The failure message names the iteration, which is the only way the count is actionable when the test appears once. 0, a negative value and a non-numeric value are usage errors rather than silent no-ops: 0 passes the non-negative check the other integer settings use, so it needs its own guard (#873 class of bug). Closes #1013 --- .env.example | 1 + CHANGELOG.md | 1 + completions/_bashunit | 1 + completions/bashunit.bash | 4 +- docs/command-line.md | 47 +++++++ src/config/env.sh | 16 +++ src/console/header.sh | 1 + src/main/test.sh | 5 + src/main/validate.sh | 9 ++ src/runner/exec.sh | 100 ++++++++++----- tests/acceptance/bashunit_repeat_test.sh | 118 ++++++++++++++++++ .../fixtures/test_bashunit_repeat.sh | 30 +++++ 12 files changed, 298 insertions(+), 35 deletions(-) create mode 100644 tests/acceptance/bashunit_repeat_test.sh create mode 100644 tests/acceptance/fixtures/test_bashunit_repeat.sh diff --git a/.env.example b/.env.example index 386d1e20..86f96141 100644 --- a/.env.example +++ b/.env.example @@ -44,6 +44,7 @@ BASHUNIT_STOP_ON_FAILURE= # Default: false (stop suite on first failur BASHUNIT_RERUN_FAILED= # Default: false (replay only last run's failing tests) BASHUNIT_ORDER_BY= # Default: defined (or defects, random) BASHUNIT_FAIL_ON_FLAKY= # Default: false (treat retry-passed tests as failed) +BASHUNIT_REPEAT= # Default: 1 (run each test N times) BASHUNIT_CHANGED= # Default: false (run only test files changed since a git ref) BASHUNIT_CHANGED_REF= # Default: empty (--changed ref: origin/HEAD, then HEAD) BASHUNIT_EXCLUDE_FILTER= # Default: empty (skip tests whose name matches) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3719366..f130f646 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased ### Added +- `--repeat ` runs each selected test n times so flakiness can be hunted before it reaches CI. The test is reported once with the aggregate outcome, a failure names the iteration it happened on, and repeat wraps `--retry` rather than the other way round (#1013) - Flaky is a first-class outcome: a test that only passed after a retry is counted separately, kept inside the pass total so the exit code is unchanged, and carried into JUnit (``), TAP, JSON, HTML and GitHub Actions along with the first attempt's failure message. `--fail-on-flaky` turns such a run red (#1012) - `--order-by ` picks the execution order: `defined` (default), `defects` (last run's failures first, whole suite still runs) or `random`. `--random-order` and `--seed` keep working unchanged (#1011) - `--changed []` runs only the test files git reports as touched since `` (default `origin/HEAD`, then `HEAD`), covering committed, staged, unstaged and untracked changes. Deletions are dropped, a rename selects its new path, and a missing work tree or unresolvable ref fails the run instead of selecting nothing (#1010) diff --git a/completions/_bashunit b/completions/_bashunit index f1cdbd4c..6e9c06fd 100644 --- a/completions/_bashunit +++ b/completions/_bashunit @@ -84,6 +84,7 @@ _bashunit() { '(-S --stop-on-failure)'{-S,--stop-on-failure}'[Stop on first failure]' \ '--test-timeout[Fail a test running longer than N seconds]:seconds:' \ '--retry[Rerun a failed test up to N extra times]:count:' \ + '--repeat[Run each selected test N times]:count:' \ '--random-order[Randomize test execution order]' \ '--fail-on-flaky[Treat tests that only passed after a retry as failures]' \ '--order-by[Execution order]:mode:(defined defects random)' \ diff --git a/completions/bashunit.bash b/completions/bashunit.bash index e6b3569f..4980b34a 100644 --- a/completions/bashunit.bash +++ b/completions/bashunit.bash @@ -20,7 +20,7 @@ _BASHUNIT_COMPLETIONS_TEST_OPTS="--assert --boot --changed --coverage --coverage --filter --help --jobs --list --list-format --log-gha --log-junit --login --no-color \ --no-coverage-report --no-output --no-output-on-failure --no-parallel \ --no-progress --no-snapshot-create --order-by --output --parallel --profile \ ---random-order --report-html \ +--random-order --repeat --report-html \ --report-json --report-junit --report-tap --rerun-failed --retry --run-all \ --seed --shard --show-incomplete --show-output --show-skipped --simple \ --skip-env-file --snapshot-report-unused --snapshot-update \ @@ -84,7 +84,7 @@ _bashunit_completions() { COMPREPLY=($(compgen -W "defined defects random" -- "$cur")) return 0 ;; - -f | --filter | --exclude-filter | --tag | --exclude-tag | --retry | --seed | --shard | --test-timeout) + -f | --filter | --exclude-filter | --tag | --exclude-tag | --repeat | --retry | --seed | --shard | --test-timeout) return 0 ;; esac diff --git a/docs/command-line.md b/docs/command-line.md index d35775c0..3d4481d9 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -80,6 +80,7 @@ bashunit test tests/ --parallel --simple | `-S, --stop-on-failure` | Stop on first failure | | `--test-timeout ` | Fail a test if it runs longer than N seconds | | `--retry ` | Re-run a failed test up to N extra times | +| `--repeat ` | Run each selected test N times; it fails if any iteration fails | | `--random-order` | Randomize test execution order | | `--order-by ` | Execution order: `defined` (default), `defects` or `random` | | `--seed ` | Seed for `--random-order` (reproducible shuffle) | @@ -823,6 +824,52 @@ BASHUNIT_RERUN_FAILED=true bashunit test tests/ ``` ::: +### Repeat + +> `bashunit test --repeat ` + +Run each selected test n times. Where [`--retry`](#test-options) mitigates +flakiness after it has already burned a CI run, `--repeat` goes looking for it: + +```bash +bashunit test tests/ --repeat 50 --filter flaky_candidate +``` + +The test is reported **once**, with the aggregate outcome, and the assertion +counts are those of the deciding iteration rather than the sum of all of them. +A failure names the iteration it happened on: + +``` +✗ Failed: My test + (failed on iteration 7 of 50) +``` + +Iterating stops at the first failing iteration: the test is already going to be +reported failed, and the remaining iterations cannot change that. + +**Interaction with `--retry`.** Repeat is the outer loop, retry the inner one. +Each iteration gets its full retry budget before the next iteration starts, so +`--repeat 2 --retry 1` runs the body at most four times, and an iteration that +recovers on its retry lets the next iteration begin. + +Notes: + +- `--repeat 1` behaves exactly as if the flag were absent. +- `--repeat 0`, a negative value and a non-numeric value are usage errors, not + silent no-ops. +- Per-test `set_up` / `tear_down` run once per iteration; `set_up_before_script` + runs once for the file, as always. +- Works under `--parallel`: each worker repeats its own test. + +::: code-group +```bash [Hammer one suspect test] +bashunit test --repeat 50 --filter flaky_candidate +``` +```bash [Env variable] +BASHUNIT_REPEAT=10 bashunit test tests/ +``` +::: + ### Flaky tests > `bashunit test --retry 2 --fail-on-flaky` diff --git a/src/config/env.sh b/src/config/env.sh index a32614fb..6e15455c 100644 --- a/src/config/env.sh +++ b/src/config/env.sh @@ -255,6 +255,8 @@ _BASHUNIT_DEFAULT_SHARD_INDEX="" _BASHUNIT_DEFAULT_SHARD_TOTAL="" # Replay only the tests recorded as failing by the previous run _BASHUNIT_DEFAULT_RERUN_FAILED="false" +# Run each selected test n times; the test fails if any iteration fails +_BASHUNIT_DEFAULT_REPEAT="1" # Treat a test that only passed after a retry as a failure for the exit code _BASHUNIT_DEFAULT_FAIL_ON_FLAKY="false" # Execution order: defined (definition order), defects (last run's failures @@ -313,6 +315,7 @@ _BASHUNIT_DEFAULT_SNAPSHOT_REPORT_UNUSED="false" # No bare ORDER_BY/FAIL_ON_FLAKY aliases, same reasoning as RETRY/SEED above. : "${BASHUNIT_ORDER_BY:=$_BASHUNIT_DEFAULT_ORDER_BY}" : "${BASHUNIT_FAIL_ON_FLAKY:=$_BASHUNIT_DEFAULT_FAIL_ON_FLAKY}" +: "${BASHUNIT_REPEAT:=$_BASHUNIT_DEFAULT_REPEAT}" : "${BASHUNIT_SHARD_INDEX:=$_BASHUNIT_DEFAULT_SHARD_INDEX}" : "${BASHUNIT_SHARD_TOTAL:=$_BASHUNIT_DEFAULT_SHARD_TOTAL}" # No bare RERUN_FAILED alias, same reasoning as RETRY/SEED above. The default @@ -373,6 +376,19 @@ function bashunit::env::test_timeout_secs() { # In-shell (no fork) so the per-test hot path can read the global instead of # capturing retry_count in a $(...) subshell every test (#764). _BASHUNIT_RETRY_VALIDATED=0 +_BASHUNIT_REPEAT_VALIDATED=1 +## +# Writes the validated repeat count into _BASHUNIT_REPEAT_VALIDATED. Mirrors +# resolve_retry_count: fork-free, and a value the validator would have rejected +# degrades to 1 rather than reaching the arithmetic in run_test. +## +function bashunit::env::resolve_repeat_count() { + case "${BASHUNIT_REPEAT:-1}" in + '' | *[!0-9]* | 0) _BASHUNIT_REPEAT_VALIDATED=1 ;; + *) _BASHUNIT_REPEAT_VALIDATED="${BASHUNIT_REPEAT:-1}" ;; + esac +} + function bashunit::env::resolve_retry_count() { case "${BASHUNIT_RETRY:-0}" in '' | *[!0-9]*) _BASHUNIT_RETRY_VALIDATED=0 ;; diff --git a/src/console/header.sh b/src/console/header.sh index b31660ba..0e2e23ae 100644 --- a/src/console/header.sh +++ b/src/console/header.sh @@ -138,6 +138,7 @@ Options: -S, --stop-on-failure Stop on first failure --test-timeout Fail a test if it runs longer than N seconds (0 = off) --retry Re-run a failed test up to N extra times (0 = off) + --repeat Run each selected test N times; it fails if any iteration fails --random-order Randomize test execution order --order-by Execution order: defined (default), defects (last run's failures first) or random --seed Seed for --random-order (reproducible shuffle) diff --git a/src/main/test.sh b/src/main/test.sh index 2f758fb4..a396b628 100644 --- a/src/main/test.sh +++ b/src/main/test.sh @@ -136,6 +136,11 @@ function bashunit::main::cmd_test() { export -n BASHUNIT_RETRY shift ;; + --repeat) + BASHUNIT_REPEAT="$2" + export -n BASHUNIT_REPEAT + shift + ;; --random-order) BASHUNIT_RANDOM_ORDER=true export -n BASHUNIT_RANDOM_ORDER diff --git a/src/main/validate.sh b/src/main/validate.sh index 2b92e186..609f4b2e 100644 --- a/src/main/validate.sh +++ b/src/main/validate.sh @@ -66,6 +66,15 @@ function bashunit::main::validate_config_or_exit() { "${BASHUNIT_RETRY:-0}" "BASHUNIT_RETRY (--retry)" bashunit::main::require_non_negative_int_or_exit \ "${BASHUNIT_TEST_TIMEOUT:-0}" "BASHUNIT_TEST_TIMEOUT (--test-timeout)" + bashunit::main::require_non_negative_int_or_exit \ + "${BASHUNIT_REPEAT:-1}" "BASHUNIT_REPEAT (--repeat)" + # 0 passes the non-negative check but means "run nothing", which would be a + # silent no-op rather than the usage error it is. + if [ "${BASHUNIT_REPEAT:-1}" -lt 1 ]; then + printf "%sError: BASHUNIT_REPEAT (--repeat) must be at least 1, got '%s'.%s\n" \ + "${_BASHUNIT_COLOR_FAILED}" "${BASHUNIT_REPEAT}" "${_BASHUNIT_COLOR_DEFAULT}" >&2 + exit 1 + fi # Empty is the documented "no minimum" default, so only a set value is checked. if [ -n "${BASHUNIT_COVERAGE_MIN:-}" ]; then bashunit::main::require_non_negative_int_or_exit \ diff --git a/src/runner/exec.sh b/src/runner/exec.sh index aa48b6dc..8497c59e 100644 --- a/src/runner/exec.sh +++ b/src/runner/exec.sh @@ -341,47 +341,69 @@ function bashunit::runner::run_test() { # failure -- the only evidence of what the flakiness looks like -- is kept here # before it is lost. local first_attempt_result="" + bashunit::env::resolve_repeat_count + local repeat_max=$_BASHUNIT_REPEAT_VALIDATED + local iteration=0 + local failed_iteration=0 local measure_duration=false bashunit::runner::needs_test_duration && measure_duration=true # Retry wraps ONLY execution: a failed attempt is judged from its encoded # result without committing, so the parse/report/counter path below still runs # exactly once (on the final attempt) and nothing is double-counted. Each fork # in --parallel retries itself before writing its single .result file. - while :; do - if [ "$measure_duration" = true ]; then - bashunit::clock::now_to_slot - start_time=$_BASHUNIT_CLOCK_NOW_OUT - fi - if bashunit::env::is_test_timeout_enabled; then - bashunit::runner::run_with_timeout "$test_file" "$fn_name" "$@" - test_execution_result="$_BASHUNIT_RUNNER_EXEC_OUT" - timed_out="$_BASHUNIT_RUNNER_TIMED_OUT" - else - test_execution_result=$(bashunit::runner::execute_test_body "$test_file" "$fn_name" "$@") - fi + # --repeat is the OUTER loop and --retry the inner one: an iteration gets its + # full retry budget before the next iteration starts. Iterating stops at the + # first failure, since the test is already going to be reported failed and the + # remaining iterations cannot change that. + while [ "$iteration" -lt "$repeat_max" ]; do + iteration=$((iteration + 1)) + retries_used=0 + first_attempt_result="" + while :; do + if [ "$measure_duration" = true ]; then + bashunit::clock::now_to_slot + start_time=$_BASHUNIT_CLOCK_NOW_OUT + fi + if bashunit::env::is_test_timeout_enabled; then + bashunit::runner::run_with_timeout "$test_file" "$fn_name" "$@" + test_execution_result="$_BASHUNIT_RUNNER_EXEC_OUT" + timed_out="$_BASHUNIT_RUNNER_TIMED_OUT" + else + test_execution_result=$(bashunit::runner::execute_test_body "$test_file" "$fn_name" "$@") + fi + + local attempt_runtime_output="${test_execution_result%%##ASSERTIONS_*}" + # Counts first: detect_runtime_error consults the exit code when the output + # text is translated and matches nothing. extract_result_counts is a pure + # read, so moving it ahead commits nothing. + bashunit::runner::extract_result_counts "$test_execution_result" + bashunit::runner::detect_runtime_error "$attempt_runtime_output" \ + "$_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT" + local attempt_runtime_error=$_BASHUNIT_RUNNER_RUNTIME_ERROR_OUT + local attempt_display_output=$_BASHUNIT_RUNNER_RUNTIME_OUTPUT_OUT + # Mirror the commit-phase failure test exactly (runtime error, non-zero exit, + # or a failed assertion); snapshot/incomplete/skipped/risky are not failures. + if [ -z "$attempt_runtime_error" ] && + [ "$_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT" -eq 0 ] && + [ "$_BASHUNIT_RUNNER_COUNTS_FAILED_OUT" -eq 0 ]; then + break + fi + # Only reached when the attempt failed, so this is the first failure. + if [ -z "$first_attempt_result" ]; then + first_attempt_result="$test_execution_result" + fi + [ "$retries_used" -ge "$retry_max" ] && break + retries_used=$((retries_used + 1)) + done - local attempt_runtime_output="${test_execution_result%%##ASSERTIONS_*}" - # Counts first: detect_runtime_error consults the exit code when the output - # text is translated and matches nothing. extract_result_counts is a pure - # read, so moving it ahead commits nothing. - bashunit::runner::extract_result_counts "$test_execution_result" - bashunit::runner::detect_runtime_error "$attempt_runtime_output" \ - "$_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT" - local attempt_runtime_error=$_BASHUNIT_RUNNER_RUNTIME_ERROR_OUT - local attempt_display_output=$_BASHUNIT_RUNNER_RUNTIME_OUTPUT_OUT - # Mirror the commit-phase failure test exactly (runtime error, non-zero exit, - # or a failed assertion); snapshot/incomplete/skipped/risky are not failures. - if [ -z "$attempt_runtime_error" ] && - [ "$_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT" -eq 0 ] && - [ "$_BASHUNIT_RUNNER_COUNTS_FAILED_OUT" -eq 0 ]; then + # The inner loop only exits with a failure once the retries are exhausted, + # so reaching here with one means this iteration is the verdict. + if [ -n "$attempt_runtime_error" ] || + [ "$_BASHUNIT_RUNNER_COUNTS_EXIT_CODE_OUT" -ne 0 ] || + [ "$_BASHUNIT_RUNNER_COUNTS_FAILED_OUT" -ne 0 ]; then + failed_iteration=$iteration break fi - # Only reached when the attempt failed, so this is the first failure. - if [ -z "$first_attempt_result" ]; then - first_attempt_result="$test_execution_result" - fi - [ "$retries_used" -ge "$retry_max" ] && break - retries_used=$((retries_used + 1)) done # The retry count lives in this shell, not in the test subshell that built the @@ -457,6 +479,13 @@ function bashunit::runner::run_test() { bashunit::state::reset_test_title bashunit::state::reset_current_test_interpolated_function_name + # Under --repeat the test is reported once, so the message has to say which + # iteration produced the failure or the count is unactionable. + local repeat_note="" + if [ "$repeat_max" -gt 1 ] && [ "$failed_iteration" -gt 0 ]; then + repeat_note=" (failed on iteration $failed_iteration of $repeat_max)" + fi + local failure_label="$label" local failure_function="$fn_name" if [ -n "$hook_failure" ]; then @@ -492,6 +521,7 @@ function bashunit::runner::run_test() { error_message="Test timed out after $(bashunit::env::test_timeout_secs)s" fi + error_message="$error_message$repeat_note" bashunit::console_results::print_error_test "$failure_function" "$error_message" "$runtime_output" bashunit::reports::add_test_failed "$test_file" "$failure_label" "$duration" "$total_assertions" "$error_message" bashunit::runner::write_failure_result_output "$test_file" "$failure_function" "$error_message" "$runtime_output" @@ -504,7 +534,11 @@ function bashunit::runner::run_test() { if [ "$current_assertions_failed" != "$_BASHUNIT_ASSERTIONS_FAILED" ]; then bashunit::state::add_tests_failed bashunit::rerun::record "$test_file" "$fn_name" - bashunit::reports::add_test_failed "$test_file" "$label" "$duration" "$total_assertions" "$subshell_output" + bashunit::reports::add_test_failed \ + "$test_file" "$label" "$duration" "$total_assertions" "$subshell_output$repeat_note" + if [ -n "$repeat_note" ]; then + bashunit::console_results::print_line "failed" "${repeat_note# }" + fi local assertion_runtime_output assertion_runtime_output="$( bashunit::runner::extract_assertion_runtime_output "$runtime_output" "$subshell_output" diff --git a/tests/acceptance/bashunit_repeat_test.sh b/tests/acceptance/bashunit_repeat_test.sh new file mode 100644 index 00000000..49609135 --- /dev/null +++ b/tests/acceptance/bashunit_repeat_test.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash + +# --repeat hammers each selected test n times to flush out flakiness before it +# reaches CI. The fixture tallies its body executions, so "did it really run n +# times" is asserted directly rather than inferred from the summary. + +function set_up_before_script() { + TEST_ENV_FILE="tests/acceptance/fixtures/.env.default" + FIXTURE="tests/acceptance/fixtures/test_bashunit_repeat.sh" +} + +function set_up() { + COUNTER_FILE="$(mktemp)" + export BASHUNIT_REPEAT_FIXTURE_COUNTER="$COUNTER_FILE" + printf '0' >"$COUNTER_FILE" +} + +function tear_down() { + rm -f "$COUNTER_FILE" + unset BASHUNIT_REPEAT_FIXTURE_COUNTER BASHUNIT_REPEAT_FIXTURE_FAIL_ON +} + +function test_repeat_runs_the_body_once_per_iteration() { + local output + output="$(./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --repeat 3 --filter test_repeat_counts "$FIXTURE")" + + assert_same "3" "$(cat "$COUNTER_FILE")" + assert_contains "Tests: 1 passed, 1 total" "$(printf '%s' "$output" | tr -s ' ')" +} + +function test_repeat_does_not_multiply_the_assertion_count() { + local output + output="$(./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --repeat 3 --filter test_repeat_counts "$FIXTURE")" + + assert_contains "Assertions: 1 passed, 1 total" "$(printf '%s' "$output" | tr -s ' ')" +} + +function test_repeat_reports_the_failing_iteration() { + export BASHUNIT_REPEAT_FIXTURE_FAIL_ON=2 + local output + output="$(./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --repeat 3 --filter test_repeat_fails "$FIXTURE")" || true + + assert_contains "Tests: 1 failed, 1 total" "$(printf '%s' "$output" | tr -s ' ')" + assert_contains "iteration 2 of 3" "$output" +} + +function test_repeat_stops_the_iterations_at_the_first_failure() { + export BASHUNIT_REPEAT_FIXTURE_FAIL_ON=2 + ./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --repeat 5 --filter test_repeat_fails "$FIXTURE" >/dev/null 2>&1 || true + + assert_same "2" "$(cat "$COUNTER_FILE")" +} + +function test_repeat_one_matches_no_flag_at_all() { + local with_flag without_flag + without_flag="$(./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --filter test_repeat_counts "$FIXTURE")" + printf '0' >"$COUNTER_FILE" + with_flag="$(./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --repeat 1 --filter test_repeat_counts "$FIXTURE")" + + assert_same "$without_flag" "$with_flag" +} + +# Repeat is the outer loop and retry the inner one: run 1 fails, the retry +# recovers it inside iteration 1, then iteration 2 runs the body once more. +function test_repeat_wraps_retry_rather_than_the_other_way_round() { + export BASHUNIT_REPEAT_FIXTURE_FAIL_ON=1 + local output + output="$(./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --repeat 2 --retry 1 --filter test_repeat_fails "$FIXTURE")" + + assert_contains "Tests: 1 passed" "$(printf '%s' "$output" | tr -s ' ')" + assert_same "3" "$(cat "$COUNTER_FILE")" +} + +function test_repeat_works_in_parallel() { + local output + output="$(./bashunit --parallel --no-color --env "$TEST_ENV_FILE" \ + --repeat 3 --filter test_repeat_counts "$FIXTURE")" + + assert_same "3" "$(cat "$COUNTER_FILE")" + assert_contains "Tests: 1 passed, 1 total" "$(printf '%s' "$output" | tr -s ' ')" +} + +function test_repeat_rejects_zero() { + local ec=0 + local output + output="$(./bashunit --no-parallel --env "$TEST_ENV_FILE" \ + --repeat 0 "$FIXTURE" 2>&1)" || ec=$? + + assert_general_error "" "" "$ec" + assert_contains "BASHUNIT_REPEAT" "$output" +} + +function test_repeat_rejects_a_non_numeric_value() { + local ec=0 + local output + output="$(./bashunit --no-parallel --env "$TEST_ENV_FILE" \ + --repeat abc "$FIXTURE" 2>&1)" || ec=$? + + assert_general_error "" "" "$ec" + assert_contains "BASHUNIT_REPEAT" "$output" +} + +function test_repeat_rejects_a_negative_value() { + local ec=0 + local output + output="$(./bashunit --no-parallel --env "$TEST_ENV_FILE" \ + --repeat -1 "$FIXTURE" 2>&1)" || ec=$? + + assert_general_error "" "" "$ec" + assert_contains "BASHUNIT_REPEAT" "$output" +} diff --git a/tests/acceptance/fixtures/test_bashunit_repeat.sh b/tests/acceptance/fixtures/test_bashunit_repeat.sh new file mode 100644 index 00000000..42fadb65 --- /dev/null +++ b/tests/acceptance/fixtures/test_bashunit_repeat.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# Body executions are tallied in a file that survives the iterations, so a test +# can assert how many times --repeat actually ran the body. +function _repeat_fixture_tick() { + local counter_file="${BASHUNIT_REPEAT_FIXTURE_COUNTER:?counter file required}" + local runs + runs=$(cat "$counter_file" 2>/dev/null || echo 0) + runs=$((runs + 1)) + printf '%s' "$runs" >"$counter_file" + echo "$runs" +} + +function test_repeat_counts_its_runs() { + _repeat_fixture_tick >/dev/null + assert_same "ok" "ok" +} + +# Fails only on the execution named by BASHUNIT_REPEAT_FIXTURE_FAIL_ON, so a +# test can place the failure on a chosen iteration. +function test_repeat_fails_on_a_chosen_run() { + local runs + runs=$(_repeat_fixture_tick) + + if [ "$runs" = "${BASHUNIT_REPEAT_FIXTURE_FAIL_ON:-0}" ]; then + assert_same "expected" "actual-on-run-$runs" + else + assert_same "ok" "ok" + fi +}