From 3ff71f736afefecfd77f84f4d3680a144d505415 Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Mon, 10 Aug 2026 16:07:21 +0200 Subject: [PATCH 1/2] fix(reports): GitHub Actions annotations never reached the pull request GitHub parses workflow commands from the job log, and bashunit only ever wrote them to a file. print_gha_annotations already emitted correct `::error file=...,line=...` lines, but the sole caller redirected them into $BASHUNIT_LOG_GHA, so a file nobody cat'd produced zero annotations. The flag was also missing from `bashunit test --help`, and the official action never wired any of it, so users of TypedDevs/bashunit@v0 got nothing at all. Annotations now go to stdout, automatically, whenever GITHUB_ACTIONS is true. --gha-annotations auto|always|never overrides the detection. `auto` also stays quiet under --output tap, whose stdout is a machine format an annotation line would corrupt. reports::is_enabled had to learn about this too: it gates row collection, so without it the arrays stayed empty and there was nothing to annotate when no other report flag was set. Printing happens after load_spooled so a --parallel run annotates the rows its workers spooled, which the parent would otherwise never have seen (#1004). --log-gha keeps writing the file and is now documented. The two sinks are independent, so using both does not duplicate anything in the job log. Closes #1014 --- .env.example | 1 + CHANGELOG.md | 1 + action.yml | 5 + completions/_bashunit | 1 + completions/bashunit.bash | 6 +- docs/command-line.md | 40 +++++++- src/config/env.sh | 19 ++++ src/console/header.sh | 2 + src/main/run.sh | 7 ++ src/main/test.sh | 5 + src/main/validate.sh | 11 +++ src/reports/collect.sh | 3 +- .../bashunit_gha_annotations_test.sh | 95 +++++++++++++++++++ 13 files changed, 191 insertions(+), 5 deletions(-) create mode 100644 tests/acceptance/bashunit_gha_annotations_test.sh diff --git a/.env.example b/.env.example index 86f96141..ef2125e2 100644 --- a/.env.example +++ b/.env.example @@ -45,6 +45,7 @@ BASHUNIT_RERUN_FAILED= # Default: false (replay only last run's fai BASHUNIT_ORDER_BY= # Default: defined (or defects, random) BASHUNIT_FAIL_ON_FLAKY= # Default: false (treat retry-passed tests as failed) BASHUNIT_REPEAT= # Default: 1 (run each test N times) +BASHUNIT_GHA_ANNOTATIONS= # Default: auto (or always, never) BASHUNIT_CHANGED= # Default: false (run only test files changed since a git ref) BASHUNIT_CHANGED_REF= # Default: empty (--changed ref: origin/HEAD, then HEAD) BASHUNIT_EXCLUDE_FILTER= # Default: empty (skip tests whose name matches) diff --git a/CHANGELOG.md b/CHANGELOG.md index f130f646..abd55725 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased ### Added +- `--gha-annotations ` controls GitHub Actions annotations on stdout; `auto` turns them on inside GitHub Actions and stays quiet everywhere else (#1014) - `--repeat ` runs each selected test n times so flakiness can be hunted before it reaches CI. The test is reported once with the aggregate outcome, a failure names the iteration it happened on, and repeat wraps `--retry` rather than the other way round (#1013) - Flaky is a first-class outcome: a test that only passed after a retry is counted separately, kept inside the pass total so the exit code is unchanged, and carried into JUnit (``), TAP, JSON, HTML and GitHub Actions along with the first attempt's failure message. `--fail-on-flaky` turns such a run red (#1012) - `--order-by ` picks the execution order: `defined` (default), `defects` (last run's failures first, whole suite still runs) or `random`. `--random-order` and `--seed` keep working unchanged (#1011) diff --git a/action.yml b/action.yml index b38035d5..78bd9f0c 100644 --- a/action.yml +++ b/action.yml @@ -27,6 +27,10 @@ inputs: description: 'If set, run "bashunit " after installing (e.g. "tests/ --strict"). Empty = install only.' required: false default: '' + annotations: + description: 'Annotate failing tests on the pull request. bashunit detects GitHub Actions on its own, so this only needs setting to turn them off ("never").' + required: false + default: 'auto' outputs: path: @@ -47,6 +51,7 @@ runs: BASHUNIT_ADD_TO_PATH: ${{ inputs.add-to-path }} BASHUNIT_VERIFY_CHECKSUM: ${{ inputs.verify-checksum }} BASHUNIT_ARGS: ${{ inputs.args }} + BASHUNIT_GHA_ANNOTATIONS: ${{ inputs.annotations }} BASHUNIT_ACTION_PATH: ${{ github.action_path }} run: | set -euo pipefail diff --git a/completions/_bashunit b/completions/_bashunit index 6e9c06fd..06a8a8c5 100644 --- a/completions/_bashunit +++ b/completions/_bashunit @@ -69,6 +69,7 @@ _bashunit() { '--exclude-filter[Skip tests whose name matches]:name:' \ '--exclude-tag[Skip tests with matching @tag]:tag:' \ '--log-junit[Write JUnit XML report]:file:_files' \ + '--gha-annotations[GitHub Actions annotations on stdout]:mode:(auto always never)' \ '--report-junit[Write JUnit XML report]:file:_files' \ '--log-gha[Write GitHub Actions workflow-commands log]:file:_files' \ '(-j --jobs)'{-j,--jobs}'[Max N parallel jobs, or auto]:jobs:(auto)' \ diff --git a/completions/bashunit.bash b/completions/bashunit.bash index 4980b34a..fcdd029d 100644 --- a/completions/bashunit.bash +++ b/completions/bashunit.bash @@ -17,7 +17,7 @@ _BASHUNIT_COMPLETIONS_DOC_OPTS="--custom -e --env --boot -h --help" _BASHUNIT_COMPLETIONS_TEST_OPTS="--assert --boot --changed --coverage --coverage-exclude \ --coverage-min --coverage-paths --coverage-report --coverage-report-html \ --debug --detailed --dry-run --env --exclude-filter --exclude-tag --fail-on-flaky --fail-on-risky --failures-only \ ---filter --help --jobs --list --list-format --log-gha --log-junit --login --no-color \ +--filter --gha-annotations --help --jobs --list --list-format --log-gha --log-junit --login --no-color \ --no-coverage-report --no-output --no-output-on-failure --no-parallel \ --no-progress --no-snapshot-create --order-by --output --parallel --profile \ --random-order --repeat --report-html \ @@ -84,6 +84,10 @@ _bashunit_completions() { COMPREPLY=($(compgen -W "defined defects random" -- "$cur")) return 0 ;; + --gha-annotations) + COMPREPLY=($(compgen -W "auto always never" -- "$cur")) + return 0 + ;; -f | --filter | --exclude-filter | --tag | --exclude-tag | --repeat | --retry | --seed | --shard | --test-timeout) return 0 ;; diff --git a/docs/command-line.md b/docs/command-line.md index 3d4481d9..ff750910 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -68,6 +68,7 @@ bashunit test tests/ --parallel --simple | `-w, --watch` | Watch files and re-run tests on change | | `--log-junit, --report-junit ` | Write JUnit XML report | | `--log-gha ` | Write GitHub Actions workflow-commands log | +| `--gha-annotations ` | Annotations on stdout: `auto` (default), `always` or `never` | | `-j, --jobs ` | Run tests in parallel with max N concurrent jobs (`auto` = CPU cores) | | `-p, --parallel` | Run tests in parallel | | `--no-parallel` | Run tests sequentially | @@ -448,15 +449,48 @@ bashunit test tests/ --log-junit results.xml bashunit test tests/ --report-html report.html ``` ```bash [GitHub Actions] -# Stream annotations straight to the runner log: -bashunit test tests/ --log-gha /dev/stdout +# Nothing to configure: annotations are automatic on a runner. +bashunit test tests/ ``` ```bash [JSON] bashunit test tests/ --report-json report.json ``` ::: -The `--log-gha` flag writes GitHub Actions workflow commands (`::error`, `::warning`, `::notice`) for failed, risky and incomplete tests, including the failing test's `file` and `line`. Point it at `/dev/stdout` (or stream a log file to stdout) on a runner and the failures appear as inline annotations in the "Files changed" tab of a pull request. +### GitHub Actions annotations + +Inside GitHub Actions, bashunit annotates failing tests on the pull request by +itself. No flag, no configuration: + +``` +::error file=tests/math_test.sh,line=42,title=Sums::Expected '4' but got '5' +``` + +GitHub parses workflow commands from the **job log**, so the annotations go to +stdout. They carry the failing test's `file` and `line`, which is what puts them +on the right line of the "Files changed" tab: `::error` for failures, `::warning` +for risky and flaky tests, `::notice` for incomplete ones. Messages are +percent-encoded, so a multi-line failure stays a single annotation. + +Detection is `GITHUB_ACTIONS=true`, and `--gha-annotations` overrides it: + +| Mode | Behaviour | +|------|-----------| +| `auto` | On inside GitHub Actions, silent everywhere else. The default. | +| `always` | On everywhere, useful for another CI that understands the format | +| `never` | Off, including inside GitHub Actions | + +```bash +bashunit test tests/ --gha-annotations never +``` + +`auto` also stays quiet under `--output tap`, whose stdout is a machine format an +annotation line would corrupt. + +The separate `--log-gha ` flag still writes the same workflow commands to a +file. It is independent of the stdout annotations, so using both does not +duplicate anything in the job log. + The `--report-json` flag writes machine-readable results for scripts, dashboards and bots. Strings are escaped in pure Bash, so no `jq` is needed to produce it. Its schema is: diff --git a/src/config/env.sh b/src/config/env.sh index 6e15455c..58642cf3 100644 --- a/src/config/env.sh +++ b/src/config/env.sh @@ -255,6 +255,8 @@ _BASHUNIT_DEFAULT_SHARD_INDEX="" _BASHUNIT_DEFAULT_SHARD_TOTAL="" # Replay only the tests recorded as failing by the previous run _BASHUNIT_DEFAULT_RERUN_FAILED="false" +# When to print GitHub Actions annotations to stdout: auto|always|never +_BASHUNIT_DEFAULT_GHA_ANNOTATIONS="auto" # Run each selected test n times; the test fails if any iteration fails _BASHUNIT_DEFAULT_REPEAT="1" # Treat a test that only passed after a retry as a failure for the exit code @@ -316,6 +318,7 @@ _BASHUNIT_DEFAULT_SNAPSHOT_REPORT_UNUSED="false" : "${BASHUNIT_ORDER_BY:=$_BASHUNIT_DEFAULT_ORDER_BY}" : "${BASHUNIT_FAIL_ON_FLAKY:=$_BASHUNIT_DEFAULT_FAIL_ON_FLAKY}" : "${BASHUNIT_REPEAT:=$_BASHUNIT_DEFAULT_REPEAT}" +: "${BASHUNIT_GHA_ANNOTATIONS:=$_BASHUNIT_DEFAULT_GHA_ANNOTATIONS}" : "${BASHUNIT_SHARD_INDEX:=$_BASHUNIT_DEFAULT_SHARD_INDEX}" : "${BASHUNIT_SHARD_TOTAL:=$_BASHUNIT_DEFAULT_SHARD_TOTAL}" # No bare RERUN_FAILED alias, same reasoning as RETRY/SEED above. The default @@ -632,6 +635,22 @@ function bashunit::env::is_fail_on_risky_enabled() { [ "$BASHUNIT_FAIL_ON_RISKY" = "true" ] } +## +# Whether workflow-command annotations go to stdout. GitHub parses them from the +# job log, so stdout is the only sink that reaches a pull request. +# +# `auto` stays quiet outside GitHub Actions, and quiet under --output tap, whose +# stdout is a machine format an annotation line would corrupt. +## +function bashunit::env::should_print_gha_annotations() { + case "${BASHUNIT_GHA_ANNOTATIONS:-auto}" in + never) return 1 ;; + always) return 0 ;; + esac + + [ "${GITHUB_ACTIONS:-}" = "true" ] && ! bashunit::env::is_tap_output_enabled +} + function bashunit::env::is_fail_on_flaky_enabled() { [ "${BASHUNIT_FAIL_ON_FLAKY:-false}" = "true" ] } diff --git a/src/console/header.sh b/src/console/header.sh index 0e2e23ae..e00076bc 100644 --- a/src/console/header.sh +++ b/src/console/header.sh @@ -125,6 +125,8 @@ Options: Supports 'a&&b' (AND) and '!a' (NOT) --exclude-tag Skip tests with matching @tag (repeatable, exclude wins) --log-junit, --report-junit Write JUnit XML report + --log-gha Write GitHub Actions annotations to a file + --gha-annotations Annotations on stdout: auto (in GitHub Actions), always or never -j, --jobs Run tests in parallel with max N concurrent jobs ("auto" = CPU cores) -p, --parallel Run tests in parallel (unlimited concurrency) --no-parallel Run tests sequentially diff --git a/src/main/run.sh b/src/main/run.sh index 68f747de..d465a3cb 100644 --- a/src/main/run.sh +++ b/src/main/run.sh @@ -160,6 +160,13 @@ function bashunit::main::exec_tests() { bashunit::reports::load_spooled + # To stdout, not to a file: GitHub reads workflow commands from the job log. + # After load_spooled so a --parallel run annotates the rows its workers + # spooled, which the parent would otherwise never have seen (#1004). + if bashunit::env::should_print_gha_annotations; then + bashunit::reports::print_gha_annotations all + fi + if [ -n "$BASHUNIT_LOG_JUNIT" ]; then bashunit::reports::generate_junit_xml "$BASHUNIT_LOG_JUNIT" fi diff --git a/src/main/test.sh b/src/main/test.sh index a396b628..b4b1abd8 100644 --- a/src/main/test.sh +++ b/src/main/test.sh @@ -240,6 +240,11 @@ function bashunit::main::cmd_test() { export -n BASHUNIT_LOG_GHA shift ;; + --gha-annotations) + BASHUNIT_GHA_ANNOTATIONS="$2" + export -n BASHUNIT_GHA_ANNOTATIONS + shift + ;; -r | --report-html) BASHUNIT_REPORT_HTML="$2" export -n BASHUNIT_REPORT_HTML diff --git a/src/main/validate.sh b/src/main/validate.sh index 609f4b2e..ace5a09c 100644 --- a/src/main/validate.sh +++ b/src/main/validate.sh @@ -153,6 +153,17 @@ function bashunit::main::validate_config_or_exit() { ;; esac + # Same shape as --output above: an unrecognised mode would otherwise fall back + # to auto and look like it was honoured. + case "${BASHUNIT_GHA_ANNOTATIONS:-auto}" in + auto | always | never) ;; + *) + printf "%sError: unsupported mode '%s' for --gha-annotations. Supported: auto, always, never.%s\n" \ + "${_BASHUNIT_COLOR_FAILED}" "${BASHUNIT_GHA_ANNOTATIONS}" "${_BASHUNIT_COLOR_DEFAULT}" >&2 + exit 1 + ;; + esac + # Same shape as --output above: an unrecognised mode would otherwise leave the # suite in definition order and look like it was honoured. case "${BASHUNIT_ORDER_BY:-defined}" in diff --git a/src/reports/collect.sh b/src/reports/collect.sh index fea413f1..2d542684 100644 --- a/src/reports/collect.sh +++ b/src/reports/collect.sh @@ -60,7 +60,8 @@ function bashunit::reports::is_enabled() { [ -n "${BASHUNIT_REPORT_HTML:-}" ] || [ -n "${BASHUNIT_LOG_GHA:-}" ] || [ -n "${BASHUNIT_REPORT_TAP:-}" ] || - [ -n "${BASHUNIT_REPORT_JSON:-}" ] + [ -n "${BASHUNIT_REPORT_JSON:-}" ] || + bashunit::env::should_print_gha_annotations } function bashunit::reports::add_test() { diff --git a/tests/acceptance/bashunit_gha_annotations_test.sh b/tests/acceptance/bashunit_gha_annotations_test.sh new file mode 100644 index 00000000..e0ce12e8 --- /dev/null +++ b/tests/acceptance/bashunit_gha_annotations_test.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +# GitHub parses workflow commands from the job log, so an annotation only lands +# on a pull request if it reaches stdout. Writing it to a file nobody cats +# produced exactly zero annotations. + +function set_up_before_script() { + TEST_ENV_FILE="tests/acceptance/fixtures/.env.default" + FIXTURE="./tests/acceptance/fixtures/test_bashunit_when_log_junit.sh" +} + +function test_annotations_reach_stdout_inside_github_actions() { + local output + output="$(GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \ + --env "$TEST_ENV_FILE" "$FIXTURE")" || true + + assert_contains "::error file=$FIXTURE" "$output" + assert_contains "title=Failure" "$output" +} + +function test_the_annotation_carries_the_failing_line() { + local output + output="$(GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \ + --env "$TEST_ENV_FILE" "$FIXTURE")" || true + + assert_matches "::error file=[^,]*,line=[0-9]+,title=" "$output" +} + +function test_nothing_extra_is_printed_outside_github_actions() { + local output + output="$(./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" "$FIXTURE")" || true + + assert_not_contains "::error" "$output" +} + +function test_never_suppresses_annotations_inside_github_actions() { + local output + output="$(GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \ + --env "$TEST_ENV_FILE" --gha-annotations never "$FIXTURE")" || true + + assert_not_contains "::error" "$output" +} + +function test_always_emits_annotations_outside_github_actions() { + local output + output="$(./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" \ + --gha-annotations always "$FIXTURE")" || true + + assert_contains "::error file=$FIXTURE" "$output" +} + +function test_a_multi_line_message_stays_one_annotation() { + local output + output="$(GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \ + --env "$TEST_ENV_FILE" "$FIXTURE")" || true + + # One failing test, so one ::error line, with the newlines percent-encoded. + assert_same "1" "$(printf '%s\n' "$output" | grep -c '^::error' | tr -d ' ')" + assert_contains "%0A" "$output" +} + +function test_log_gha_still_writes_the_file_without_duplicating_stdout() { + local log_file + log_file="$(bashunit::temp_file)" + + local output + output="$(GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \ + --env "$TEST_ENV_FILE" --log-gha "$log_file" "$FIXTURE")" || true + + assert_contains "::error" "$(cat "$log_file")" + assert_same "1" "$(printf '%s\n' "$output" | grep -c '^::error' | tr -d ' ')" +} + +function test_annotations_survive_parallel_aggregation() { + local output + output="$(GITHUB_ACTIONS=true ./bashunit --parallel --no-color \ + --env "$TEST_ENV_FILE" "$FIXTURE")" || true + + assert_contains "::error file=$FIXTURE" "$output" +} + +function test_log_gha_appears_in_the_help() { + assert_contains "--log-gha" "$(./bashunit test --help)" +} + +function test_an_unknown_mode_is_a_usage_error() { + local ec=0 + local output + output="$(./bashunit --no-parallel --env "$TEST_ENV_FILE" \ + --gha-annotations sometimes "$FIXTURE" 2>&1)" || ec=$? + + assert_general_error "" "" "$ec" + assert_contains "sometimes" "$output" + assert_contains "auto, always, never" "$output" +} From c92917a9c6e75a7ab96a1c52dba45317ceaf3cfb Mon Sep 17 00:00:00 2001 From: Chemaclass Date: Mon, 10 Aug 2026 16:26:44 +0200 Subject: [PATCH 2/2] fix(reports): only the outermost run annotates the job log GITHUB_ACTIONS is inherited by every child process, so auto-detection alone made every nested bashunit run annotate its parent's job log with its own fixtures' failures. bashunit's own acceptance suite spawns ~258 nested runs, which is how CI caught it; a user's script under test that calls bashunit would have polluted their log the same way. A run claims the log by exporting a marker and reading it first, so the outermost process wins and every descendant stays quiet. The marker is deliberately exported, unlike the run-mode flags: the nested run is exactly the consumer that has to see it. The reports unit tests ask whether a *file* report was configured. Annotations are a second reason to collect rows and switch themselves on inside GitHub Actions, so they are pinned off there to keep the answer independent of the ambient environment. --- src/config/env.sh | 19 +++- .../bashunit_gha_annotations_test.sh | 31 +++++-- tests/unit/config/env_test.sh | 86 +++++++++++++++++++ tests/unit/reports/reports_test.sh | 7 ++ 4 files changed, 135 insertions(+), 8 deletions(-) diff --git a/src/config/env.sh b/src/config/env.sh index 58642cf3..b0841b53 100644 --- a/src/config/env.sh +++ b/src/config/env.sh @@ -319,6 +319,21 @@ _BASHUNIT_DEFAULT_SNAPSHOT_REPORT_UNUSED="false" : "${BASHUNIT_FAIL_ON_FLAKY:=$_BASHUNIT_DEFAULT_FAIL_ON_FLAKY}" : "${BASHUNIT_REPEAT:=$_BASHUNIT_DEFAULT_REPEAT}" : "${BASHUNIT_GHA_ANNOTATIONS:=$_BASHUNIT_DEFAULT_GHA_ANNOTATIONS}" + +# GITHUB_ACTIONS is inherited by every child process, so a nested bashunit run +# (bashunit's own acceptance suite, or a user's script under test that calls +# bashunit) would annotate the parent's job log with its own fixtures' +# failures. Only the outermost run owns that log. +# +# Deliberately exported, unlike the run-mode flags: the nested run is exactly +# the consumer that has to see it. Reading it before claiming it is what makes +# the outermost process the one that wins. +if [ -n "${_BASHUNIT_GHA_ANNOTATIONS_CLAIMED:-}" ]; then + _BASHUNIT_IS_OUTERMOST_RUN=false +else + _BASHUNIT_IS_OUTERMOST_RUN=true +fi +export _BASHUNIT_GHA_ANNOTATIONS_CLAIMED=1 : "${BASHUNIT_SHARD_INDEX:=$_BASHUNIT_DEFAULT_SHARD_INDEX}" : "${BASHUNIT_SHARD_TOTAL:=$_BASHUNIT_DEFAULT_SHARD_TOTAL}" # No bare RERUN_FAILED alias, same reasoning as RETRY/SEED above. The default @@ -648,7 +663,9 @@ function bashunit::env::should_print_gha_annotations() { always) return 0 ;; esac - [ "${GITHUB_ACTIONS:-}" = "true" ] && ! bashunit::env::is_tap_output_enabled + [ "${_BASHUNIT_IS_OUTERMOST_RUN:-true}" = true ] && + [ "${GITHUB_ACTIONS:-}" = "true" ] && + ! bashunit::env::is_tap_output_enabled } function bashunit::env::is_fail_on_flaky_enabled() { diff --git a/tests/acceptance/bashunit_gha_annotations_test.sh b/tests/acceptance/bashunit_gha_annotations_test.sh index e0ce12e8..07bc265a 100644 --- a/tests/acceptance/bashunit_gha_annotations_test.sh +++ b/tests/acceptance/bashunit_gha_annotations_test.sh @@ -3,6 +3,11 @@ # GitHub parses workflow commands from the job log, so an annotation only lands # on a pull request if it reaches stdout. Writing it to a file nobody cats # produced exactly zero annotations. +# +# Clearing _BASHUNIT_GHA_ANNOTATIONS_CLAIMED below is how a nested run says +# "pretend I am the top-level one": this suite is itself a bashunit run and has +# already claimed the job log for its process tree, which is the very pollution +# the marker exists to prevent. function set_up_before_script() { TEST_ENV_FILE="tests/acceptance/fixtures/.env.default" @@ -11,7 +16,7 @@ function set_up_before_script() { function test_annotations_reach_stdout_inside_github_actions() { local output - output="$(GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \ + output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \ --env "$TEST_ENV_FILE" "$FIXTURE")" || true assert_contains "::error file=$FIXTURE" "$output" @@ -20,7 +25,7 @@ function test_annotations_reach_stdout_inside_github_actions() { function test_the_annotation_carries_the_failing_line() { local output - output="$(GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \ + output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \ --env "$TEST_ENV_FILE" "$FIXTURE")" || true assert_matches "::error file=[^,]*,line=[0-9]+,title=" "$output" @@ -28,14 +33,26 @@ function test_the_annotation_carries_the_failing_line() { function test_nothing_extra_is_printed_outside_github_actions() { local output - output="$(./bashunit --no-parallel --no-color --env "$TEST_ENV_FILE" "$FIXTURE")" || true + output="$(GITHUB_ACTIONS='' ./bashunit --no-parallel --no-color \ + --env "$TEST_ENV_FILE" "$FIXTURE")" || true assert_not_contains "::error" "$output" } -function test_never_suppresses_annotations_inside_github_actions() { +# The claim marker is left alone here, so this is a genuinely nested run. Under +# CI it inherits GITHUB_ACTIONS=true and must still stay quiet, or every nested +# run in a suite would annotate the parent's log with its own fixtures. +function test_a_nested_run_never_annotates_the_parents_log() { local output output="$(GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \ + --env "$TEST_ENV_FILE" "$FIXTURE")" || true + + assert_not_contains "::error" "$output" +} + +function test_never_suppresses_annotations_inside_github_actions() { + local output + output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \ --env "$TEST_ENV_FILE" --gha-annotations never "$FIXTURE")" || true assert_not_contains "::error" "$output" @@ -51,7 +68,7 @@ function test_always_emits_annotations_outside_github_actions() { function test_a_multi_line_message_stays_one_annotation() { local output - output="$(GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \ + output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \ --env "$TEST_ENV_FILE" "$FIXTURE")" || true # One failing test, so one ::error line, with the newlines percent-encoded. @@ -64,7 +81,7 @@ function test_log_gha_still_writes_the_file_without_duplicating_stdout() { log_file="$(bashunit::temp_file)" local output - output="$(GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \ + output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_ACTIONS=true ./bashunit --no-parallel --no-color \ --env "$TEST_ENV_FILE" --log-gha "$log_file" "$FIXTURE")" || true assert_contains "::error" "$(cat "$log_file")" @@ -73,7 +90,7 @@ function test_log_gha_still_writes_the_file_without_duplicating_stdout() { function test_annotations_survive_parallel_aggregation() { local output - output="$(GITHUB_ACTIONS=true ./bashunit --parallel --no-color \ + output="$(_BASHUNIT_GHA_ANNOTATIONS_CLAIMED='' GITHUB_ACTIONS=true ./bashunit --parallel --no-color \ --env "$TEST_ENV_FILE" "$FIXTURE")" || true assert_contains "::error file=$FIXTURE" "$output" diff --git a/tests/unit/config/env_test.sh b/tests/unit/config/env_test.sh index 20312ea8..c85a424f 100644 --- a/tests/unit/config/env_test.sh +++ b/tests/unit/config/env_test.sh @@ -501,3 +501,89 @@ function test_create_scratch_dirs_fails_loudly_when_a_directory_cannot_be_create assert_contains "cannot create the scratch directory" "$output" assert_contains "$base/blocker/run" "$output" } + +function _gha_annotations_decision() { + if bashunit::env::should_print_gha_annotations; then echo "print"; else echo "quiet"; fi +} + +function test_gha_annotations_auto_prints_for_an_outermost_run_in_github_actions() { + local decision + # shellcheck disable=SC2034 # read by the predicate through the environment + decision=$( + BASHUNIT_GHA_ANNOTATIONS=auto + GITHUB_ACTIONS=true + _BASHUNIT_IS_OUTERMOST_RUN=true + BASHUNIT_OUTPUT_FORMAT="" + _gha_annotations_decision + ) + + assert_same "print" "$decision" +} + +function test_gha_annotations_auto_stays_quiet_outside_github_actions() { + local decision + # shellcheck disable=SC2034 # read by the predicate through the environment + decision=$( + BASHUNIT_GHA_ANNOTATIONS=auto + GITHUB_ACTIONS="" + _BASHUNIT_IS_OUTERMOST_RUN=true + _gha_annotations_decision + ) + + assert_same "quiet" "$decision" +} + +function test_gha_annotations_auto_stays_quiet_in_a_nested_run() { + local decision + # shellcheck disable=SC2034 # read by the predicate through the environment + decision=$( + BASHUNIT_GHA_ANNOTATIONS=auto + GITHUB_ACTIONS=true + _BASHUNIT_IS_OUTERMOST_RUN=false + BASHUNIT_OUTPUT_FORMAT="" + _gha_annotations_decision + ) + + assert_same "quiet" "$decision" +} + +# TAP owns stdout as a machine format; an annotation line would corrupt it. +function test_gha_annotations_auto_stays_quiet_under_tap_output() { + local decision + # shellcheck disable=SC2034 # read by the predicate through the environment + decision=$( + BASHUNIT_GHA_ANNOTATIONS=auto + GITHUB_ACTIONS=true + _BASHUNIT_IS_OUTERMOST_RUN=true + BASHUNIT_OUTPUT_FORMAT=tap + _gha_annotations_decision + ) + + assert_same "quiet" "$decision" +} + +function test_gha_annotations_always_prints_even_when_nested_and_outside_ci() { + local decision + # shellcheck disable=SC2034 # read by the predicate through the environment + decision=$( + BASHUNIT_GHA_ANNOTATIONS=always + GITHUB_ACTIONS="" + _BASHUNIT_IS_OUTERMOST_RUN=false + _gha_annotations_decision + ) + + assert_same "print" "$decision" +} + +function test_gha_annotations_never_wins_over_github_actions() { + local decision + # shellcheck disable=SC2034 # read by the predicate through the environment + decision=$( + BASHUNIT_GHA_ANNOTATIONS=never + GITHUB_ACTIONS=true + _BASHUNIT_IS_OUTERMOST_RUN=true + _gha_annotations_decision + ) + + assert_same "quiet" "$decision" +} diff --git a/tests/unit/reports/reports_test.sh b/tests/unit/reports/reports_test.sh index 1d4a5f22..1f10d738 100644 --- a/tests/unit/reports/reports_test.sh +++ b/tests/unit/reports/reports_test.sh @@ -24,6 +24,12 @@ function set_up() { unset BASHUNIT_LOG_GHA unset BASHUNIT_REPORT_TAP + # These tests ask whether a *file* report was configured. Stdout annotations + # are a second reason to collect rows, and they switch themselves on inside + # GitHub Actions, so pin them off to keep the answer independent of the + # ambient CI environment. + export BASHUNIT_GHA_ANNOTATIONS=never + # Create temp file for output tests _TEMP_OUTPUT_FILE=$(mktemp) } @@ -36,6 +42,7 @@ function tear_down() { unset BASHUNIT_LOG_JUNIT unset BASHUNIT_REPORT_HTML unset BASHUNIT_LOG_GHA + unset BASHUNIT_GHA_ANNOTATIONS } function _reports_is_enabled_state() {