Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .machine_readable/Debtfile.a2ml
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ forgotten.
### gate-scripts-without-tests
- description: Scripts under scripts/ with no matching scripts/tests/<name>-test.sh — a gate with no test has never been shown able to fail Re-baselined 2026-09-22 (round 2, issue #953): the committed count had fossilized at 31 while the tree measured 38+; set to measured 40 with a declared ceiling raise. Falls automatically as tests land.
- probe: n=0; for f in $(git ls-files 'scripts/*.sh'); do b=$(basename "$f" .sh); case "$b" in *-test) continue;; esac; if [ ! -f "scripts/tests/${b}-test.sh" ] && [ ! -f "scripts/tests/${b#check-}-test.sh" ] && [ ! -f "scripts/tests/${b#run-}-test.sh" ]; then n=$((n+1)); fi; done; echo "$n"
- count: 40
- ceiling: 40
- count: 35
- ceiling: 35
- severity: high
- policy: remediable
- tri: eliminate
Expand Down
71 changes: 71 additions & 0 deletions scripts/tests/descriptile-policy-test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: MPL-2.0
#
# Proves check-descriptile-policy.sh CAN FAIL, and fails for the right reasons.
#
# The checker scans `scripts/*.sh`, and as a git pathspec that glob also matches
# scripts/tests/ — so it scans THIS file. The retired descriptile paths below are
# therefore ASSEMBLED FROM FRAGMENTS and never appear literally, or the real gate
# would turn red on its own test.
set -euo pipefail
SCRIPT="$(cd "$(dirname "$0")/.." && pwd)/check-descriptile-policy.sh"
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT

MR=".machine_readable"
STATE="$MR/STATE.a2ml" # retired flat path
META6="$MR/6a2/META.a2ml" # retired 6a2/ path
ANCHOR="$MR/ANCHOR.a2ml" # retired flat path
LIVE="$MR/descriptiles/STATE.a2ml" # the live location

pass=0; fail=0
expect() { # expect <wanted-exit> <label> (runs the checker in the repo $WORK/t)
local want="$1" label="$2" got=0
git -C "$WORK/t" add -A >/dev/null
(cd "$WORK/t" && bash "$SCRIPT") >"$WORK/out" 2>&1 || got=$?
if [ "$got" = "$want" ]; then pass=$((pass+1)); echo " ok $label"
else fail=$((fail+1)); echo " FAIL $label (wanted exit $want, got $got)"; sed 's/^/ /' "$WORK/out"; fi
}
fresh() {
rm -rf "$WORK/t"; mkdir -p "$WORK/t/.github/workflows" "$WORK/t/scripts"
git -C "$WORK/t" init -q
}
wf() { printf '%s\n' "on: push" "jobs:" " a:" " runs-on: ubuntu-latest" " steps:" " - run: $1" > "$WORK/t/.github/workflows/ci.yml"; }

fresh; wf "echo hello"
expect 0 "a workflow that tests no descriptile path is clean"

fresh; wf "[ -f $STATE ]"
expect 1 "a workflow requiring the retired flat path with -f is rejected"
grep -q '::error file=.github/workflows/ci.yml::' "$WORK/out" \
&& { pass=$((pass+1)); echo " ok the rejection is a ::error annotation naming the file"; } \
|| { fail=$((fail+1)); echo " FAIL the rejection does not annotate the file"; }

fresh; wf "test -e $META6"
expect 1 "a workflow requiring the retired 6a2/ path with -e is rejected"

fresh; printf '%s\n' '#!/usr/bin/env bash' "check_file \"$ANCHOR\"" > "$WORK/t/scripts/gate.sh"
expect 1 "a script calling check_file on a quoted retired path is rejected"

fresh; printf '%s\n' '#!/usr/bin/env bash' "echo \"\$( [ -f $STATE ] && echo yes )\"" > "$WORK/t/scripts/gate.sh"
expect 1 "a file test hidden inside a double-quoted command substitution is rejected"

fresh; printf '%s\n' "check:" " test -f $STATE" > "$WORK/t/Justfile"
expect 1 "a Justfile recipe requiring a retired path is rejected"

fresh; wf "[ -f $LIVE ]"
expect 0 "a file test on the live descriptiles/ location is accepted"

fresh; printf '%s\n' '#!/usr/bin/env bash' "# old: [ -f $STATE ]" > "$WORK/t/scripts/gate.sh"
expect 0 "a commented-out example is not enforcement"

fresh; printf '%s\n' '#!/usr/bin/env bash' "echo \"we used to run -f on $STATE\"" > "$WORK/t/scripts/gate.sh"
expect 0 "quoted prose that mentions the path is not enforcement"

fresh; printf '%s\n' '#!/usr/bin/env bash' "[ -f $STATE ]" > "$WORK/t/scripts/gate.sh"
git -C "$WORK/t" add -A >/dev/null; git -C "$WORK/t" rm -q --cached scripts/gate.sh
(cd "$WORK/t" && bash "$SCRIPT") >"$WORK/out" 2>&1 && got=0 || got=$?
if [ "$got" = 0 ]; then pass=$((pass+1)); echo " ok an untracked file is outside the gate's scope"
else fail=$((fail+1)); echo " FAIL an untracked file was scanned (got $got)"; fi

echo "PASS=$pass FAIL=$fail"
[ "$fail" -eq 0 ]
64 changes: 64 additions & 0 deletions scripts/tests/language-guide-test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: MPL-2.0
#
# Proves check-language-guide.sh CAN FAIL, and fails for the right reasons.
#
# A per-language testing guide that silently omits a required section, the
# R1..R9 requirement mapping, or its SPDX header is a false-completeness hole.
# Each omission must be rejected on its own.
set -euo pipefail
SCRIPT="$(cd "$(dirname "$0")/.." && pwd)/check-language-guide.sh"
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT
cd "$WORK"

SECTIONS=("Requirement mapping" "Tools" "Recommended CI pipeline" "Best practices" "Known gaps" "Resources")

# guide <marker> [section-to-omit] [omit-r9] [omit-spdx] → a guide on stdout
guide() {
local mark="$1" omit="${2:-}" no_r9="${3:-}" no_spdx="${4:-}" s
[ -n "$no_spdx" ] || echo "// SPDX-License-Identifier: CC-BY-SA-4.0"
echo "= Example testing guide"
echo
for s in "${SECTIONS[@]}"; do
[ "$s" = "$omit" ] && continue
echo "$mark $s"
[ "$s" = "Requirement mapping" ] && { echo "R1 unit tests"; [ -n "$no_r9" ] || echo "R9 mutation testing"; }
echo "body"
done
}

pass=0; fail=0
expect() { # expect <wanted-exit> <label> <file...>
local want="$1" label="$2" got=0; shift 2
bash "$SCRIPT" "$@" >out 2>&1 || got=$?
if [ "$got" = "$want" ]; then pass=$((pass+1)); echo " ok $label"
else fail=$((fail+1)); echo " FAIL $label (wanted exit $want, got $got)"; sed 's/^/ /' out; fi
}

guide "==" > ok.adoc
expect 0 "a complete AsciiDoc guide is valid" ok.adoc

guide "##" > ok.md
expect 0 "a complete Markdown-heading guide is valid" ok.md

for s in "${SECTIONS[@]}"; do
f="no-${s// /-}.adoc"
guide "==" "$s" > "$f"
expect 1 "a guide missing '$s' is rejected" "$f"
done

guide "==" "" no-r9 > no-r9.adoc
expect 1 "a guide whose mapping never reaches R9 is rejected" no-r9.adoc

guide "==" "" "" no-spdx > no-spdx.adoc
expect 1 "a guide with no SPDX header is rejected" no-spdx.adoc

guide "==" | sed 's/^== Known gaps$/== Known gaps (none)/' > renamed.adoc
expect 1 "a renamed section heading does not satisfy the requirement" renamed.adoc

expect 1 "one bad guide among good ones fails the run" ok.adoc no-r9.adoc ok.md

expect 1 "a guide that does not exist is rejected" missing.adoc

echo "PASS=$pass FAIL=$fail"
[ "$fail" -eq 0 ]
91 changes: 91 additions & 0 deletions scripts/tests/mustfile-structure-test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: MPL-2.0
#
# Proves check-mustfile-structure.sh CAN FAIL, and fails for the right reasons.
#
# Its whole job is to reject a HOLLOW CHECK — a '### <id>' block that looks like
# enforcement but carries neither a `- run:` nor a `- verification:`, or that
# has no severity. A Mustfile with no checks at all is also a failure, never a
# vacuous pass.
set -euo pipefail
SCRIPT="$(cd "$(dirname "$0")/.." && pwd)/check-mustfile-structure.sh"
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT
cd "$WORK"

pass=0; fail=0
expect() { # expect <wanted-exit> <label> (Mustfile content on stdin)
local want="$1" label="$2" got=0
cat > Mustfile.a2ml
bash "$SCRIPT" Mustfile.a2ml >out 2>&1 || got=$?
if [ "$got" = "$want" ]; then pass=$((pass+1)); echo " ok $label"
else fail=$((fail+1)); echo " FAIL $label (wanted exit $want, got $got)"; sed 's/^/ /' out; fi
}

expect 0 "a check with severity + run is valid" <<'EOF'
### alpha
- severity: critical
- run: test -f README.adoc
EOF

expect 0 "a check with severity + verification is valid" <<'EOF'
### alpha
- severity: high
- verification: governance — reviewed by the owner each release
EOF

expect 0 "several valid checks are valid" <<'EOF'
### alpha
- severity: critical
- run: true
### beta
- severity: low
- verification: manual
EOF

expect 1 "a check with neither run nor verification is rejected (hollow)" <<'EOF'
### alpha
- severity: critical
- description: looks like enforcement, discharges nothing
EOF
grep -q 'hollow check' out \
&& { pass=$((pass+1)); echo " ok the hollow rejection says 'hollow check'"; } \
|| { fail=$((fail+1)); echo " FAIL the hollow rejection does not say 'hollow check'"; }

expect 1 "a check with no severity is rejected" <<'EOF'
### alpha
- run: true
EOF

expect 1 "one hollow check among valid ones is rejected" <<'EOF'
### alpha
- severity: critical
- run: true
### beta
- severity: high
### gamma
- severity: low
- verification: manual
EOF

expect 1 "the LAST check is validated too (flush at end of file)" <<'EOF'
### alpha
- severity: critical
- run: true
### omega
- description: trailing hollow check
EOF

expect 1 "a Mustfile that declares no checks is rejected" <<'EOF'
# Mustfile
- severity: critical
- run: true
EOF

expect 1 "an empty Mustfile is rejected" </dev/null

got=0; bash "$SCRIPT" "$WORK/does-not-exist.a2ml" >/dev/null 2>&1 || got=$?
if [ "$got" = 2 ]; then pass=$((pass+1)); echo " ok a missing Mustfile exits 2"
else fail=$((fail+1)); echo " FAIL a missing Mustfile (wanted exit 2, got $got)"; fi

echo "PASS=$pass FAIL=$fail"
[ "$fail" -eq 0 ]
60 changes: 60 additions & 0 deletions scripts/tests/shell-test-suite-test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: MPL-2.0
#
# Proves run-shell-test-suite.sh CAN FAIL, and fails for the right reasons.
#
# This runner is the gate that makes every other test count. If it passed on
# zero discovered tests, or swallowed a failing test, every suite behind it
# would be green by construction. Each fixture tree below is built in a scratch
# directory, so this test never re-enters the real suite.
set -euo pipefail
SCRIPT="$(cd "$(dirname "$0")/.." && pwd)/run-shell-test-suite.sh"
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT

pass=0; fail=0
expect() { # expect <wanted-exit> <label> (runs the suite with cwd = $WORK/t)
local want="$1" label="$2" got=0
(cd "$WORK/t" && bash "$SCRIPT") >"$WORK/out" 2>&1 || got=$?
if [ "$got" = "$want" ]; then pass=$((pass+1)); echo " ok $label"
else fail=$((fail+1)); echo " FAIL $label (wanted exit $want, got $got)"; sed 's/^/ /' "$WORK/out"; fi
}
said() { # said <label> <fixed-string> (asserts on the last run's output)
if grep -qF -- "$2" "$WORK/out"; then pass=$((pass+1)); echo " ok $1"
else fail=$((fail+1)); echo " FAIL $1 (output lacks: $2)"; fi
}
fresh() { rm -rf "$WORK/t"; mkdir -p "$WORK/t"; }
t_pass() { mkdir -p "$(dirname "$WORK/t/$1")"; printf '#!/usr/bin/env bash\nexit 0\n' > "$WORK/t/$1"; }
t_fail() { mkdir -p "$(dirname "$WORK/t/$1")"; printf '#!/usr/bin/env bash\nexit 3\n' > "$WORK/t/$1"; }

fresh
expect 1 "no test directories at all fails closed"
said "the zero-discovery failure says discovery is broken" "discovery is broken"

fresh; mkdir -p "$WORK/t/tests" "$WORK/t/scripts/tests"
expect 1 "empty test directories fail closed"

fresh; t_pass tests/a.sh
expect 0 "one passing test under tests/ passes"

fresh; t_pass scripts/tests/a.sh
expect 0 "one passing test under scripts/tests/ passes"

fresh; t_pass tests/a.sh; t_pass scripts/tests/b.sh
expect 0 "passing tests in both directories pass"
said "both directories are discovered" "Discovered 2 test file(s)."

fresh; t_pass tests/a.sh; t_fail scripts/tests/b.sh; t_pass scripts/tests/c.sh
expect 1 "one failing test among passing ones fails the run"
said "the failing test is named with its exit status" "scripts/tests/b.sh failed (exit 3)"
said "the failure count is reported" "1 of 3 test file(s) failed."

fresh; t_fail tests/a.sh; t_fail scripts/tests/b.sh
expect 1 "every test failing fails the run"
said "both failures are counted" "2 of 2 test file(s) failed."

fresh; t_pass tests/a.sh; printf 'exit 0\n' > "$WORK/t/tests/notes.txt"
expect 0 "non-.sh files are not run as tests"
said "only the .sh file is discovered" "Discovered 1 test file(s)."

echo "PASS=$pass FAIL=$fail"
[ "$fail" -eq 0 ]
62 changes: 62 additions & 0 deletions scripts/tests/uuid-v7-test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: MPL-2.0
#
# Proves check-uuid-v7.sh CAN FAIL, and fails for the right reasons.
#
# The checker runs over this very repository (uuid-v7.yml), so every UUID in
# this file is ASSEMBLED AT RUNTIME from fragments. A literal non-v7 UUID here
# would turn the real gate red on its own test.
set -euo pipefail
SCRIPT="$(cd "$(dirname "$0")/.." && pwd)/check-uuid-v7.sh"
WORK="$(mktemp -d)"; trap 'rm -rf "$WORK"' EXIT

uuid() { printf '%s-%s-%s-%s-%s' "$@"; }
V7_8="$(uuid 01890a5d ac96 774b 8cce b302099a8057)" # version 7, variant 8
V7_B="$(uuid 01890A5D AC96 774B BCCE B302099A8057)" # version 7, variant b, upper case
V4="$(uuid 3f2504e0 4f89 41d3 9a0c 0305e82c3301)" # version 4
V7_C="$(uuid 01890a5d ac96 774b ccce b302099a8057)" # version 7, NON-RFC variant c

pass=0; fail=0
expect() { # expect <wanted-exit> <label> (runs the checker over $WORK/t)
local want="$1" label="$2" got=0
bash "$SCRIPT" "$WORK/t" >"$WORK/out" 2>&1 || got=$?
if [ "$got" = "$want" ]; then pass=$((pass+1)); echo " ok $label"
else fail=$((fail+1)); echo " FAIL $label (wanted exit $want, got $got)"; sed 's/^/ /' "$WORK/out"; fi
}
fresh() { rm -rf "$WORK/t"; mkdir -p "$WORK/t"; }

fresh; printf 'id = "%s"\n' "$V7_8" > "$WORK/t/a.toml"
expect 0 "a v7 UUID (variant 8) is accepted"

fresh; printf 'id: %s\n' "$V7_B" > "$WORK/t/a.yml"
expect 0 "an upper-case v7 UUID (variant b) is accepted"

fresh; printf 'no identifiers here\n' > "$WORK/t/a.txt"
expect 0 "a file with no UUID is accepted"

fresh
expect 0 "an empty tree is accepted"

fresh; printf 'id = "%s"\n' "$V4" > "$WORK/t/a.toml"
expect 1 "a v4 UUID is rejected"
grep -q "a.toml: non-v7 UUID literal ($V4)" "$WORK/out" \
&& { pass=$((pass+1)); echo " ok the rejection names the file and the UUID"; } \
|| { fail=$((fail+1)); echo " FAIL the rejection does not name the file and the UUID"; }

fresh; printf 'id = "%s"\n' "$V7_C" > "$WORK/t/a.toml"
expect 1 "version 7 with a non-RFC variant nibble is rejected"

fresh; printf 'ok = "%s"\nbad = "%s"\n' "$V7_8" "$V4" > "$WORK/t/a.toml"
expect 1 "a v4 on a later line than a v7 is still rejected"

fresh; mkdir -p "$WORK/t/sub/deeper"; printf '%s\n' "$V4" > "$WORK/t/sub/deeper/x.json"
expect 1 "a v4 in a nested directory is rejected"

fresh; mkdir -p "$WORK/t/.git"; printf '%s\n' "$V4" > "$WORK/t/.git/packed-refs"
expect 0 "a v4 inside .git/ is ignored"

fresh; printf '\000%s\n' "$V4" > "$WORK/t/blob.bin"
expect 0 "a v4 inside a binary file is ignored"

echo "PASS=$pass FAIL=$fail"
[ "$fail" -eq 0 ]
Loading