From 92d9b25ca7fd42a820ccf8f100b8420392c2bbbe Mon Sep 17 00:00:00 2001 From: "@scott-wueschinski-GTMify" Date: Wed, 29 Jul 2026 08:23:17 -0400 Subject: [PATCH] feat: tier A checks for settings.json, hooks, and skill frontmatter Completes tier A so gtmify-config can call this workflow. Three checks aimed at the class of breakage that does not announce itself: config that silently stops working on somebody else's machine. settings_gate.sh resolves every script path settings.json references and asserts it exists and is executable. The interesting part is the path mapping: commands are authored as "$HOME"/.claude/hooks/x.sh because that is where they live on a workstation, where ~/.claude/hooks symlinks into this repo. CI has no ~/.claude, so that prefix is rewritten to the repo root or every hook reads as missing. hook_smoke.sh runs a hook ONLY if the hook advertises a --dry-run flag, detected by grepping the script. Hooks are arbitrary code with side effects; one of them commits and pushes, another writes to Supabase. Executing them blindly in CI would be reckless. Against gtmify-config that means 1 ran and 10 skipped, and the report says exactly that rather than implying full coverage. frontmatter_gate.sh asserts skills declare a name and description. A skill with no description never triggers and never errors either: it looks installed and does nothing, which is the most expensive kind of broken. TWO OF MY OWN BUGS, both caught by running against the real repo rather than trusting the code, and both worth recording because they are easy to reintroduce: 1. The settings token regex reported all 9 hooks as missing. A character class that accepts a quote starts matching at the CLOSING quote of "$HOME", yielding $HOME"/.claude/... which resolves to nothing. Quotes are now stripped before paths are extracted. 2. The frontmatter check reported 55 failures, every one false. Git pathspec wildcards match across slashes by default, so `agents/*.md` reached `agents/instructions/*.md`, which are Paperclip instruction files and prose docs with no frontmatter by design. Now uses `:(glob)` magic so `*` stops at a slash, and the default scope is skills plus .claude/agents only. A gate with 55 false positives gets deleted the same day, so scope here is deliberately narrow and extendable with --glob rather than broad and wrong. Incidental finding while testing, not fixed here: gtmify-config/agents/ contains no Claude Code agent definitions at all, only Paperclip instructions and documentation, despite CLAUDE.md describing "6 global agents in ~/.claude/agents/". All three no-op cleanly in a repo that has none of these files. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/gate.yml | 20 ++++++ scripts/frontmatter_gate.sh | 113 +++++++++++++++++++++++++++++++++ scripts/hook_smoke.sh | 99 +++++++++++++++++++++++++++++ scripts/settings_gate.sh | 123 ++++++++++++++++++++++++++++++++++++ 4 files changed, 355 insertions(+) create mode 100755 scripts/frontmatter_gate.sh create mode 100755 scripts/hook_smoke.sh create mode 100755 scripts/settings_gate.sh diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index 173a61e..d267ffc 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -124,6 +124,26 @@ jobs: bash .ci-workflows/scripts/shell_gate.sh --base "${{ steps.base.outputs.ref }}" fi + - name: Settings gate + if: inputs.tier == 'a' + # Resolves every script path settings.json references and asserts it exists + # and is executable. Rewrites the "$HOME"/.claude/ prefix to the repo root, + # because CI has no ~/.claude and every hook would otherwise read as missing. + run: bash .ci-workflows/scripts/settings_gate.sh + + - name: Hook smoke + if: inputs.tier == 'a' + # Only runs hooks that advertise a --dry-run flag. A hook is arbitrary code + # with side effects (one commits and pushes), so anything without a declared + # no-op mode is reported as skipped rather than executed. + run: bash .ci-workflows/scripts/hook_smoke.sh + + - name: Frontmatter gate + if: inputs.tier == 'a' + # A skill with no description never triggers, and never errors either. It + # looks installed and does nothing. + run: bash .ci-workflows/scripts/frontmatter_gate.sh + - name: Set up Node if: inputs.tier == 'b' && inputs.build_cmd != '' uses: actions/setup-node@v4 diff --git a/scripts/frontmatter_gate.sh b/scripts/frontmatter_gate.sh new file mode 100755 index 0000000..108d8c5 --- /dev/null +++ b/scripts/frontmatter_gate.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# +# frontmatter_gate.sh: every skill and agent must declare a name and a description. +# +# WHY THIS EXISTS +# +# A skill's description is the only thing the model sees when deciding whether the +# skill applies. A skill with no description, or with frontmatter that failed to +# parse, does not error: it simply never triggers. It looks installed and does +# nothing, which is the most expensive kind of broken because nobody investigates a +# feature they believe is working. +# +# Frontmatter is parsed by hand rather than with PyYAML, which is not guaranteed to +# be present on a runner. Checking that two keys exist and are non-empty does not +# need a YAML parser, and adding a dependency to a gate makes the gate fragile. +# +# SCOPE, AND WHY IT IS NARROW +# +# Only files that MUST carry frontmatter are checked. An earlier version also swept +# `agents/*.md`, which produced 55 failures against gtmify-config, every one of them +# a false positive: git pathspec wildcards match across slashes by default, so that +# glob reached `agents/instructions/*.md`, which are Paperclip instruction files and +# prose documentation with no frontmatter by design. `:(glob)` magic is used below +# precisely so `*` stops at a slash. +# +# Pass --glob to add targets, for example a repo that keeps real Claude Code agent +# definitions in .claude/agents/. +# +# Usage: frontmatter_gate.sh [--glob ]... +set -euo pipefail + +REPO_ROOT="$(git rev-parse --show-toplevel)" +cd "$REPO_ROOT" + +globs=(':(glob)skills/*/SKILL.md' ':(glob).claude/agents/*.md') + +while [ $# -gt 0 ]; do + case "$1" in + --glob) shift; globs+=(":(glob)${1:-}") ;; + -h|--help) sed -n '2,28p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "frontmatter_gate: unknown argument: $1" >&2; exit 2 ;; + esac + shift +done + +targets=() +while IFS= read -r -d '' f; do targets+=("$f"); done < <( + { git ls-files -z -- "${globs[@]}" 2>/dev/null || true; } +) + +if [ "${#targets[@]}" -eq 0 ]; then + echo ">> frontmatter gate: no skills or agents in this repo, nothing to check." + exit 0 +fi + +echo ">> frontmatter gate: checking ${#targets[@]} file(s)" + +failures=0 + +for f in "${targets[@]}"; do + problem="$(F="$f" python3 <<'PY' +import os, sys + +path = os.environ["F"] +with open(path, encoding="utf-8", errors="replace") as fh: + lines = fh.read().split("\n") + +if not lines or lines[0].strip() != "---": + sys.stdout.write("no YAML frontmatter; file must open with ---") + sys.exit(0) + +body = [] +closed = False +for line in lines[1:]: + if line.strip() == "---": + closed = True + break + body.append(line) + +if not closed: + sys.stdout.write("frontmatter is never closed with a second ---") + sys.exit(0) + +found = {} +for line in body: + if line[:1].isspace() or not line.strip() or line.lstrip().startswith("#"): + continue + if ":" not in line: + continue + key, _, value = line.partition(":") + found[key.strip()] = value.strip().strip('"').strip("'") + +missing = [k for k in ("name", "description") if not found.get(k)] +if missing: + sys.stdout.write("missing or empty: " + ", ".join(missing)) +PY +)" + + if [ -n "$problem" ]; then + printf ' FAIL %s\n %s\n' "$f" "$problem" + failures=$((failures + 1)) + else + printf ' ok %s\n' "$f" + fi +done + +if [ "$failures" -ne 0 ]; then + echo + echo "!! frontmatter gate FAILED: ${failures} file(s) would never trigger." + exit 1 +fi + +echo ">> frontmatter gate passed." diff --git a/scripts/hook_smoke.sh b/scripts/hook_smoke.sh new file mode 100755 index 0000000..6ee6220 --- /dev/null +++ b/scripts/hook_smoke.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# +# hook_smoke.sh: run each hook's own safe self-check so a broken hook is caught +# here rather than on somebody's machine mid-session. +# +# THE SAFETY RULE THAT SHAPES THIS SCRIPT +# +# A hook is arbitrary code with side effects. auto_commit_on_exit.sh commits and +# pushes; log_session_transcript.sh writes to Supabase. Executing hooks blindly in +# CI would be reckless, so this script NEVER runs a hook unless the hook itself +# advertises a no-op flag: it greps for --dry-run handling and only then invokes it +# with --dry-run. A hook without that flag is reported as skipped, not run. +# +# That means coverage here is partial by design, and the report says so rather than +# implying every hook was exercised. A silent partial pass reading as full coverage +# is the exact failure this repo's other checks exist to prevent. +# +# Usage: hook_smoke.sh [--dir ] +set -euo pipefail + +HOOK_DIR="hooks" + +while [ $# -gt 0 ]; do + case "$1" in + --dir) shift; HOOK_DIR="${1:-}" ;; + -h|--help) sed -n '2,18p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "hook_smoke: unknown argument: $1" >&2; exit 2 ;; + esac + shift +done + +REPO_ROOT="$(git rev-parse --show-toplevel)" +DIR="$REPO_ROOT/$HOOK_DIR" + +if [ ! -d "$DIR" ]; then + echo ">> hook smoke: no $HOOK_DIR/ directory, nothing to check." + exit 0 +fi + +ran=0 +skipped=0 +failures=0 + +echo ">> hook smoke: scanning $HOOK_DIR/" + +while IFS= read -r -d '' h; do + name="$(basename "$h")" + + # Skip macOS artifacts, which is not hypothetical: every repo in this workspace + # has an `Icon` file inside .git/hooks, and gtmify-config tracks one in hooks/. + case "$name" in + Icon|Icon*|.DS_Store|._*) continue ;; + esac + + case "$name" in + *.sh) ;; + *) continue ;; + esac + + if ! grep -q -- '--dry-run' "$h"; then + printf ' skip %-32s no --dry-run flag; not safe to execute in CI\n' "$name" + skipped=$((skipped + 1)) + continue + fi + + rc=0 + # `cmd || rc=$?` rather than set +e / set -e around it: re-enabling errexit inside + # a loop turns it back on for everything after, and a later non-zero status then + # kills the script before it can print a total. + timeout 60 bash "$h" --dry-run >/tmp/hooksmoke.out 2>&1 || rc=$? + + if [ "$rc" -eq 0 ]; then + printf ' ok %-32s --dry-run exited 0\n' "$name" + ran=$((ran + 1)) + else + printf ' FAIL %-32s --dry-run exited %s\n' "$name" "$rc" + sed 's/^/ /' /tmp/hooksmoke.out | head -20 + failures=$((failures + 1)) + fi +done < <(find "$DIR" -maxdepth 1 -type f -print0) + +rm -f /tmp/hooksmoke.out + +echo +echo " ran ${ran}, skipped ${skipped} (no safe flag), failed ${failures}" + +if [ "$failures" -ne 0 ]; then + echo "!! hook smoke FAILED." + exit 1 +fi + +if [ "$ran" -eq 0 ]; then + echo ">> hook smoke: nothing was executable in dry-run mode. Coverage here is zero," + echo " which is a real gap rather than a pass. Add --dry-run support to a hook to" + echo " bring it under this check." + exit 0 +fi + +echo ">> hook smoke passed." diff --git a/scripts/settings_gate.sh b/scripts/settings_gate.sh new file mode 100755 index 0000000..0d61f8b --- /dev/null +++ b/scripts/settings_gate.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# +# settings_gate.sh: verify a Claude Code config repo's settings.json is valid and +# every script it points at actually exists and can run. +# +# WHY THIS EXISTS +# +# settings.json wires hooks that fire on every session, on every machine. A typo in +# a path, or a hook that loses its executable bit, does not fail loudly at commit +# time; it fails later, quietly, on somebody's machine, usually as "why did the +# session not log anything". Checking it costs a second. +# +# THE PATH MAPPING THAT MAKES THIS WORK IN CI +# +# Hook commands are written as "$HOME"/.claude/hooks/x.sh because that is where they +# live on a workstation, where ~/.claude/hooks is a symlink into this repo. CI has no +# ~/.claude at all, so every $HOME/.claude/ prefix is rewritten to the repo root +# before the file is looked up. Without that rewrite this check would report every +# hook as missing. +# +# Usage: settings_gate.sh [--settings ] +set -euo pipefail + +SETTINGS="settings.json" + +while [ $# -gt 0 ]; do + case "$1" in + --settings) shift; SETTINGS="${1:-}" ;; + -h|--help) sed -n '2,20p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "settings_gate: unknown argument: $1" >&2; exit 2 ;; + esac + shift +done + +REPO_ROOT="$(git rev-parse --show-toplevel)" + +if [ ! -f "$REPO_ROOT/$SETTINGS" ]; then + echo ">> settings gate: no $SETTINGS in this repo, nothing to check." + exit 0 +fi + +echo ">> settings gate: checking $SETTINGS" + +REPO_ROOT="$REPO_ROOT" SETTINGS="$SETTINGS" python3 <<'PY' +import json, os, re, stat, sys + +root = os.environ["REPO_ROOT"] +rel = os.environ["SETTINGS"] +path = os.path.join(root, rel) + +try: + with open(path) as f: + cfg = json.load(f) +except json.JSONDecodeError as e: + print(f"!! {rel} is not valid JSON: {e}") + sys.exit(1) + +print(f" ok {rel} parses as JSON") + +# Hook commands are authored for a workstation, where ~/.claude/hooks symlinks into +# this repo. Rewrite that prefix to the repo root so the files resolve in CI too. +def resolve(cmd_token): + t = cmd_token.strip().strip('"').strip("'") + for prefix in ('"$HOME"/.claude/', '$HOME/.claude/', '~/.claude/'): + if t.startswith(prefix): + return os.path.join(root, t[len(prefix):]), t + if t.startswith("/"): + return t, t + return os.path.join(root, t), t + +failures = 0 +checked = 0 + +def check_command(where, command): + global failures, checked + # Strip quotes BEFORE extracting paths. Commands are authored as + # "$HOME"/.claude/hooks/x.sh, and a character class that accepts a quote will + # happily start matching at the closing one, yielding $HOME"/.claude/... which + # then resolves to nothing. Removing quotes first makes the token unambiguous. + # (Found by running this against the real settings.json, where an earlier version + # of this regex reported all 9 hooks as missing.) + cleaned = command.replace('"', "").replace("'", "") + tokens = re.findall(r'[\w$./~\-]+\.(?:sh|py|js|mjs)', cleaned) + if not tokens: + return + for tok in tokens: + resolved, original = resolve(tok) + checked += 1 + if not os.path.exists(resolved): + print(f" FAIL {where}: {original}") + print(f" does not exist (looked in {os.path.relpath(resolved, root)})") + failures += 1 + continue + mode = os.stat(resolved).st_mode + if not (mode & stat.S_IXUSR): + print(f" FAIL {where}: {original}") + print(f" exists but is not executable; fix with: chmod +x {os.path.relpath(resolved, root)}") + failures += 1 + continue + print(f" ok {where}: {original}") + +for event, matchers in (cfg.get("hooks") or {}).items(): + if not isinstance(matchers, list): + continue + for m in matchers: + for h in (m.get("hooks") or []): + cmd = h.get("command") + if isinstance(cmd, str): + check_command(f"hooks.{event}", cmd) + +status_line = cfg.get("statusLine") +if isinstance(status_line, dict) and isinstance(status_line.get("command"), str): + check_command("statusLine", status_line["command"]) + +if checked == 0: + print(" NOTE: no script paths found in settings.json; nothing to resolve.") + +if failures: + print(f"\n!! settings gate FAILED: {failures} of {checked} referenced script(s) unusable.") + sys.exit(1) + +print(f"\n>> settings gate passed. {checked} referenced script(s) exist and are executable.") +PY