From 3777d9c83544cd6e42cde7fd8f378f9a86914346 Mon Sep 17 00:00:00 2001 From: Remigiusz Antczak Date: Wed, 28 Oct 2026 18:57:46 +0100 Subject: [PATCH] Peer lessons batch: reply_cap yields to user-level cap hook, fitness_tick, flake_holder, finding_guard, baseline_guard, freeze_gate, flow_alarm, prevention_lint, quarantine ticket/cap, merged-not-wired allowlist Skipped as already covered: top-3 build order when everything is P0 (queue_guard.py ranked top-N), dedupe hook for issue creation (file_finding.py, now hook-enforced). Review fixes: reply_cap yields only to an existing executable hook, fitness_tick alarms on an empty dir, flow_alarm hook gh timeout 10s with notice, prevention_lint matches whole header names, flake_holder lists up to 1000 PRs. Tests: new per-feature files run via scripts/ci-tests.d/peer-lessons-batch.sh (reply_cap 19/19, fitness_tick 6/6, flow_alarm 10/10, prevention_lint 7/7, flake_holder 4/4, hermetic lint 5/5) Co-Authored-By: Claude Opus 5.5 --- .claude/skills/INDEX.md | 8 +- .../skills/agentic-delivery/INDEX-scripts.md | 7 + .claude/skills/agentic-delivery/INDEX.md | 10 +- .../agentic-delivery/references/full-map.md | 2 +- .../references/red-gate-triage.md | 4 + .../references/template-postmortem.md | 11 +- .../references/unattended-operating-mode.md | 2 +- .../scripts/baseline_guard.py | 66 +++++++++ .../agentic-delivery/scripts/finding_guard.py | 79 +++++++++++ .../agentic-delivery/scripts/fitness_tick.py | 100 +++++++++++++ .../agentic-delivery/scripts/flake_holder.py | 50 +++++++ .../agentic-delivery/scripts/flake_ledger.py | 22 ++- .../agentic-delivery/scripts/flow_alarm.py | 134 ++++++++++++++++++ .../agentic-delivery/scripts/freeze_gate.py | 53 +++++++ .../scripts/prevention_lint.py | 67 +++++++++ .../agentic-delivery/scripts/reply_cap.py | 46 +++++- .../agentic-delivery/scripts/wired_check.py | 10 +- .../templates/operating-layer.settings.json | 20 +++ .../templates/pre-push-verify.sh | 18 +++ changelog.d/wave-ai-peer-lessons-batch.md | 12 ++ scripts/ci-tests.d/peer-lessons-batch.sh | 11 ++ scripts/perun_doctor.py | 2 +- scripts/test_baseline_guard.py | 53 +++++++ scripts/test_finding_guard.py | 51 +++++++ scripts/test_fitness_tick.py | 75 ++++++++++ scripts/test_flake_holder.py | 58 ++++++++ scripts/test_flow_alarm.py | 103 ++++++++++++++ scripts/test_freeze_gate.py | 63 ++++++++ scripts/test_orphan_allowlist_wired.py | 50 +++++++ scripts/test_prevention_lint.py | 53 +++++++ scripts/test_quarantine_ticket_cap.py | 67 +++++++++ 31 files changed, 1284 insertions(+), 23 deletions(-) create mode 100644 .claude/skills/agentic-delivery/scripts/baseline_guard.py create mode 100644 .claude/skills/agentic-delivery/scripts/finding_guard.py create mode 100644 .claude/skills/agentic-delivery/scripts/fitness_tick.py create mode 100644 .claude/skills/agentic-delivery/scripts/flake_holder.py create mode 100644 .claude/skills/agentic-delivery/scripts/flow_alarm.py create mode 100644 .claude/skills/agentic-delivery/scripts/freeze_gate.py create mode 100644 .claude/skills/agentic-delivery/scripts/prevention_lint.py create mode 100644 changelog.d/wave-ai-peer-lessons-batch.md create mode 100644 scripts/ci-tests.d/peer-lessons-batch.sh create mode 100644 scripts/test_baseline_guard.py create mode 100644 scripts/test_finding_guard.py create mode 100644 scripts/test_fitness_tick.py create mode 100644 scripts/test_flake_holder.py create mode 100644 scripts/test_flow_alarm.py create mode 100644 scripts/test_freeze_gate.py create mode 100644 scripts/test_orphan_allowlist_wired.py create mode 100644 scripts/test_prevention_lint.py create mode 100644 scripts/test_quarantine_ticket_cap.py diff --git a/.claude/skills/INDEX.md b/.claude/skills/INDEX.md index e239241d9..e2a9a5f58 100644 --- a/.claude/skills/INDEX.md +++ b/.claude/skills/INDEX.md @@ -43,6 +43,7 @@ Run `python3 /scripts/` (`bash` for `.sh`). `name: purpose — argu - weekly_receipt.py: Weekly value receipt: a one-screen plain-English card from… — [--days 7] [--repo DIR] [--session… ### agentic-delivery +- baseline_guard.py: store a test baseline only when it is not poisoned by load. — --fails FILE --rerun 'CMD {id}' --out… - bg_admit.py: admission for background and shadow jobs, plus a lint for… — admit SCRIPT... - board_post.py: compose one typed, capped, privacy-linted coordination post. — --repo O/N --issue N --agent ID --type… - board_state.py: render a coordination issue's current state into its body. — --repo OWNER/NAME --issue N [--now… @@ -55,8 +56,12 @@ Run `python3 /scripts/` (`bash` for `.sh`). `name: purpose — argu - feedback_ledger.py: one durable requirement ledger across every feedback source. — [--ledger PATH] [options] - feeder_order.py: order the work queue so a fix for a recurring failure class… — --ledger FILE [--min N] [--max-run M]… - file_finding.py: duplicate check to run BEFORE anything files an issue. — [--repo OWNER/NAME] [--force REASON]… +- fitness_tick.py: run every fix's fitness check once per operating-loop tick — [--dir D] [--state F] +- flake_holder.py: warn when a flaky-test fix touches a test file another open PR… — BASE HEAD +- flow_alarm.py: three flow checks that read the delivery queue, not the code. — symptom [--hours H] [--red-since TS] - focus_gate.py: no work outside the owner's open priority until that priority… — status [common options] [--evidence… - focus_kit.py: keep agents on task across long sessions and compactions… — handoff set|add SECTION TEXT |… +- freeze_gate.py: while a freeze is on, allow only deletions and red-trunk fixes. — BASE HEAD - handback.py: lane handback, gates to merge — --sha SHA --base REF --branch BRANCH… - host_probe.py: the fan-out spawn/hold predicate, computed once, not… — --lane-type {cpu,io,light,heavy}… - job_lease.py: per-repo lease for long jobs (ship and QA scripts) + a unique… — run [--name NAME] [--timeout SECS] --… @@ -74,6 +79,7 @@ Run `python3 /scripts/` (`bash` for `.sh`). `name: purpose — argu - pr_state.py: one command, one compact block: a PR's head, base, checks… — [--repo OWNER/NAME] [--state PATH]… - preflight.py: run cheap gates first, report every failure together. — CONFIG.json [--all] [--cache FILE] - premise_check.py: before handing PRs or issues to a lane or peer, say which are… — N [N ...] [--repo OWNER/NAME]… +- prevention_lint.py: every prevention/action item needs an owner, a tracking id and… — FILE.md - priority_gate.py: no presentation-only PR while an aged P0/mechanism issue has… — --labels-json FILE [options] - queue_guard.py: single priority queue: print the next issue to pull, flag P0… — --issues issues.json --priorities… - reap_own.sh: kill your own dev servers older than MAX_AGE_S whose cwd is… @@ -100,7 +106,7 @@ Run `python3 /scripts/` (`bash` for `.sh`). `name: purpose — argu - vacuous_test_gate.py: a changed test file must really run and assert. — [REPORT] (--changed PATH... | --base… - vendor_drift.py: list vendored skill files in TARGET that differ from the… — TARGET [--src DIR] - wired_check.py: warn when a NEW script has no caller. — BASE HEAD [--root DIR] -- No documented CLI: _clean_union.sh, _dirty_guard.sh, _lock.sh, admission_check.py, board_common.py, bypass_guard.py, ceo_altitude_guard.py, comms_guard.py, context_nudge.py, cost_statusline.py, decision_gate.py, destructive_guard.py, destructive_guard_core.py, flake_ledger.py, gh_rest.py, handback_cap.py, handoff_inject.py, heavy_gate.py, host_accept.py, host_installer.py, host_safety.py, landing_watchdog.py, lane_cap.py, lane_resume_nudge.py, lkg.py, outcome_claim_guard.py, peer_ack.py, perun_auto_update.py, pipe_mask_guard.py, pipeline_profile.py, plan_gate.py, pregate_parity.py, red_main.py, repeat_poll_guard.py, reply_cap.py, restore_ledger.py, scope_creep_check.py, session_brief.py, stall_guard.py, subagent_brief_lint.py, surface_check.py, terse_reply_check.py +- No documented CLI: _clean_union.sh, _dirty_guard.sh, _lock.sh, admission_check.py, board_common.py, bypass_guard.py, ceo_altitude_guard.py, comms_guard.py, context_nudge.py, cost_statusline.py, decision_gate.py, destructive_guard.py, destructive_guard_core.py, finding_guard.py, flake_ledger.py, gh_rest.py, handback_cap.py, handoff_inject.py, heavy_gate.py, host_accept.py, host_installer.py, host_safety.py, landing_watchdog.py, lane_cap.py, lane_resume_nudge.py, lkg.py, outcome_claim_guard.py, peer_ack.py, perun_auto_update.py, pipe_mask_guard.py, pipeline_profile.py, plan_gate.py, pregate_parity.py, red_main.py, repeat_poll_guard.py, reply_cap.py, restore_ledger.py, scope_creep_check.py, session_brief.py, stall_guard.py, subagent_brief_lint.py, surface_check.py, terse_reply_check.py ### business-ops - bizops_check.py: Deterministic arithmetic checks business-ops already states… — [input.json] (stdin when no path) diff --git a/.claude/skills/agentic-delivery/INDEX-scripts.md b/.claude/skills/agentic-delivery/INDEX-scripts.md index 15f332797..3cdcd9692 100644 --- a/.claude/skills/agentic-delivery/INDEX-scripts.md +++ b/.claude/skills/agentic-delivery/INDEX-scripts.md @@ -8,6 +8,7 @@ Generated by `scripts/skill_index.py` (do not hand-edit). The script half of `IN | `scripts/_dirty_guard.sh` | Sourced by scripts that run git checkout\|switch\|reset on a caller-given tree. | — | | `scripts/_lock.sh` | Sourced by train scripts. | — | | `scripts/admission_check.py` | flag a per-PR admission config that omits a test class the batch (merge train) runs, and a stale sp… | — | +| `scripts/baseline_guard.py` | store a test baseline only when it is not poisoned by load. | baseline_guard.py --fails FILE --rerun 'CMD {id}' --out BASELINE.json [--normal N] [--max-factor F] [--max-load L] | | `scripts/bg_admit.py` | admission for background and shadow jobs, plus a lint for mktemp without a cleanup trap. | bg_admit.py admit SCRIPT... | | `scripts/board_common.py` | shared post grammar and forge I/O for the board_* scripts. | — | | `scripts/board_post.py` | compose one typed, capped, privacy-linted coordination post. | board_post.py --repo O/N --issue N --agent ID --type CLAIM --refs '#12' \ --body-file note.md [--ttl 90] [--post] board… | @@ -29,9 +30,14 @@ Generated by `scripts/skill_index.py` (do not hand-edit). The script half of `IN | `scripts/feedback_ledger.py` | one durable requirement ledger across every feedback source. | feedback_ledger.py [--ledger PATH] [options] feedback_ledger.py --selftest | | `scripts/feeder_order.py` | order the work queue so a fix for a recurring failure class runs before features. | feeder_order.py --ledger FILE [--min N] [--max-run M] < items.json | | `scripts/file_finding.py` | duplicate check to run BEFORE anything files an issue. | file_finding.py [--repo OWNER/NAME] [--force REASON] TITLE [KEYWORD...] | +| `scripts/finding_guard.py` | PreToolUse(Bash) hook: deny gh issue create when file_finding.py finds the same finding already fil… | — | +| `scripts/fitness_tick.py` | run every fix's fitness check once per operating-loop tick | fitness_tick.py [--dir D] [--state F] | +| `scripts/flake_holder.py` | warn when a flaky-test fix touches a test file another open PR already changes. | flake_holder.py BASE HEAD | | `scripts/flake_ledger.py` | per-test pass/fail ledger keyed by tree SHA, with a time-boxed quarantine list. | — | +| `scripts/flow_alarm.py` | three flow checks that read the delivery queue, not the code. | flow_alarm.py symptom [--hours H] [--red-since TS] | | `scripts/focus_gate.py` | no work outside the owner's open priority until that priority is done or blocked. | focus_gate.py status [common options] [--evidence FILE] focus_gate.py check [common options] [--item REF] [--paths P ..… | | `scripts/focus_kit.py` | keep agents on task across long sessions and compactions: handoff, memory lint, docs map. | focus_kit.py handoff set\|add SECTION TEXT \| memory-lint [DIR] \| docs-index [--check] \| measure | +| `scripts/freeze_gate.py` | while a freeze is on, allow only deletions and red-trunk fixes. | freeze_gate.py BASE HEAD | | `scripts/gh_rest.py` | curl REST fallback for gh api when gh's Go TLS fails inside a macOS sandbox. | — | | `scripts/handback.py` | lane handback, gates to merge | handback.py --sha SHA --base REF --branch BRANCH --gate-cmd CMD [--gate-cmd CMD ...] | | `scripts/handback_cap.py` | SubagentStop hook: cap a subagent handback, log how far it is from fields-only, and audit each lane… | — | @@ -67,6 +73,7 @@ Generated by `scripts/skill_index.py` (do not hand-edit). The script half of `IN | `scripts/preflight.py` | run cheap gates first, report every failure together. | preflight.py CONFIG.json [--all] [--cache FILE] | | `scripts/pregate_parity.py` | may a faster checker run fail-fast before the authoritative gate? | — | | `scripts/premise_check.py` | before handing PRs or issues to a lane or peer, say which are already done or stale. | premise_check.py N [N ...] [--repo OWNER/NAME] [--timeout S] | +| `scripts/prevention_lint.py` | every prevention/action item needs an owner, a tracking id and a verifiable end state. | prevention_lint.py FILE.md | | `scripts/priority_gate.py` | no presentation-only PR while an aged P0/mechanism issue has no PR citing it. | priority_gate.py --labels-json FILE [options] priority_gate.py --repo OWNER/REPO --pr N [options] priority_gate.py --se… | | `scripts/queue_guard.py` | single priority queue: print the next issue to pull, flag P0 inflation. | queue_guard.py --issues issues.json --priorities PRIORITIES.md [--max-unranked N] issues.json gh issue list --json numb… | | `scripts/reap_own.sh` | kill your own dev servers older than MAX_AGE_S whose cwd is under ROOT (your own lane or session tr… | reap_own.sh | diff --git a/.claude/skills/agentic-delivery/INDEX.md b/.claude/skills/agentic-delivery/INDEX.md index 0f201d3f4..ec9d63b1a 100644 --- a/.claude/skills/agentic-delivery/INDEX.md +++ b/.claude/skills/agentic-delivery/INDEX.md @@ -11,7 +11,7 @@ Generated by `scripts/skill_index.py` (do not hand-edit; rerun it after changing | `references/fanout-host-sizing.md` | sizing a fan-out or capping in-flight write lanes, cutting per-lane cycle-time or handback verbosit… | 12456 | | `references/fast-agentic-delivery.md` | sizing a heavy-lane fan-out on a shared machine (the full environment-probe procedure and the fan-o… | 1703 | | `references/focus-kit.md` | a session runs long or compacts, a new session must resume the work, or memory files and docs grow… | 1308 | -| `references/full-map.md` | a delivery needs more than the SKILL.md fast path: the full gate table with every hard condition, t… | 6095 | +| `references/full-map.md` | a delivery needs more than the SKILL.md fast path: the full gate table with every hard condition, t… | 6122 | | `references/gate-epistemology.md` | adding a gate (4) | 2950 | | `references/host-enforcement.md` | claiming that state, permissions, or spend are *enforced* rather than merely followed, briefing a w… | 11228 | | `references/incident-response.md` | drafting an incident runbook, defining severity levels or status/comms templates, setting up a brea… | 3264 | @@ -22,14 +22,14 @@ Generated by `scripts/skill_index.py` (do not hand-edit; rerun it after changing | `references/pipeline-optimizer.md` | a delivery pipeline is slow (a commit or push hook over about a minute, CI over about ten minutes,… | 4838 | | `references/production-grade-product-playbook.md` | you are shaping a user-facing product surface in G3 Design, so you build to the domain-P quality ba… | 1658 | | `references/project-state.md` | beginning multi-step delivery, changing objectives, transferring ownership, checkpointing, or recov… | 1885 | -| `references/red-gate-triage.md` | a gate, union, or CI check is red and you must decide whether it is real: attributing a red union (… | 6796 | +| `references/red-gate-triage.md` | a gate, union, or CI check is red and you must decide whether it is real: attributing a red union (… | 7062 | | `references/retrospective.md` | running G10, or when a target's own incident/postmortem practice is in scope for a review. | 1211 | | `references/roles.md` | standing up (or running) the full delivery team on a repo: who does what, when a hat fires, which g… | 6830 | | `references/support-ops.md` | the product has users who file tickets, bug reports, or feature requests, and you need a support sy… | 1113 | | `references/template-adr.md` | G3 Design must record a decision — an interface, NFR budget, or data/security choice — as an ADR (c… | 660 | | `references/template-onepager.md` | G0 Intake turns a vague idea into a brief the owner can approve in one reply, or when /idea drafts… | 416 | -| `references/template-postmortem.md` | G10 or an incident meets retrospective.md's mandatory-trigger criteria and you must write the blame… | 846 | -| `references/unattended-operating-mode.md` | an agent or a Conductor is granted a block of unattended time to work a backlog — an overnight or m… | 6684 | +| `references/template-postmortem.md` | G10 or an incident meets retrospective.md's mandatory-trigger criteria and you must write the blame… | 879 | +| `references/unattended-operating-mode.md` | an agent or a Conductor is granted a block of unattended time to work a backlog — an overnight or m… | 6980 | | `references/unattended-trackers.md` | deciding whether an open issue's fix is absent, placing an auto-close keyword or a gate-parsed PR-b… | 11803 | | `references/verification-handback.md` | delegating visual/parity work by measured number, running or finalizing verification, writing a lan… | 15206 | | `references/work-tracking.md` | a repo uses a work tracker (Linear first-class | 1051 | @@ -37,4 +37,4 @@ Generated by `scripts/skill_index.py` (do not hand-edit; rerun it after changing ## Scripts -99 scripts. Read `INDEX-scripts.md` when you need a script (a trigger and CLI per script). `../INDEX.md`, `### agentic-delivery`, has only each one-line purpose and arguments. +106 scripts. Read `INDEX-scripts.md` when you need a script (a trigger and CLI per script). `../INDEX.md`, `### agentic-delivery`, has only each one-line purpose and arguments. diff --git a/.claude/skills/agentic-delivery/references/full-map.md b/.claude/skills/agentic-delivery/references/full-map.md index 96043cd4d..7e4380407 100644 --- a/.claude/skills/agentic-delivery/references/full-map.md +++ b/.claude/skills/agentic-delivery/references/full-map.md @@ -246,7 +246,7 @@ paths or generated files; else drain green PRs back-to-back (`train_land.sh`). collision*). - **A forked lane with a narrower brief than its inherited context** (research-only, "change nothing"): prefer a fresh unit; verify from effects — `references/verification-handback.md` **A context-inheriting fork is not a blank slate**. - **Inventory the repo's own scripts before spawning a lane for a repeatable step.** On start, read `scripts/` and the README; if a tick, train, bisect, land, or release script exists, run it instead of having an LLM lane do the step (or hand-resolve a generated-file conflict) by judgment. -- **New script?** `python3 scripts/wired_check.py HEAD`: a WARN means no caller (own test excluded); wire it. +- **New script?** `python3 scripts/wired_check.py HEAD`: a WARN means no caller (own test excluded); wire it. A script named only on an orphan/unused allowlist added in the same PR is `merged-not-wired`, never wired. - **Hand-back / reruns / drift?** `wired_check.py HEAD --strict --runs ` fails a new script with no caller or no observed-run line; `stage_run.py --run-dir -- CMD` refuses a reused run dir; `python3 .claude/skills/agentic-delivery/scripts/lkg.py verdict ` fails (rerun) when its `sha` is not HEAD; `python3 .claude/skills/agentic-delivery/scripts/vendor_drift.py ` exits 1 listing vendored skill files that differ from canonical (sha256). - **P0 count inflated / "what next?":** read when many open issues are P0. Don't relabel; rank a top-N in PRIORITIES.md, tag the rest `p0:unranked`; `scripts/queue_guard.py` prints the next pull and flags inflation. - Serialize shared-state edits, migrations, generated files, and the diff --git a/.claude/skills/agentic-delivery/references/red-gate-triage.md b/.claude/skills/agentic-delivery/references/red-gate-triage.md index bd155537a..48ec96dc0 100644 --- a/.claude/skills/agentic-delivery/references/red-gate-triage.md +++ b/.claude/skills/agentic-delivery/references/red-gate-triage.md @@ -33,6 +33,10 @@ audit gate in every train; run the audit on the bare base first and report `BASE `scripts/train_land.sh` implements (1) and (3) behind optional `RATCHET_CMD` and `BASE_AUDIT_CMD` (tested in `scripts/test-train-scripts.sh`); (2) is the same measurement taken on the merged result. +**Before a flaky-test fix, list the open PRs that touch that test file** (`gh pr list --state open --json number,headRefName,files`) and add the fix to the holder's PR rather than opening a second one; the pre-push hook runs `flake_holder.py`, which warns naming the holder (warn, never refuse). + +**Never store a poisoned baseline.** Store a baseline only through `python3 .claude/skills/agentic-delivery/scripts/baseline_guard.py --fails F --rerun 'CMD {id}' --out B.json --normal N`: it re-runs each failure solo (a solo pass is load, dropped), records the 1-minute load at start, and refuses a run with far more failures than normal. **A quarantine entry needs a ticket and an expiry** (`flake_ledger.py quarantine --ticket T`); a full list (`--max`, default 10) exits 1 and `--file-issue` files a P0, it is not a note for a human. **A freeze is a file, not a message**: while `.perun-freeze` (or `/PERUN_FREEZE`) exists, `freeze_gate.py BASE HEAD` (run by pre-push) lets through only pure deletions and commits with a `Red-Trunk-Fix:` line. + **Load flakes must not sink a train.** A train that is all-or-nothing under load can merge nothing for hours: failures that looked new all passed when re-run alone. With `RERUN_CMD` (and optionally `BASELINE_FAILS`), `train_land.sh` re-runs each new `FAIL ` from the browser gate alone up to 3 times (`train_flake.sh`); an id is real only when it fails all 3, diff --git a/.claude/skills/agentic-delivery/references/template-postmortem.md b/.claude/skills/agentic-delivery/references/template-postmortem.md index 703e2ccde..0f5302af5 100644 --- a/.claude/skills/agentic-delivery/references/template-postmortem.md +++ b/.claude/skills/agentic-delivery/references/template-postmortem.md @@ -63,12 +63,13 @@ fired and didn't?> ## Action items -| Action | Type (mitigate / prevent / process) | Owner | Bug/issue | Status | -|---|---|---|---|---| -| | | | | Open / In progress / Done | +| Action | Type (mitigate / prevent / process) | Owner | Bug/issue | Done when (verifiable) | Status | +|---|---|---|---|---|---| +| | | | | | Open / In progress / Done | -Every action item is owned and tracked to closure — an item with no owner -or no closing reference is not done. If this root cause has appeared in a +Every action item is owned and tracked to closure — an item with no owner, +tracking id or verifiable end state is not done +(`python3 .claude/skills/agentic-delivery/scripts/prevention_lint.py FILE.md` fails it). If this root cause has appeared in a prior postmortem, the missing action item is the finding: name which regression test, alert, or gate should have existed and did not. diff --git a/.claude/skills/agentic-delivery/references/unattended-operating-mode.md b/.claude/skills/agentic-delivery/references/unattended-operating-mode.md index a4a42e734..67d077aa2 100644 --- a/.claude/skills/agentic-delivery/references/unattended-operating-mode.md +++ b/.claude/skills/agentic-delivery/references/unattended-operating-mode.md @@ -46,7 +46,7 @@ loop, not a single task**. Frame it as the default stance: **While the owner is away, `defer` alone — never a posted message**: a decision request posted mid-run scrolls past and is unfindable later; the ledger, not the chat, is the record. Before deferring, check `task_ledger.py questions` for a - near-duplicate already answered, so a settled question is never re-asked. Before filing an issue, search closed issues and PRs by meaning; a hit is a regression to reopen, not a new filing. On the + near-duplicate already answered, so a settled question is never re-asked. Before filing an issue, search closed issues and PRs by meaning; a hit is a regression to reopen, not a new filing: the operating-layer `finding_guard.py` PreToolUse hook runs `file_finding.py` on every `gh issue create --title` and denies a match (override: `# finding-force: ` in the command; a failed search fails open). **Every fix ships one executable fitness check** in `.perun/fitness/` (exit 1 plus a one-line reason on failure) that the loop runs each tick with `python3 .claude/skills/agentic-delivery/scripts/fitness_tick.py` (prints `ALARM name: reason`); close the fix only when `fitness_tick.py closable NAME` passes (2 consecutive real passing ticks and a caller that runs the tick). **One symptom alarm, not a monitor per invariant**: each tick run `python3 .claude/skills/agentic-delivery/scripts/flow_alarm.py symptom` (ready PRs but nothing merged in 2h, or `--red-since` trunk red over 1h). **Stop starting**: the same script is a PreToolUse hook that denies `gh pr create` while 5 or more PRs sit ready-but-unmerged (`PERUN_STOP_STARTING_READY`; override `# priority: `), because waiting work is queue. **Measure before optimizing agents**: `flow_alarm.py split export.json` prints median waiting-to-start, coding and ready-to-merged hours and the largest stage. On the owner's **next** message, whatever it says, **answer it first**, then append the full open batch from `questions` — never lead with the backlog of asks. Once `answer` records a reply, the ledger drops that item from the pending batch (it diff --git a/.claude/skills/agentic-delivery/scripts/baseline_guard.py b/.claude/skills/agentic-delivery/scripts/baseline_guard.py new file mode 100644 index 000000000..fd89ae342 --- /dev/null +++ b/.claude/skills/agentic-delivery/scripts/baseline_guard.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""baseline_guard.py — store a test baseline only when it is not poisoned by load. + +WHY: a baseline taken on a loaded host holds failures that are load, not code; every later PR is then compared to a wrong +baseline. Before a baseline is stored: re-run its failures one at a time, record the host load at start, and refuse a run +whose failure count is far above normal. + + baseline_guard.py --fails FILE --rerun 'CMD {id}' --out BASELINE.json [--normal N] [--max-factor F] [--max-load L] + + --fails FILE failing test ids of the candidate baseline run, one per line (blank lines ignored). + --rerun CMD solo re-run command (shlex-split, no shell); `{id}` is replaced by the test id; exit 0 = passes solo. + --normal N the usual failure count of a healthy run. With it, more than F x max(N, 1) failures (F default 3) REFUSES. + --max-load L refuse when the 1-minute load average at start is above L (default: no limit; the value is always recorded). +Writes BASELINE.json {"fails": [ids that still fail solo], "flaky": [ids that passed solo], "load1": X, "stored_at": epoch} and +exits 0. REFUSE (exit 1, one `REFUSE baseline: ` line, nothing written): too many failures, or load above --max-load. +Exit 2 on usage error or unreadable FILE. A solo re-run that cannot start counts as still failing. Side effects: the one +output file, written only on acceptance. +""" +import argparse +import json +import os +import shlex +import subprocess +import sys +import time + + +def solo_passes(cmd, tid): + try: + argv = [a.replace("{id}", tid) for a in shlex.split(cmd)] + return subprocess.run(argv, capture_output=True, timeout=1800).returncode == 0 + except (OSError, ValueError, IndexError, subprocess.TimeoutExpired): + return False + + +def main(argv=None): + ap = argparse.ArgumentParser(prog="baseline_guard.py") + ap.add_argument("--fails", required=True) + ap.add_argument("--rerun", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--normal", type=int) + ap.add_argument("--max-factor", type=float, default=3.0) + ap.add_argument("--max-load", type=float) + a = ap.parse_args(argv) + try: + ids = sorted({ln.strip() for ln in open(a.fails, errors="replace") if ln.strip()}) + except OSError as e: + print(f"baseline_guard: {e}", file=sys.stderr) + return 2 + load1 = os.getloadavg()[0] if hasattr(os, "getloadavg") else 0.0 + if a.max_load is not None and load1 > a.max_load: + print(f"REFUSE baseline: load1 {load1:.1f} above --max-load {a.max_load}; re-take it on a quiet host") + return 1 + if a.normal is not None and len(ids) > a.max_factor * max(a.normal, 1): + print(f"REFUSE baseline: {len(ids)} failures, far above normal ({a.normal} x {a.max_factor:g}); the run is suspect") + return 1 + flaky = [t for t in ids if solo_passes(a.rerun, t)] + out = {"fails": [t for t in ids if t not in flaky], "flaky": flaky, "load1": round(load1, 2), "stored_at": int(time.time())} + with open(a.out, "w") as fh: + json.dump(out, fh, indent=1) + print(f"stored baseline: {len(out['fails'])} real, {len(flaky)} passed solo (dropped), load1={out['load1']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.claude/skills/agentic-delivery/scripts/finding_guard.py b/.claude/skills/agentic-delivery/scripts/finding_guard.py new file mode 100644 index 000000000..e83dbbec6 --- /dev/null +++ b/.claude/skills/agentic-delivery/scripts/finding_guard.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""PreToolUse(Bash) hook: deny `gh issue create` when `file_finding.py` finds the same finding already filed. + +WHY: most pipeline-filed issues repeated one already closed. `file_finding.py` (the dedupe search over open and closed issues +and PRs) only helps when it is run; this hook runs it on every `gh issue create --title ...`, so the check cannot be skipped. + +Match: a Bash command containing `gh issue create` with `--title X`, `--title=X` or `-t X` (shlex tokens; `--repo R` is passed +through). Then `file_finding.py` (next to this file) runs on the title: exit 1 (a duplicate) -> deny, exit 2 with the +matches on stderr so the agent comments on or reopens the existing item. Explicit override: put `finding-force: REASON` +(non-empty reason) anywhere in the command, e.g. a trailing `# finding-force: distinct root cause`; the reason is logged by +file_finding.py to `$PERUN_FINDING_LOG`. FAIL OPEN: a non-Bash tool, no title, unparseable input, missing file_finding.py, +a failed or timed-out `gh` search (file_finding exit 2) all allow with at most a one-line stderr note. +Side effects: the --force log line, only on an override. +""" +import json +import os +import re +import shlex +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) + + +def opt(toks, longs, short=None): + for i, t in enumerate(toks): + for lo in longs: + if t == lo and i + 1 < len(toks): + return toks[i + 1] + if t.startswith(lo + "="): + return t[len(lo) + 1:] + if short and t == short and i + 1 < len(toks): + return toks[i + 1] + return "" + + +def verdict(cmd): + """(exit code, stderr text) for one Bash command.""" + if not re.search(r"\bgh\s+issue\s+create\b", cmd): + return 0, "" + try: + toks = shlex.split(cmd, comments=False) + except ValueError: + return 0, "" + title = opt(toks, ("--title",), "-t") + if not title: + return 0, "" + cli = [sys.executable, os.path.join(HERE, "file_finding.py")] + repo = opt(toks, ("--repo",), "-R") + if repo: + cli += ["--repo", repo] + force = re.search(r"finding-force:\s*(\S.*)", cmd) + if force: + cli += ["--force", force.group(1).strip("'\" ")] + try: + r = subprocess.run(cli + [title], capture_output=True, text=True, timeout=30) + except (OSError, subprocess.TimeoutExpired): + return 0, "finding_guard: duplicate search did not run; allowing" + if r.returncode == 1: + return 2, ("Blocked `gh issue create`: a matching issue or PR already exists (below). Comment on or reopen it instead. " + "If this is genuinely different, add `# finding-force: ` to the command.\n" + r.stdout.strip()) + return 0, "" if r.returncode == 0 else "finding_guard: duplicate search failed; allowing" + + +def main(): + try: + data = json.load(sys.stdin) + if data.get("tool_name", "Bash") != "Bash": + return 0 + code, msg = verdict((data.get("tool_input") or {}).get("command") or "") + except Exception: + return 0 + if msg: + print(msg, file=sys.stderr) + return code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.claude/skills/agentic-delivery/scripts/fitness_tick.py b/.claude/skills/agentic-delivery/scripts/fitness_tick.py new file mode 100644 index 000000000..d1c699982 --- /dev/null +++ b/.claude/skills/agentic-delivery/scripts/fitness_tick.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""fitness_tick.py — run every fix's fitness check once per operating-loop tick; close a fix only on evidence. + +WHY: a fix with no executable check regresses silently, and a fix closed on one lucky run was never observed holding. +Every fix ships ONE executable file in the fitness dir (default `.perun/fitness/`): exit 0 = healthy, exit 1 = broken, +its first output line is the reason. The operating loop calls `fitness_tick.py` each tick. + + fitness_tick.py [--dir D] [--state F] run all executables in D (sorted); print `ALARM name: reason` per + failing check (one line each) and `ok name` per pass; exit 1 on any + ALARM. Records consecutive passes per check in F (default + `.perun/fitness-state.json`). Non-executable files are ignored; a check + that times out (60s) or cannot start is an ALARM. + fitness_tick.py closable NAME [--dir D] [--state F] [--root R] + exit 0 only when NAME exists in D, has >= 2 consecutive passing + ticks (two real runs), AND a caller exists: some file under R + (default `.`, excluding .git, tests, this script and the state file) + that names `fitness_tick.py`, so the loop really runs it. + Otherwise exit 1 with the one missing reason. +Exit 2 on usage error. Side effects: the state file write. Never edits or deletes a check. +""" +import json +import os +import subprocess +import sys + +NEED = 2 + + +def args(argv): + o = {"--dir": ".perun/fitness", "--state": ".perun/fitness-state.json", "--root": "."} + rest = [] + it = iter(argv) + for a in it: + if a in o: + o[a] = next(it, "") + else: + rest.append(a) + return o, rest + + +def load(p): + try: + return json.load(open(p)) + except (OSError, ValueError): + return {} + + +def tick(o): + st, bad = load(o["--state"]), 0 + names = sorted(n for n in os.listdir(o["--dir"]) if os.access(os.path.join(o["--dir"], n), os.X_OK)) \ + if os.path.isdir(o["--dir"]) else [] + if not names: + print(f"ALARM no executable fitness checks in {o['--dir']}: nothing is being verified") + return 2 + for n in names: + try: + r = subprocess.run([os.path.join(o["--dir"], n)], capture_output=True, text=True, timeout=60) + ok, why = r.returncode == 0, ((r.stdout + r.stderr).strip().splitlines() or [f"exit {r.returncode}"])[0] + except (OSError, subprocess.TimeoutExpired) as e: + ok, why = False, f"did not run: {e}" + st[n] = {"pass": st.get(n, {}).get("pass", 0) + 1 if ok else 0} + print(f"ok {n}" if ok else f"ALARM {n}: {why}") + bad += not ok + os.makedirs(os.path.dirname(o["--state"]) or ".", exist_ok=True) + json.dump(st, open(o["--state"], "w"), indent=1) + return 1 if bad else 0 + + +def has_caller(o): + skip = {os.path.realpath(__file__), os.path.realpath(o["--state"])} + r = subprocess.run(["grep", "-rIl", "--exclude-dir=.git", "--exclude=test*", "-F", "fitness_tick.py", o["--root"]], + capture_output=True, text=True) + return any(os.path.realpath(f) not in skip for f in r.stdout.split()) + + +def closable(o, name): + if not os.access(os.path.join(o["--dir"], name), os.X_OK): + print(f"NOT CLOSABLE {name}: no executable check in {o['--dir']}") + elif load(o["--state"]).get(name, {}).get("pass", 0) < NEED: + print(f"NOT CLOSABLE {name}: needs {NEED} consecutive passing ticks") + elif not has_caller(o): + print(f"NOT CLOSABLE {name}: nothing calls fitness_tick.py, so the check never runs") + else: + print(f"CLOSABLE {name}") + return 0 + return 1 + + +def main(argv): + o, rest = args(argv) + if not rest: + return tick(o) + if rest[0] == "closable" and len(rest) == 2: + return closable(o, rest[1]) + print(__doc__.split("\n\n")[0], file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.claude/skills/agentic-delivery/scripts/flake_holder.py b/.claude/skills/agentic-delivery/scripts/flake_holder.py new file mode 100644 index 000000000..ed5fbc6c9 --- /dev/null +++ b/.claude/skills/agentic-delivery/scripts/flake_holder.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""flake_holder.py BASE HEAD — warn when a flaky-test fix touches a test file another open PR already changes. + +WHY: two PRs fixing the same flaky test file conflict, double the CI, and the second one lands a stale view of the test. +The fix belongs in the holder's PR. Run by the pre-push hook before a push. + +Trigger: some commit subject in BASE..HEAD mentions "flak" (flaky, flake, flakiness; case-insensitive). Then every test file +(`test` in its path) changed in BASE..HEAD is looked up in the open PRs (`gh pr list --state open`, other branches only). +Prints `WARN: is also changed by open PR #N (): add the fix to that PR instead of opening a second one` per +holder. WARN-ONLY: always exits 0, and a missing `gh`, no network or unparseable output prints nothing (fail open). +`PERUN_GH` overrides the gh binary (tests). Side effects: none (read-only). +""" +import json +import os +import re +import subprocess +import sys + + +def out(*cmd): + return subprocess.run(cmd, capture_output=True, text=True, timeout=60) + + +def holders(changed, prs, branch): + """[(file, pr number, branch)] for each changed test file that an open PR on another branch also changes.""" + return [(f, p["number"], p.get("headRefName", "")) for p in prs if p.get("headRefName") != branch + for f in changed if f in {x.get("path") for x in p.get("files") or []}] + + +def main(argv): + if len(argv) != 2: + sys.exit("usage: flake_holder.py BASE HEAD") + base, head = argv + try: + subjects = out("git", "log", "--format=%s", f"{base}..{head}").stdout + if not re.search(r"flak", subjects, re.I): + return 0 + changed = [f for f in out("git", "diff", "--name-only", f"{base}...{head}").stdout.split() if "test" in f.lower()] + branch = out("git", "branch", "--show-current").stdout.strip() + r = out(os.environ.get("PERUN_GH", "gh"), "pr", "list", "--state", "open", "--limit", "1000", + "--json", "number,headRefName,files") + for f, n, b in holders(changed, json.loads(r.stdout or "[]"), branch) if changed and r.returncode == 0 else (): + print(f"WARN: {f} is also changed by open PR #{n} ({b}): add the fix to that PR instead of opening a second one") + except Exception: + pass + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.claude/skills/agentic-delivery/scripts/flake_ledger.py b/.claude/skills/agentic-delivery/scripts/flake_ledger.py index 0853cab9d..b95e6d92b 100644 --- a/.claude/skills/agentic-delivery/scripts/flake_ledger.py +++ b/.claude/skills/agentic-delivery/scripts/flake_ledger.py @@ -23,7 +23,11 @@ Under flock; a torn (unterminated) last line from a crash is dropped before appending. Re-recording a log only repeats evidence: counts are per distinct tree, so it is harmless. report [--n N] per-test table (flake trees, status) plus the quarantine list with expiry state. - quarantine [--n N] [--days D] add FLAKY, non-consistent, unlisted tests. Prints `QUARANTINE id expires=DATE`. + quarantine [--n N] [--days D] [--ticket T] [--max M] [--file-issue] + add FLAKY, non-consistent, unlisted tests, each with an expiry and the tracking ticket T + (`report` flags an entry without one as NO-TICKET). Prints `QUARANTINE id expires=DATE + [ticket=T]`. More than M entries (default 10) prints `FULL quarantine list`, exits 1 and, with + --file-issue, files a P0 issue (`gh issue create --label P0`) instead of leaving a note. renew ID (--days D | --expires DATE) owner review: set a new expiry for a listed id. check ID... [--tree T] [--quarantine -] `QUARANTINED id (expires DATE)` (non-blocking, report it) or `BLOCKING id (reason)`; @@ -189,6 +193,7 @@ def main(argv=None): s = sub.add_parser("serial"); s.add_argument("--n", type=int, default=3) s = sub.add_parser("serial-move"); s.add_argument("ids", nargs="+") s = sub.add_parser("quarantine"); s.add_argument("--n", type=int, default=3); s.add_argument("--days", type=int, default=14) + s.add_argument("--ticket", default=""); s.add_argument("--max", type=int, default=10); s.add_argument("--file-issue", action="store_true") s = sub.add_parser("renew"); s.add_argument("id"); g = s.add_mutually_exclusive_group(required=True) g.add_argument("--days", type=int); g.add_argument("--expires") s = sub.add_parser("check"); s.add_argument("ids", nargs="+"); s.add_argument("--tree"); s.add_argument("--quarantine", choices=["-"]) @@ -234,9 +239,17 @@ def main(argv=None): q, moved = load_q(qpath), load_moved(a.dir) for t in sorted(stats): if t not in q and t not in moved and flake_trees(stats, t) >= a.n and not consistent(stats, t, latest[t]): - q[t] = {"since": today.isoformat(), "expires": (today + datetime.timedelta(days=a.days)).isoformat()} - print(f"QUARANTINE {t} expires={q[t]['expires']}") + q[t] = {"since": today.isoformat(), "expires": (today + datetime.timedelta(days=a.days)).isoformat(), "ticket": a.ticket} + print(f"QUARANTINE {t} expires={q[t]['expires']}" + (f" ticket={a.ticket}" if a.ticket else "")) save_q(a.dir, q) + if len(q) > a.max: # a full list is a P0 to fix or delete tests, not a note for a human to read + print(f"FULL quarantine list: {len(q)} > {a.max}") + if a.file_issue: + r = subprocess.run(["gh", "issue", "create", "--label", "P0", "--title", f"Quarantine list full ({len(q)} > {a.max}): fix or delete flaky tests", + "--body", "The flake quarantine list is over its cap. Fix or delete tests until it is at or under the cap; each entry needs a ticket and an expiry."], + capture_output=True, text=True) + print(r.stdout.strip() or f"issue not filed: {r.stderr.strip()}") + return 1 return 0 stats, latest = load(a.dir) @@ -247,7 +260,8 @@ def main(argv=None): status = "consistent-fail" if consistent(stats, t, latest[t]) else "FLAKY" if ft >= a.n else "watch" if ft else "ok" print(f"{status}\t{t}\tflake_trees={ft}\ttrees={len(stats[t])}") for t, e in sorted(q.items()): - print(f"{'EXPIRED' if day(e['expires']) < today else 'QUARANTINED'}\t{t}\texpires={e['expires']}") + print(f"{'EXPIRED' if day(e['expires']) < today else 'QUARANTINED'}\t{t}\texpires={e['expires']}" + + ("" if e.get("ticket") else "\tNO-TICKET")) return 0 if a.cmd == "serial": moved = load_moved(a.dir) diff --git a/.claude/skills/agentic-delivery/scripts/flow_alarm.py b/.claude/skills/agentic-delivery/scripts/flow_alarm.py new file mode 100644 index 000000000..b47a34b0d --- /dev/null +++ b/.claude/skills/agentic-delivery/scripts/flow_alarm.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""flow_alarm.py — three flow checks that read the delivery queue, not the code. + + flow_alarm.py PreToolUse(Bash) hook (stdin JSON): STOP STARTING. Denies (exit 2) `gh pr create` while + ready-but-unmerged PRs >= PERUN_STOP_STARTING_READY (default 5): waiting work is queue, and + a new PR only lengthens it. Override for priority work: `# priority: ` in the command. + Non-PR commands, a failed `gh`, unparseable input -> allow (fail open). + flow_alarm.py symptom [--hours H] [--red-since TS] + ONE alarm instead of a monitor per invariant. Prints `ALARM flow: N ready PR(s) and nothing + merged in Xh` when ready PRs exist and the last merge is older than H (default 2), and + `ALARM flow: trunk red for Xh` when --red-since (ISO-8601 or epoch) is older than 1h. Exit 1 + on any ALARM, 0 healthy, 2 when `gh` fails (fail closed: an unread queue is not healthy). + flow_alarm.py split FILE.json value-stream split BEFORE optimizing agents. FILE is a list of {"queuedAt","startedAt", + "readyAt","mergedAt"} (ISO-8601, from the tracker/PR export). Prints median hours for + `waiting-to-start`, `coding`, `ready-to-merged` (queued->started, started->ready, ready-> + merged) and the largest stage; rows missing a timestamp are skipped. Exit 2 on bad input. + +Ready = open, not draft, at least one check and every check SUCCESS/NEUTRAL/SKIPPED. `PERUN_GH` overrides the gh binary (tests). +`--now EPOCH` pins the clock. Side effects: none (read-only). +""" +import json +import os +import re +import statistics +import subprocess +import sys +import time +from datetime import datetime, timezone + +OK = {"SUCCESS", "NEUTRAL", "SKIPPED"} + + +def ts(v): + s = str(v) + if re.fullmatch(r"\d+(\.\d+)?", s): + return float(s) + d = datetime.fromisoformat(s.replace("Z", "+00:00")) + return (d if d.tzinfo else d.replace(tzinfo=timezone.utc)).timestamp() + + +def gh_json(*a, timeout=60): + r = subprocess.run([os.environ.get("PERUN_GH", "gh"), *a], capture_output=True, text=True, timeout=timeout) + if r.returncode: + raise RuntimeError(r.stderr.strip() or "gh failed") + return json.loads(r.stdout or "[]") + + +def ready(pr): + ch = pr.get("statusCheckRollup") or [] + return (not pr.get("isDraft") and bool(ch) + and all((c.get("conclusion") or c.get("state") or "").upper() in OK for c in ch)) + + +def open_ready(timeout=60): + return sum(ready(p) for p in gh_json("pr", "list", "--state", "open", "--limit", "100", "--json", "isDraft,statusCheckRollup", + timeout=timeout)) + + +def hook(): + try: + data = json.load(sys.stdin) + cmd = (data.get("tool_input") or {}).get("command") or "" + if data.get("tool_name", "Bash") != "Bash" or not re.search(r"\bgh\s+pr\s+create\b", cmd) or "# priority:" in cmd: + return 0 + n, cap = open_ready(10), int(os.environ.get("PERUN_STOP_STARTING_READY", "5")) + except subprocess.TimeoutExpired: + print("flow_alarm: gh timed out (10s); stop-starting check skipped this call", file=sys.stderr) + return 0 + except Exception: + return 0 + if n >= cap: + print(f"Blocked `gh pr create`: {n} ready-but-unmerged PRs (cap {cap}). Waiting work is queue; help it land " + "(review, merge, unblock) before starting more. Priority work: add `# priority: ` to the command.", + file=sys.stderr) + return 2 + return 0 + + +def symptom(argv, now): + hrs = float(argv[argv.index("--hours") + 1]) if "--hours" in argv else 2.0 + try: + n = open_ready() + last = gh_json("pr", "list", "--state", "merged", "--limit", "1", "--json", "mergedAt") + red = ts(argv[argv.index("--red-since") + 1]) if "--red-since" in argv else None + except (RuntimeError, OSError, ValueError, IndexError, subprocess.TimeoutExpired) as e: + print(f"flow_alarm: could not read the queue: {e}", file=sys.stderr) + return 2 + age = (now - ts(last[0]["mergedAt"])) / 3600 if last else float("inf") + bad = 0 + if n and age > hrs: + print(f"ALARM flow: {n} ready PR(s) and nothing merged in {'ever' if age == float('inf') else f'{age:.1f}h'}") + bad = 1 + if red is not None and now - red > 3600: + print(f"ALARM flow: trunk red for {(now - red) / 3600:.1f}h") + bad = 1 + return bad + + +def split(path): + try: + rows = json.load(open(path)) + names = (("waiting-to-start", "queuedAt", "startedAt"), ("coding", "startedAt", "readyAt"), + ("ready-to-merged", "readyAt", "mergedAt")) + med = {n: [] for n, _, _ in names} + for r in rows: + if all(r.get(k) for k in ("queuedAt", "startedAt", "readyAt", "mergedAt")): + for n, a, b in names: + med[n].append((ts(r[b]) - ts(r[a])) / 3600) + if not med["coding"]: + raise ValueError("no row has all four timestamps") + except (OSError, ValueError, TypeError, AttributeError) as e: + print(f"flow_alarm: {e}", file=sys.stderr) + return 2 + m = {n: statistics.median(v) for n, v in med.items()} + for n, v in m.items(): + print(f"{n}\tmedian_h={v:.1f}") + print(f"largest\t{max(m, key=m.get)}\trows={len(med['coding'])}") + return 0 + + +def main(argv): + now = float(argv[argv.index("--now") + 1]) if "--now" in argv else time.time() + if not argv or argv[0] == "--now": + return hook() + if argv[0] == "symptom": + return symptom(argv, now) + if argv[0] == "split" and len(argv) >= 2: + return split(argv[1]) + print(__doc__.split("\n\n")[0], file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.claude/skills/agentic-delivery/scripts/freeze_gate.py b/.claude/skills/agentic-delivery/scripts/freeze_gate.py new file mode 100644 index 000000000..cabbd89dd --- /dev/null +++ b/.claude/skills/agentic-delivery/scripts/freeze_gate.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""freeze_gate.py BASE HEAD — while a freeze is on, allow only deletions and red-trunk fixes. + +WHY: a freeze (release stabilisation, incident) that lives in a chat message is broken by the next lane. This makes it a +file the push/merge preflight reads, like DCR_HOLD but selective: removing code and fixing a red trunk stay possible. + +Freeze is ON while `$PERUN_FREEZE` (a path) or `/PERUN_FREEZE` or a repo-root `.perun-freeze` exists; its first +line is the reason. Off -> exit 0 silently. On, BASE..HEAD is allowed when EITHER every changed file is a deletion +(`git diff --name-status BASE...HEAD` shows only `D`), OR a commit message in BASE..HEAD carries the line `Red-Trunk-Fix: ` with a non-empty value. Otherwise prints `FROZEN: ; only deletions and red-trunk fixes land` and exits 1. +An empty diff is allowed. Exit 2 on usage or git error (fail closed: a freeze that cannot be checked blocks). The trailer is +self-reported; the trunk's red state is the control (`red_main.py`). Side effects: none (read-only). +""" +import os +import re +import subprocess +import sys + + +def git(*a): + p = subprocess.run(["git", *a], capture_output=True, text=True) + if p.returncode: + sys.exit(f"freeze_gate: git {a[0]} failed: {p.stderr.strip()}") + return p.stdout + + +def freeze_file(): + env = os.environ.get("PERUN_FREEZE") + cands = [env] if env else [os.path.join(git("rev-parse", "--git-common-dir").strip(), "PERUN_FREEZE"), + os.path.join(git("rev-parse", "--show-toplevel").strip(), ".perun-freeze")] + return next((c for c in cands if c and os.path.exists(c)), None) + + +def main(argv): + if len(argv) != 2: + print("usage: freeze_gate.py BASE HEAD", file=sys.stderr) + return 2 + base, head = argv + f = freeze_file() + if not f: + return 0 + reason = (open(f, errors="replace").readline().strip() if os.path.isfile(f) else "") or "(no reason given)" + status = [ln.split("\t")[0][:1] for ln in git("diff", "--name-status", f"{base}...{head}").splitlines() if ln] + if all(s == "D" for s in status): + return 0 + if re.search(r"^Red-Trunk-Fix:[ \t]*\S", git("log", "--format=%B", f"{base}..{head}"), re.M): + return 0 + print(f"FROZEN: {reason}; only deletions and red-trunk fixes land (a commit line `Red-Trunk-Fix: `)") + return 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.claude/skills/agentic-delivery/scripts/prevention_lint.py b/.claude/skills/agentic-delivery/scripts/prevention_lint.py new file mode 100644 index 000000000..538e03da5 --- /dev/null +++ b/.claude/skills/agentic-delivery/scripts/prevention_lint.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""prevention_lint.py FILE.md — every prevention/action item needs an owner, a tracking id and a verifiable end state. + +WHY: "we will add monitoring" with no name, no ticket and no way to tell it is done is a wish. Reads each markdown table whose +header row has an `Owner` column, a tracking column (`Bug/issue`, `Ticket`, `Issue` or `Id`) and an end-state column (`Done when`, +`End state` or `Verify`). For every data row that has any text: Owner must be non-empty (not TBD/none/-), the tracking cell must +contain an id (`#123`, `ABC-123` or a URL), and the end-state cell must be non-empty (not TBD/none/-). Prints +`FAIL line N: ` per defect and exits 1; exits 0 when every row passes; exits 2 when the file is unreadable or +has NO table with all three columns (fail closed: a plan without the columns is itself the defect). Blank template rows are skipped. +Side effects: none. +""" +import re +import sys + +ID = re.compile(r"#\d+|\b[A-Z][A-Z0-9]+-\d+\b|https?://\S+") +EMPTY = {"", "-", "tbd", "none", "n/a", "?"} + + +def col(header, *names): + return next((i for i, h in enumerate(header) if re.sub(r"\s*\(.*\)", "", h).strip().lower() in names), None) + + +def cells(line): + return [c.strip() for c in line.strip().strip("|").split("|")] + + +def lint(text): + """(defect lines, number of tables checked).""" + bad, tables, hdr = [], 0, None + for n, line in enumerate(text.splitlines(), 1): + if not line.strip().startswith("|"): + hdr = None + continue + c = cells(line) + if hdr is None: + o, t, e = col(c, "owner"), col(c, "bug/issue", "ticket", "issue", "id"), col(c, "done when", "end state", "verify") + hdr = (o, t, e) if None not in (o, t, e) else False + tables += bool(hdr) + continue + if hdr is False or all(set(x) <= set("-: ") for x in c) or not any(c): + continue + o, t, e = hdr + for name, i, ok in (("owner", o, lambda v: v.lower() not in EMPTY), ("tracking id", t, lambda v: bool(ID.search(v))), + ("verifiable end state", e, lambda v: v.lower() not in EMPTY)): + if i >= len(c) or not ok(c[i]): + bad.append(f"FAIL line {n}: {name} missing") + return bad, tables + + +def main(argv): + if len(argv) != 1: + print("usage: prevention_lint.py FILE.md", file=sys.stderr) + return 2 + try: + bad, tables = lint(open(argv[0], errors="replace").read()) + except OSError as e: + print(f"prevention_lint: {e}", file=sys.stderr) + return 2 + if not tables: + print("FAIL: no table with Owner, tracking and end-state columns") + return 2 + print("\n".join(bad) or "ok") + return 1 if bad else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.claude/skills/agentic-delivery/scripts/reply_cap.py b/.claude/skills/agentic-delivery/scripts/reply_cap.py index 5bcc7b3ce..94cffa133 100644 --- a/.claude/skills/agentic-delivery/scripts/reply_cap.py +++ b/.claude/skills/agentic-delivery/scripts/reply_cap.py @@ -8,12 +8,15 @@ Allowed without a block: `stop_hook_active` true (one block per turn, no loop), `REPLY_CAP=off`, and a last user message that asks for detail ("detail", "details", "full", "explain"). Register on Stop and UserPromptSubmit only. +A different reply_cap script on a user-level Stop hook (`CLAUDE_USER_SETTINGS`, default ~/.claude/settings.json) makes this +one skip, so an over-cap reply gets one rewrite demand, not two. Caps: REPLY_CAP_LINES (3), REPLY_CAP_CHARS (500). Log: REPLY_CAP_LOG, else `$XDG_CACHE_HOME|~/.cache`/perun/reply-cap.log, lines `YYYY-MM-DD lines=N chars=M`; count a day with `grep -c ^2026-10-26 FILE`. Fail open: any error exits 0. Side effects: the log append only. Tests: `reply_cap.py --selftest`. """ import json import os +import re import sys import tempfile import time @@ -56,9 +59,27 @@ def log_path(): return p or os.path.join(os.environ.get("XDG_CACHE_HOME") or os.path.expanduser("~/.cache"), "perun", "reply-cap.log") +def other_cap_hook(): + """True when a user-level Stop hook runs a reply_cap script other than this file (it already enforces the cap).""" + try: + sp = os.environ.get("CLAUDE_USER_SETTINGS") or os.path.expanduser("~/.claude/settings.json") + with open(sp) as fh: + groups = (json.load(fh).get("hooks") or {}).get("Stop") or [] + me = os.path.realpath(__file__) + for g in groups: + for h in g.get("hooks") or []: + for tok in re.findall(r"[^\s\"']*reply[_-]cap[^\s\"']*", h.get("command") or ""): + p = os.path.realpath(os.path.expandvars(os.path.expanduser(tok))) + if p != me and os.path.isfile(p) and os.access(p, os.X_OK): + return True + except Exception: + pass + return False + + def check(data): """(exit code, stderr message). Appends the counter line when it blocks.""" - if data.get("stop_hook_active") or os.environ.get("REPLY_CAP", "").lower() == "off": + if data.get("stop_hook_active") or os.environ.get("REPLY_CAP", "").lower() == "off" or other_cap_hook(): return 0, "" reply, user = data.get("last_assistant_message") or "", "" p = data.get("transcript_path") @@ -97,7 +118,8 @@ def _selftest(): tmp = tempfile.mkdtemp(prefix="reply-cap-") log = os.environ["REPLY_CAP_LOG"] = os.path.join(tmp, "sub", "cap.log") os.environ.pop("REPLY_CAP", None) - tr = os.path.join(tmp, "t.jsonl") + os.environ["CLAUDE_USER_SETTINGS"] = os.path.join(tmp, "none.json") # the host's real settings must not leak in + tr =os.path.join(tmp, "t.jsonl") def write(user, reply): with open(tr, "w") as fh: @@ -123,7 +145,25 @@ def write(user, reply): os.environ.pop("REPLY_CAP") res.append(("last_assistant_message-used", check({"last_assistant_message": long_})[0] == 2)) res.append(("missing-transcript-allows", check({"transcript_path": tr + "x"}) == (0, ""))) - n = len(open(log).read().splitlines()) + us = os.environ["CLAUDE_USER_SETTINGS"] = os.path.join(tmp, "user.json") + + def settings(cmd): + json.dump({"hooks": {"Stop": [{"hooks": [{"type": "command", "command": cmd}]}]}}, open(us, "w")) + settings('python3 "/elsewhere/reply_cap.py"') + res.append(("nonexistent-other-cap-hook-still-blocks", check({"transcript_path": tr})[0] == 2)) + other = os.path.join(tmp, "reply_cap.py") + open(other, "w").write("#!/bin/sh\n") + settings(f'python3 "{other}"') + res.append(("non-executable-other-cap-hook-still-blocks", check({"transcript_path": tr})[0] == 2)) + os.chmod(other, 0o755) + res.append(("other-user-level-cap-hook-skips", check({"transcript_path": tr}) == (0, ""))) + settings(f'python3 "{os.path.abspath(__file__)}"') + res.append(("own-user-level-entry-still-blocks", check({"transcript_path": tr})[0] == 2)) + settings("python3 other_hook.py") + res.append(("unrelated-stop-hook-still-blocks", check({"transcript_path": tr})[0] == 2)) + os.environ["CLAUDE_USER_SETTINGS"] = os.path.join(tmp, "missing.json") + res.append(("missing-user-settings-still-blocks", check({"transcript_path": tr})[0] == 2)) + n =len(open(log).read().splitlines()) r = subprocess.run([sys.executable, __file__], input=json.dumps({"hook_event_name": "UserPromptSubmit"}), capture_output=True, text=True) res.append(("prompt-submit-injects-cap", r.returncode == 0 and "Reply cap" in r.stdout and "3 short lines" in r.stdout)) diff --git a/.claude/skills/agentic-delivery/scripts/wired_check.py b/.claude/skills/agentic-delivery/scripts/wired_check.py index 161ecc265..724851065 100644 --- a/.claude/skills/agentic-delivery/scripts/wired_check.py +++ b/.claude/skills/agentic-delivery/scripts/wired_check.py @@ -22,6 +22,7 @@ CODE_EXT = (".json", ".yml", ".yaml", ".sh", ".py") RUN_RE = re.compile(r"python3?\b|\bbash\b|\bsh\b|\./|\brun\b", re.I) SKIP = ("INDEX.md", "CHANGELOG.md", "SHA256SUMS") +ALLOW_RE = re.compile(r"orphan|unused|allow[-_ ]?list|\bunreferenced\b", re.I) def git(root, *a, ok=(0,)): @@ -34,7 +35,10 @@ def git(root, *a, ok=(0,)): def counts(path, hits): """True when `path` is a caller kind that counts and `hits` (its lines naming the script) qualify.""" base, low = os.path.basename(path), path.lower() - if TEST_RE.search(path) or "/evals/" in path or base in SKIP or low.startswith("changelog.d/"): + if TEST_RE.search(path) or "/evals/" in path or base in SKIP or low.startswith("changelog.d/") or ALLOW_RE.search(base): + return False + hits = [h for h in hits if not ALLOW_RE.search(h)] # a name on an orphan/unused allowlist line is not a caller + if not hits: return False if base == "Makefile" or low.endswith(CODE_EXT): return True @@ -79,8 +83,10 @@ def main(argv): by_file.setdefault(parts[1], []).append(parts[3]) if not any(counts(f, h) for f, h in by_file.items()): unwired += 1 + listed = any(ALLOW_RE.search(os.path.basename(f)) or any(ALLOW_RE.search(x) for x in h) for f, h in by_file.items()) print("WARN: %s has no caller (hook/settings, CI, command, package.json, Makefile, script, " - "or a run line in SKILL.md/references)" % s) + "or a run line in SKILL.md/references)" % s + ( + "; merged-not-wired: it is only named on an orphan/unused allowlist added with it" if listed else "")) if strict and not re.search(r"(?&2 || true fi + # Freeze (a PERUN_FREEZE / .perun-freeze file): only deletions and Red-Trunk-Fix commits may push; exit 1 refuses. + freeze_gate=".claude/skills/agentic-delivery/scripts/freeze_gate.py" + if [ -f "${freeze_gate}" ]; then + fg_rc=0 + python3 "${freeze_gate}" "${base_sha}" "${local_sha}" >&2 || fg_rc=$? + if [ "${fg_rc}" -ne 0 ]; then + printf 'pre-push-verify: FAIL -- freeze gate (lines above).\n' >&2 + fail=1 + continue + fi + fi + + # Warn-only: a flaky-test fix whose test file another open PR already changes (names the holder); never fails the push. + flake_holder=".claude/skills/agentic-delivery/scripts/flake_holder.py" + if [ -f "${flake_holder}" ]; then + python3 "${flake_holder}" "${base_sha}" "${local_sha}" >&2 || true + fi + # Tests that read the real clock (red at the next week rollover) or bet on timing under load: findings print # as warnings; only exit 1 (a timing bet in a test file ADDED in this push range) blocks. Exit 2 never blocks. clock_lint=".claude/skills/agentic-delivery/scripts/clock_lint.py" diff --git a/changelog.d/wave-ai-peer-lessons-batch.md b/changelog.d/wave-ai-peer-lessons-batch.md new file mode 100644 index 000000000..38708c3bc --- /dev/null +++ b/changelog.d/wave-ai-peer-lessons-batch.md @@ -0,0 +1,12 @@ +### Added +- agentic-delivery: `fitness_tick.py` runs each fix's executable fitness check every operating-loop tick (one-line `ALARM`), and `closable NAME` passes only after 2 consecutive passing ticks and a caller. `flake_holder.py` (pre-push, warn-only) names the open PR that already changes a flaky-test fix's test file. `finding_guard.py` (operating-layer PreToolUse hook) runs the `file_finding.py` dedupe search on every `gh issue create --title` and denies a match (`# finding-force: ` overrides; a failed search fails open). +- agentic-delivery: `wired_check.py` now reports a new script that appears only on an orphan/unused allowlist as `merged-not-wired`. +- agentic-delivery: `baseline_guard.py` stores a test baseline only after solo re-runs, with the host load recorded, and refuses a run far above the normal failure count. `freeze_gate.py` (pre-push) enforces a freeze file: only pure deletions and `Red-Trunk-Fix:` commits pass. `flow_alarm.py` is one symptom alarm (ready work but nothing merged in 2h, or trunk red over 1h), a stop-starting PreToolUse hook that denies `gh pr create` while 5 or more PRs are ready-but-unmerged, and a value-stream split (`split`). `prevention_lint.py` fails a postmortem action item that lacks an owner, tracking id or verifiable end state. + +### Changed +- agentic-delivery: `reply_cap.py` skips when a user-level Stop hook already runs a different reply-cap script, so an over-cap reply gets one rewrite demand, not two. +- agentic-delivery: `flake_ledger.py quarantine` takes `--ticket`, flags ticketless entries in `report`, and on a full list (`--max`, default 10) exits 1 and can file a P0 issue (`--file-issue`). +size-budget-raise: .claude/skills/agentic-delivery/references/unattended-operating-mode.md 26739→27923 fitness-check close rule, finding_guard hook, symptom alarm, stop-starting, value-stream split +size-budget-raise: .claude/skills/agentic-delivery/references/red-gate-triage.md 27186→28248 flaky-fix holder rule, baseline guard, quarantine ticket, freeze file +size-budget-raise: .claude/skills/agentic-delivery/references/full-map.md 24381→24488 merged-not-wired allowlist note +size-budget-raise: .claude/skills/agentic-delivery/references/template-postmortem.md 3387→3519 done-when column and prevention lint diff --git a/scripts/ci-tests.d/peer-lessons-batch.sh b/scripts/ci-tests.d/peer-lessons-batch.sh new file mode 100644 index 000000000..9deec8160 --- /dev/null +++ b/scripts/ci-tests.d/peer-lessons-batch.sh @@ -0,0 +1,11 @@ +# Registers the peer-lessons batch tests with CI (see the "Per-PR tests" step in ci.yml). +python3 scripts/test_fitness_tick.py +python3 scripts/test_flake_holder.py +python3 scripts/test_finding_guard.py +python3 scripts/test_orphan_allowlist_wired.py +python3 scripts/test_freeze_gate.py +python3 scripts/test_flow_alarm.py +python3 scripts/test_baseline_guard.py +python3 scripts/test_quarantine_ticket_cap.py +python3 scripts/test_prevention_lint.py +python3 -c 'import json,sys; t=json.load(open(".claude/skills/agentic-delivery/templates/operating-layer.settings.json")); c=[h["command"] for e in t["hooks"]["PreToolUse"] for h in e["hooks"]]; sys.exit(0 if all(any(s in x for x in c) for s in ("finding_guard.py", "flow_alarm.py")) else 1)' diff --git a/scripts/perun_doctor.py b/scripts/perun_doctor.py index 16441bafe..359d340ac 100644 --- a/scripts/perun_doctor.py +++ b/scripts/perun_doctor.py @@ -19,7 +19,7 @@ SRC = Path(__file__).resolve().parent.parent SKIP = {".git", "node_modules", ".venv", "__pycache__", "worktrees", "skill-backups"} SKILL_HOSTS = (".claude", ".cursor", ".agents", ".codex", ".gemini", ".opencode", ".github", ".windsurf", ".hermes", ".kiro", ".cline", ".roo", ".factory", ".qwen", ".grok") # every root install.sh can write -HOOK_SCRIPTS = ("pipe_mask_guard.py", "bypass_guard.py", "destructive_guard.py", "subagent_start_inject.py", "handback_cap.py", "context_nudge.py", "lane_resume_nudge.py", "ceo_altitude_guard.py", "comms_guard.py", "outcome_claim_guard.py", "landing_watchdog.py", "stall_guard.py", "reply_cap.py", "decision_gate.py", "repeat_poll_guard.py", "subagent_brief_lint.py") # operating-layer template +HOOK_SCRIPTS = ("pipe_mask_guard.py", "bypass_guard.py", "destructive_guard.py", "subagent_start_inject.py", "handback_cap.py", "context_nudge.py", "lane_resume_nudge.py", "ceo_altitude_guard.py", "comms_guard.py", "outcome_claim_guard.py", "landing_watchdog.py", "stall_guard.py", "reply_cap.py", "decision_gate.py", "repeat_poll_guard.py", "subagent_brief_lint.py", "finding_guard.py", "flow_alarm.py") # operating-layer template def sha(p): diff --git a/scripts/test_baseline_guard.py b/scripts/test_baseline_guard.py new file mode 100644 index 000000000..88ee28029 --- /dev/null +++ b/scripts/test_baseline_guard.py @@ -0,0 +1,53 @@ +"""baseline_guard.py: solo re-run drops load failures, records load, refuses a suspect run, writes nothing on refusal.""" +import json +import os +import pathlib +import subprocess +import sys +import tempfile +import unittest + +SCRIPT = pathlib.Path(__file__).resolve().parent.parent / ".claude/skills/agentic-delivery/scripts/baseline_guard.py" + + +def run(ids, *extra): + d = tempfile.mkdtemp() + pathlib.Path(d, "fails.txt").write_text("\n".join(ids) + "\n") + rerun = pathlib.Path(d, "rerun.sh") # test id starting with "real" fails solo, anything else passes + rerun.write_text('#!/bin/sh\ncase "$1" in real*) exit 1;; esac\nexit 0\n') + rerun.chmod(0o755) + out = os.path.join(d, "base.json") + r = subprocess.run([sys.executable, str(SCRIPT), "--fails", f"{d}/fails.txt", "--rerun", f"{rerun} {{id}}", "--out", out, *extra], + capture_output=True, text=True, env={**os.environ, "PERUN_GATE_CORES": "4"}) + return r, out + + +class T(unittest.TestCase): + def test_solo_passes_are_dropped_and_load_recorded(self): + r, out = run(["real_a", "load_b", "load_c"]) + self.assertEqual(r.returncode, 0) + b = json.load(open(out)) + self.assertEqual((b["fails"], b["flaky"]), (["real_a"], ["load_b", "load_c"])) + self.assertIn("load1", b) + + def test_refuses_far_above_normal_and_writes_nothing(self): + r, out = run([f"real_{i}" for i in range(10)], "--normal", "2") + self.assertEqual(r.returncode, 1) + self.assertIn("REFUSE baseline: 10 failures", r.stdout) + self.assertFalse(os.path.exists(out)) + + def test_within_normal_is_stored(self): + self.assertEqual(run([f"real_{i}" for i in range(6)], "--normal", "2")[0].returncode, 0) + + def test_refuses_when_host_load_above_max(self): + r, out = run(["real_a"], "--max-load", "-1") + self.assertEqual((r.returncode, os.path.exists(out)), (1, False)) + + def test_unreadable_fails_file_exit_2(self): + r = subprocess.run([sys.executable, str(SCRIPT), "--fails", "/nonexistent/f", "--rerun", "true", "--out", "/dev/null"], + capture_output=True, text=True, env={**os.environ, "PERUN_GATE_CORES": "4"}) + self.assertEqual(r.returncode, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_finding_guard.py b/scripts/test_finding_guard.py new file mode 100644 index 000000000..59ffbe57c --- /dev/null +++ b/scripts/test_finding_guard.py @@ -0,0 +1,51 @@ +"""finding_guard.py: deny a duplicate `gh issue create`, explicit override, fail open on tool error.""" +import json +import os +import pathlib +import subprocess +import sys +import tempfile +import unittest + +SCRIPT = pathlib.Path(__file__).resolve().parent.parent / ".claude/skills/agentic-delivery/scripts/finding_guard.py" +HIT = '[{"number":5,"title":"Queue stalls after retry timeout","state":"CLOSED","url":"https://example.com/5"}]' +CREATE = 'gh issue create --title "Queue stalls after retry timeout" --body x' + + +class T(unittest.TestCase): + def hook(self, cmd, gh_out="[]", gh_rc=0, tool="Bash"): + d = tempfile.mkdtemp() + gh = pathlib.Path(d, "gh") + gh.write_text(f"#!/bin/sh\n[ {gh_rc} -ne 0 ] && exit {gh_rc}\necho '{gh_out}'\n") + gh.chmod(0o755) + env = {**os.environ, "PATH": f"{d}:{os.environ['PATH']}", "PERUN_FINDING_LOG": f"{d}/force.log"} + r = subprocess.run([sys.executable, str(SCRIPT)], input=json.dumps({"tool_name": tool, "tool_input": {"command": cmd}}), + capture_output=True, text=True, env=env) + return r, pathlib.Path(d, "force.log") + + def test_denies_duplicate_and_names_it(self): + r, _ = self.hook(CREATE, HIT) + self.assertEqual(r.returncode, 2) + self.assertIn("#5 [CLOSED]", r.stderr) + + def test_allows_when_no_match(self): + self.assertEqual(self.hook(CREATE)[0].returncode, 0) + + def test_explicit_override_allows_and_logs_reason(self): + r, log = self.hook(CREATE + " # finding-force: distinct root cause", HIT) + self.assertEqual(r.returncode, 0) + self.assertIn("distinct root cause", log.read_text()) + + def test_fail_open_on_search_error_and_garbage(self): + r, _ = self.hook(CREATE, HIT, gh_rc=1) + self.assertEqual((r.returncode, "allowing" in r.stderr), (0, True)) + r = subprocess.run([sys.executable, str(SCRIPT)], input="not json", capture_output=True, text=True) + self.assertEqual(r.returncode, 0) + + def test_ignores_other_commands_tools_and_untitled_creates(self): + for cmd, tool in (("gh issue list --search queue", "Bash"), ("gh issue create --body x", "Bash"), (CREATE, "Edit")): + self.assertEqual(self.hook(cmd, HIT, tool=tool)[0].returncode, 0, cmd) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_fitness_tick.py b/scripts/test_fitness_tick.py new file mode 100644 index 000000000..9b74e1ed1 --- /dev/null +++ b/scripts/test_fitness_tick.py @@ -0,0 +1,75 @@ +"""fitness_tick.py: ALARM lines, consecutive-pass counting, and the closable gate (2 passes + a caller).""" +import os +import pathlib +import subprocess +import sys +import tempfile +import unittest + +SCRIPT = pathlib.Path(__file__).resolve().parent.parent / ".claude/skills/agentic-delivery/scripts/fitness_tick.py" + + +def run(root, *a): + return subprocess.run([sys.executable, str(SCRIPT), "--dir", f"{root}/fit", "--state", f"{root}/st.json", + "--root", f"{root}/repo", *a], capture_output=True, text=True) + + +class T(unittest.TestCase): + def setUp(self): + self.d = tempfile.mkdtemp() + os.makedirs(f"{self.d}/fit") + os.makedirs(f"{self.d}/repo") + + def check(self, name, code, msg="boom"): + p = f"{self.d}/fit/{name}" + pathlib.Path(p).write_text(f"#!/bin/sh\necho {msg}\nexit {code}\n") + os.chmod(p, 0o755) + + def test_alarm_one_line_and_exit_1(self): + self.check("a", 1, "queue stuck") + r = run(self.d) + self.assertEqual((r.returncode, r.stdout), (1, "ALARM a: queue stuck\n")) + + def test_missing_or_empty_dir_alarms_exit_2(self): + r = run(self.d) # fit/ exists but is empty + self.assertEqual((r.returncode, r.stdout.startswith("ALARM no executable fitness checks")), (2, True)) + os.rmdir(f"{self.d}/fit") + self.assertEqual(run(self.d).returncode, 2) + + def test_pass_exit_0_and_non_executable_ignored(self): + self.check("a", 0) + pathlib.Path(f"{self.d}/fit/notes.txt").write_text("x") + r = run(self.d) + self.assertEqual((r.returncode, r.stdout), (0, "ok a\n")) + + def test_closable_needs_two_passes_then_caller(self): + self.check("a", 0) + run(self.d) + r = run(self.d, "closable", "a") + self.assertEqual(r.returncode, 1) + self.assertIn("2 consecutive", r.stdout) + run(self.d) + r = run(self.d, "closable", "a") + self.assertEqual(r.returncode, 1) + self.assertIn("nothing calls fitness_tick.py", r.stdout) + pathlib.Path(f"{self.d}/repo/loop.sh").write_text("python3 fitness_tick.py\n") + self.assertEqual(run(self.d, "closable", "a").stdout, "CLOSABLE a\n") + + def test_failure_resets_the_streak(self): + self.check("a", 0) + run(self.d) + run(self.d) + self.check("a", 1) + run(self.d) + self.check("a", 0) + run(self.d) + pathlib.Path(f"{self.d}/repo/loop.sh").write_text("python3 fitness_tick.py\n") + self.assertEqual(run(self.d, "closable", "a").returncode, 1) + + def test_missing_check_not_closable_and_usage_exit_2(self): + self.assertEqual(run(self.d, "closable", "nope").returncode, 1) + self.assertEqual(run(self.d, "bogus").returncode, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_flake_holder.py b/scripts/test_flake_holder.py new file mode 100644 index 000000000..0f8ad4c03 --- /dev/null +++ b/scripts/test_flake_holder.py @@ -0,0 +1,58 @@ +"""flake_holder.py: names the open PR holding a flaky test's file; warn-only; silent when not a flake fix.""" +import json +import os +import pathlib +import subprocess +import sys +import tempfile +import unittest + +SCRIPT = pathlib.Path(__file__).resolve().parent.parent / ".claude/skills/agentic-delivery/scripts/flake_holder.py" +PRS = [{"number": 7, "headRefName": "other", "files": [{"path": "tests/test_a.py"}]}, + {"number": 8, "headRefName": "mine", "files": [{"path": "tests/test_a.py"}]}] + + +class T(unittest.TestCase): + def repo(self, subject): + d = tempfile.mkdtemp() + g = lambda *a: subprocess.run(["git", "-C", d, *a], check=True, capture_output=True) # noqa: E731 + g("init", "-b", "mine") + g("config", "user.email", "a@example.com") + g("config", "user.name", "A") + g("commit", "--allow-empty", "-m", "base") + os.makedirs(f"{d}/tests") + pathlib.Path(f"{d}/tests/test_a.py").write_text("x\n") + g("add", ".") + g("commit", "-m", subject) + return d + + def run_(self, d, gh_out, rc=0): + gh = pathlib.Path(d, "gh.sh") # inside the repo dir but untracked: fine for a read-only diff + gh.write_text(f"#!/bin/sh\necho \"$*\" > {d}/args.txt\ncat <<'EOF'\n{gh_out}\nEOF\nexit {rc}\n") + gh.chmod(0o755) + r = subprocess.run([sys.executable, str(SCRIPT), "HEAD~1", "HEAD"], cwd=d, capture_output=True, text=True, + env={**os.environ, "PERUN_GH": str(gh)}) + return r + + def test_names_holder_and_skips_own_branch(self): + r = self.run_(self.repo("fix flaky test_a"), json.dumps(PRS)) + self.assertEqual(r.returncode, 0) + self.assertEqual(r.stdout.count("WARN"), 1) + self.assertIn("tests/test_a.py is also changed by open PR #7 (other)", r.stdout) + + def test_pr_list_limit_is_1000_not_100(self): + d = self.repo("fix flaky test_a") + self.run_(d, "[]") + self.assertIn("--limit 1000", pathlib.Path(d, "args.txt").read_text()) + + def test_silent_when_not_a_flake_fix(self): + self.assertEqual(self.run_(self.repo("add feature"), json.dumps(PRS)).stdout, "") + + def test_fail_open_on_gh_error_or_garbage(self): + d = self.repo("deflake it: flakiness") + self.assertEqual((self.run_(d, "", 1).returncode, self.run_(d, "", 1).stdout), (0, "")) + self.assertEqual((self.run_(d, "not json").returncode, self.run_(d, "not json").stdout), (0, "")) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_flow_alarm.py b/scripts/test_flow_alarm.py new file mode 100644 index 000000000..0f2a3fd25 --- /dev/null +++ b/scripts/test_flow_alarm.py @@ -0,0 +1,103 @@ +"""flow_alarm.py: stop-starting hook, one symptom alarm, value-stream split.""" +import json +import os +import pathlib +import subprocess +import sys +import tempfile +import unittest + +SCRIPT = pathlib.Path(__file__).resolve().parent.parent / ".claude/skills/agentic-delivery/scripts/flow_alarm.py" +GREEN = {"isDraft": False, "statusCheckRollup": [{"conclusion": "SUCCESS"}]} +RED = {"isDraft": False, "statusCheckRollup": [{"conclusion": "FAILURE"}]} +DRAFT = {"isDraft": True, "statusCheckRollup": [{"conclusion": "SUCCESS"}]} +NOW = 1_800_000_000 + + +def run(args, open_prs=(), merged=(), stdin="", fail=False, env=None): + d = tempfile.mkdtemp() + pathlib.Path(d, "open.json").write_text(json.dumps(list(open_prs))) + pathlib.Path(d, "merged.json").write_text(json.dumps(list(merged))) + gh = pathlib.Path(d, "gh") + gh.write_text(f'#!/bin/sh\n[ {int(fail)} -eq 1 ] && exit 1\ncase "$*" in *"--state open"*) cat {d}/open.json;; ' + f'*"--state merged"*) cat {d}/merged.json;; esac\n') + gh.chmod(0o755) + return subprocess.run([sys.executable, str(SCRIPT), *args, "--now", str(NOW)], input=stdin, capture_output=True, text=True, + env={**os.environ, "PERUN_GH": str(gh), **(env or {})}) + + +def hook(cmd, ready, env=None): + return run([], [GREEN] * ready, stdin=json.dumps({"tool_name": "Bash", "tool_input": {"command": cmd}}), env=env) + + +class T(unittest.TestCase): + def test_hook_blocks_pr_create_at_threshold(self): + r = hook("gh pr create --title x", 5) + self.assertEqual(r.returncode, 2) + self.assertIn("5 ready-but-unmerged", r.stderr) + + def test_hook_allows_below_threshold_priority_and_other_commands(self): + self.assertEqual(hook("gh pr create --title x", 4).returncode, 0) + self.assertEqual(hook("gh pr create --title x # priority: red trunk fix", 9).returncode, 0) + self.assertEqual(hook("gh pr list", 9).returncode, 0) + self.assertEqual(hook("gh pr create", 2, {"PERUN_STOP_STARTING_READY": "2"}).returncode, 2) + + def test_hook_gh_timeout_degrades_with_notice(self): + import importlib.util + import unittest.mock as m + spec = importlib.util.spec_from_file_location("fa", SCRIPT) + fa = importlib.util.module_from_spec(spec) + spec.loader.exec_module(fa) + seen = [] + + def slow(*a, timeout=60): + seen.append(timeout) + raise subprocess.TimeoutExpired("gh", timeout) + stdin = json.dumps({"tool_name": "Bash", "tool_input": {"command": "gh pr create"}}) + with m.patch.object(fa, "gh_json", slow), m.patch("sys.stdin", __import__("io").StringIO(stdin)), \ + m.patch("sys.stderr", __import__("io").StringIO()) as err: + self.assertEqual(fa.hook(), 0) + self.assertEqual(seen, [10]) + self.assertIn("timed out", err.getvalue()) + + def test_hook_fails_open(self): + self.assertEqual(run([], fail=True, stdin='{"tool_input":{"command":"gh pr create"}}').returncode, 0) + self.assertEqual(run([], stdin="garbage").returncode, 0) + + def test_draft_and_red_prs_are_not_ready(self): + r = run([], [GREEN, RED, DRAFT] * 3, stdin='{"tool_input":{"command":"gh pr create"}}', + env={"PERUN_STOP_STARTING_READY": "4"}) + self.assertEqual(r.returncode, 0) # 3 ready < 4 + + def test_symptom_ready_but_nothing_landed(self): + r = run(["symptom"], [GREEN], [{"mergedAt": "2027-01-15T05:00:00Z"}]) # epoch 1.8e9 = 2027-01-15T08:00:00Z + self.assertEqual((r.returncode, r.stdout), (1, "ALARM flow: 1 ready PR(s) and nothing merged in 3.0h\n")) + + def test_symptom_healthy_cases(self): + self.assertEqual(run(["symptom"], [GREEN], [{"mergedAt": "2027-01-15T07:30:00Z"}]).returncode, 0) + self.assertEqual(run(["symptom"], [], [{"mergedAt": "2020-01-01T00:00:00Z"}]).returncode, 0) # idle, not stuck + + def test_symptom_trunk_red_and_fail_closed(self): + r = run(["symptom", "--red-since", str(NOW - 7200)], [], [{"mergedAt": "2027-01-15T07:30:00Z"}]) + self.assertEqual((r.returncode, r.stdout), (1, "ALARM flow: trunk red for 2.0h\n")) + self.assertEqual(run(["symptom"], fail=True).returncode, 2) + + def test_split_prints_medians_and_largest(self): + f = pathlib.Path(tempfile.mkdtemp(), "v.json") + row = {"queuedAt": "2027-01-01T00:00:00Z", "startedAt": "2027-01-01T10:00:00Z", + "readyAt": "2027-01-01T12:00:00Z", "mergedAt": "2027-01-01T13:00:00Z"} + f.write_text(json.dumps([row, {"queuedAt": "2027-01-01T00:00:00Z"}])) + r = run(["split", str(f)]) + self.assertEqual(r.returncode, 0) + self.assertIn("waiting-to-start\tmedian_h=10.0", r.stdout) + self.assertIn("largest\twaiting-to-start\trows=1", r.stdout) + + def test_split_bad_input_exit_2(self): + f = pathlib.Path(tempfile.mkdtemp(), "v.json") + f.write_text("[]") + self.assertEqual(run(["split", str(f)]).returncode, 2) + self.assertEqual(run(["bogus"]).returncode, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_freeze_gate.py b/scripts/test_freeze_gate.py new file mode 100644 index 000000000..0f9a5814e --- /dev/null +++ b/scripts/test_freeze_gate.py @@ -0,0 +1,63 @@ +"""freeze_gate.py: freeze off allows all; on allows only deletions or a Red-Trunk-Fix trailer.""" +import os +import pathlib +import subprocess +import sys +import tempfile +import unittest + +SCRIPT = pathlib.Path(__file__).resolve().parent.parent / ".claude/skills/agentic-delivery/scripts/freeze_gate.py" + + +def repo(change, msg="work"): + d = tempfile.mkdtemp() + g = lambda *a: subprocess.run(["git", "-C", d, *a], check=True, capture_output=True) # noqa: E731 + g("init", "-b", "main") + g("config", "user.email", "a@example.com") + g("config", "user.name", "A") + pathlib.Path(d, "old.txt").write_text("x\n") + g("add", ".") + g("commit", "-m", "base") + g("checkout", "-b", "feat") + change(d) + g("add", "-A") + g("commit", "--allow-empty", "-m", msg) + return d + + +def gate(d, frozen=True): + if frozen: + pathlib.Path(d, ".perun-freeze").write_text("release stabilisation\n") + return subprocess.run([sys.executable, str(SCRIPT), "main", "feat"], cwd=d, capture_output=True, text=True, + env={k: v for k, v in os.environ.items() if k != "PERUN_FREEZE"}) + + +add = lambda d: pathlib.Path(d, "new.txt").write_text("y\n") # noqa: E731 +rm = lambda d: os.remove(f"{d}/old.txt") # noqa: E731 + + +class T(unittest.TestCase): + def test_no_freeze_allows_anything(self): + self.assertEqual(gate(repo(add), frozen=False).returncode, 0) + + def test_freeze_blocks_addition_with_reason(self): + r = gate(repo(add)) + self.assertEqual(r.returncode, 1) + self.assertIn("FROZEN: release stabilisation", r.stdout) + + def test_freeze_allows_pure_deletion(self): + self.assertEqual(gate(repo(rm)).returncode, 0) + + def test_freeze_blocks_mixed_delete_and_add(self): + self.assertEqual(gate(repo(lambda d: (rm(d), add(d)))).returncode, 1) + + def test_freeze_allows_red_trunk_fix_trailer(self): + self.assertEqual(gate(repo(add, "fix\n\nRed-Trunk-Fix: unit suite red")).returncode, 0) + self.assertEqual(gate(repo(add, "fix\n\nRed-Trunk-Fix:")).returncode, 1) + + def test_bad_args_exit_2(self): + self.assertEqual(subprocess.run([sys.executable, str(SCRIPT)], capture_output=True).returncode, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_orphan_allowlist_wired.py b/scripts/test_orphan_allowlist_wired.py new file mode 100644 index 000000000..fad961cb2 --- /dev/null +++ b/scripts/test_orphan_allowlist_wired.py @@ -0,0 +1,50 @@ +"""wired_check.py: a new script named only on an orphan/unused allowlist is merged-not-wired, not wired.""" +import os +import pathlib +import subprocess +import sys +import tempfile +import unittest + +SCRIPT = pathlib.Path(__file__).resolve().parent.parent / ".claude/skills/agentic-delivery/scripts/wired_check.py" + + +def wired(files): + d = tempfile.mkdtemp() + g = lambda *a: subprocess.run(["git", "-C", d, *a], check=True, capture_output=True) # noqa: E731 + g("init", "-b", "main") + g("config", "user.email", "a@example.com") + g("config", "user.name", "A") + g("commit", "--allow-empty", "-m", "base") + g("checkout", "-b", "feat") + for p, text in files.items(): + os.makedirs(os.path.dirname(f"{d}/{p}"), exist_ok=True) + pathlib.Path(f"{d}/{p}").write_text(text) + g("add", ".") + g("commit", "-m", "add") + return subprocess.run([sys.executable, str(SCRIPT), "main", "feat", "--root", d], capture_output=True, text=True).stdout + + +class T(unittest.TestCase): + def test_allowlist_line_does_not_wire(self): + out = wired({"scripts/tool.sh": "echo hi\n", "scripts/check.sh": 'ORPHAN_ALLOW="tool.sh"\n'}) + self.assertIn("scripts/tool.sh has no caller", out) + self.assertIn("merged-not-wired", out) + + def test_allowlist_named_file_does_not_wire(self): + out = wired({"scripts/tool.sh": "echo hi\n", "scripts/unused-allowlist.json": '["tool.sh"]\n'}) + self.assertIn("merged-not-wired", out) + + def test_real_caller_still_wires_even_with_an_allowlist_entry(self): + out = wired({"scripts/tool.sh": "echo hi\n", "scripts/run.sh": "bash scripts/tool.sh\n", + "scripts/unused-allowlist.json": '["tool.sh"]\n'}) + self.assertNotIn("scripts/tool.sh has no caller", out) # (run.sh, the caller, is itself new and unwired) + + def test_plain_unwired_has_no_allowlist_note(self): + out = wired({"scripts/tool.sh": "echo hi\n"}) + self.assertIn("has no caller", out) + self.assertNotIn("merged-not-wired", out) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_prevention_lint.py b/scripts/test_prevention_lint.py new file mode 100644 index 000000000..6adff87f9 --- /dev/null +++ b/scripts/test_prevention_lint.py @@ -0,0 +1,53 @@ +"""prevention_lint.py: each action item needs owner, tracking id, verifiable end state.""" +import pathlib +import subprocess +import sys +import tempfile +import unittest + +SCRIPT = pathlib.Path(__file__).resolve().parent.parent / ".claude/skills/agentic-delivery/scripts/prevention_lint.py" +HEAD = "| Action | Type | Owner | Bug/issue | Done when |\n|---|---|---|---|---|\n" + + +def lint(body, head=HEAD): + f = pathlib.Path(tempfile.mkdtemp(), "pm.md") + f.write_text(head + body) + return subprocess.run([sys.executable, str(SCRIPT), str(f)], capture_output=True, text=True) + + +class T(unittest.TestCase): + def test_complete_row_passes(self): + r = lint("| add alert | prevent | jane | #42 | alert fires in the staging drill |\n") + self.assertEqual((r.returncode, r.stdout), (0, "ok\n")) + + def test_each_missing_field_is_named(self): + r = lint("| a | prevent | | no ticket | |\n") + self.assertEqual(r.returncode, 1) + for w in ("owner missing", "tracking id missing", "verifiable end state missing"): + self.assertIn(w, r.stdout) + + def test_placeholder_values_fail_and_blank_template_row_is_skipped(self): + self.assertEqual(lint("| a | p | TBD | #1 | check passes |\n").returncode, 1) + self.assertEqual(lint("| | | | | |\n| a | p | jane | ABC-7 | test green |\n").returncode, 0) + + def test_header_names_match_whole_not_substring(self): + # "Provider" contains "id", "Ownership" contains "owner": neither is the tracking/owner column. + r = lint("| a | b | c | d | e |\n", "| Provider | Ownership | Verified by | Action | Notes |\n|---|---|---|---|---|\n") + self.assertNotEqual(r.returncode, 0) + self.assertIn("no table", r.stdout) + r = lint("| a | jane | #4 | ok at drill |\n", "| Action | Owner | Bug/issue | Done when (verifiable) |\n|---|---|---|---|\n") + self.assertEqual(r.returncode, 0) + + def test_url_counts_as_id(self): + self.assertEqual(lint("| a | p | jane | https://example.com/i/3 | done |\n").returncode, 0) + + def test_table_without_the_columns_fails_closed(self): + r = lint("| a | b |\n", head="| Action | Owner |\n|---|---|\n") + self.assertEqual(r.returncode, 2) + + def test_unreadable_file_exit_2(self): + self.assertEqual(subprocess.run([sys.executable, str(SCRIPT), "/nonexistent.md"], capture_output=True).returncode, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_quarantine_ticket_cap.py b/scripts/test_quarantine_ticket_cap.py new file mode 100644 index 000000000..5e8b8681a --- /dev/null +++ b/scripts/test_quarantine_ticket_cap.py @@ -0,0 +1,67 @@ +"""flake_ledger quarantine: each entry carries a ticket and expiry; a full list exits 1 (and can file a P0 issue).""" +import contextlib +import io +import os +import stat +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / ".claude/skills/agentic-delivery/scripts")) +import flake_ledger as fl # noqa: E402 + + +class T(unittest.TestCase): + def setUp(self): + self._t = tempfile.TemporaryDirectory() + self.d = self._t.name + self.addCleanup(self._t.cleanup) + for t in ("t1", "t2", "t3"): + self.cli("record", "--tree", t, stdin="".join(f"FAIL x{i}\nPASS x{i}\n" for i in range(3))) + + def cli(self, *args, stdin=""): + out = io.StringIO() + with mock.patch("sys.stdin", io.StringIO(stdin)), contextlib.redirect_stdout(out): + rc = fl.main(["--dir", self.d, "--today", "2026-10-16", *args]) + return rc, out.getvalue() + + def test_ticket_is_stored_printed_and_missing_ticket_flagged(self): + rc, out = self.cli("quarantine", "--ticket", "T-9") + self.assertEqual(rc, 0) + self.assertIn("QUARANTINE x0 expires=2026-10-30 ticket=T-9", out) + self.assertNotIn("NO-TICKET", self.cli("report")[1]) + self.assertIn("expires=2026-10-30\tNO-TICKET", self._ticketless()) + + def _ticketless(self): + import json + q = Path(self.d) / "quarantine.json" + data = json.loads(q.read_text()) + data["x0"].pop("ticket") + q.write_text(json.dumps(data)) + return self.cli("report")[1] + + def test_full_list_exits_1_and_names_the_cap(self): + rc, out = self.cli("quarantine", "--max", "2") + self.assertEqual(rc, 1) + self.assertIn("FULL quarantine list: 3 > 2", out) + + def test_full_list_files_p0_issue_when_asked(self): + bindir = tempfile.mkdtemp() + log = os.path.join(bindir, "args") + gh = Path(bindir, "gh") + gh.write_text(f'#!/bin/sh\necho "$@" > {log}\necho https://example.com/issues/1\n') + gh.chmod(gh.stat().st_mode | stat.S_IXUSR) + with mock.patch.dict(os.environ, {"PATH": f"{bindir}:{os.environ['PATH']}"}): + rc, out = self.cli("quarantine", "--max", "2", "--file-issue") + self.assertEqual(rc, 1) + self.assertIn("https://example.com/issues/1", out) + self.assertIn("--label P0", Path(log).read_text()) + + def test_under_cap_exit_0(self): + self.assertEqual(self.cli("quarantine", "--max", "3")[0], 0) + + +if __name__ == "__main__": + unittest.main()