From 057926383b63a8c2eb683966f13dba72d269dafd Mon Sep 17 00:00:00 2001 From: Zachary Allen <93971118+zacha0dev@users.noreply.github.com> Date: Sat, 26 Sep 2026 18:28:15 -0400 Subject: [PATCH] v2 review pass: commit the guard hooks, make the demo honest (v0.2.0) - add tools/hooks/guard.py (DENY[0] force-push, [1] reset --hard, [2] test deselection, [3] outbound sends) for Claude Code and Copilot, .github/hooks/guard.json, and 43 hook tests; CI was failing without them - settings.json: python + $CLAUDE_PROJECT_DIR paths, Bash|PowerShell matcher - Stop hook uses decision/reason and cannot loop - validator: pathlib + UTF-8 (Windows crash), checks hook paths, agent names, skill references, flow handoffs and README counts - sync_claude: UTF-8/LF so --check passes on Windows; .gitattributes - lead, quality-auditor, reporter, triager get edit, bounded to their files - triage demo runs --runxfail; report numbers match the real tree - learning example labelled illustrative; "self-improving" claims softened - handoff order triager -> test-writer -> implementer everywhere - /sweep /triage /reinforce for Claude Code and Copilot - deploy/: honest runner placeholder, run-once.sh path and placeholder checks - mcp.json: read-only GitHub endpoint, pinned playwright with named tools - README why + quickstart; llms.txt, CITATION.cff, SECURITY.md, CONTRIBUTING.md; CHANGELOG fleet history Checks: validate_fleet ok; sync --check ok; pytest 49 passed, 1 xfailed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FYwugpFSkj3pMYZQ15vrMD --- .claude/agents/changelog-keeper.md | 2 +- .claude/agents/lead.md | 3 +- .claude/agents/quality-auditor.md | 3 +- .claude/agents/reporter.md | 3 +- .claude/agents/triager.md | 6 +- .claude/commands/reinforce.md | 6 + .claude/commands/sweep.md | 6 + .claude/commands/triage.md | 9 + .claude/settings.json | 6 +- .claude/skills/score-session/SKILL.md | 4 +- .claude/skills/triage-issue/SKILL.md | 4 +- .gitattributes | 3 + .github/agents/changelog-keeper.agent.md | 2 +- .github/agents/lead.agent.md | 3 +- .github/agents/quality-auditor.agent.md | 3 +- .github/agents/reporter.agent.md | 3 +- .github/agents/triager.agent.md | 6 +- .github/copilot-instructions.md | 1 + .github/hooks/guard.json | 14 ++ .github/mcp.json | 14 +- .github/prompts/reinforce.prompt.md | 5 + .github/prompts/triage.prompt.md | 2 +- CHANGELOG.md | 49 +++++- CITATION.cff | 18 ++ CONTRIBUTING.md | 19 +++ README.md | 71 +++++--- SECURITY.md | 9 + deploy/Dockerfile | 17 +- deploy/README.md | 15 +- deploy/mcp-hosted.json | 4 +- deploy/run-once.sh | 55 ++++-- docs/hosted-runners.md | 8 +- fleet/board.md | 2 +- fleet/corpus/known-issues.md | 2 +- fleet/evals/cases/triage-ki-1.md | 11 +- fleet/evals/rubric.md | 2 + fleet/evals/scorecards.md | 8 +- fleet/learning/example-reinforcement.md | 4 +- flows/README.md | 2 +- llms.txt | 31 ++++ reports/2026-09-21-triager-ki-1.md | 4 +- tests/test_guard.py | 96 +++++++++++ tests/test_require_report.py | 32 ++++ tools/hooks/guard.py | 208 +++++++++++++++++++++++ tools/hooks/require_report.py | 45 +++-- tools/runner.py | 54 ++++++ tools/sync_claude.py | 17 +- tools/validate_fleet.py | 180 +++++++++++++++----- 48 files changed, 913 insertions(+), 158 deletions(-) create mode 100644 .claude/commands/reinforce.md create mode 100644 .claude/commands/sweep.md create mode 100644 .claude/commands/triage.md create mode 100644 .gitattributes create mode 100644 .github/hooks/guard.json create mode 100644 .github/prompts/reinforce.prompt.md create mode 100644 CITATION.cff create mode 100644 CONTRIBUTING.md create mode 100644 SECURITY.md create mode 100644 llms.txt create mode 100644 tests/test_guard.py create mode 100644 tests/test_require_report.py create mode 100644 tools/hooks/guard.py create mode 100644 tools/runner.py diff --git a/.claude/agents/changelog-keeper.md b/.claude/agents/changelog-keeper.md index 13955f0..dce1af8 100644 --- a/.claude/agents/changelog-keeper.md +++ b/.claude/agents/changelog-keeper.md @@ -7,7 +7,7 @@ tools: Read, Edit, Write # changelog-keeper — Knowledge ## Do -- Add one line under `## [Unreleased]` in `CHANGELOG.md`, in the right section: Added / Changed / Fixed / Removed. +- Add one line under `[Unreleased]` in the part of `CHANGELOG.md` the change belongs to (Fleet or Sample application), in the right section: Added / Changed / Fixed / Removed. - Write it for a user, not a developer: what they can now do or no longer hit. ## Bounds diff --git a/.claude/agents/lead.md b/.claude/agents/lead.md index bcca7a4..8a0e819 100644 --- a/.claude/agents/lead.md +++ b/.claude/agents/lead.md @@ -1,7 +1,7 @@ --- name: lead description: Routes incoming work to the right agent, runs the master sweep, and keeps one next action visible per item. Use for any request that does not name an agent. -tools: Read, Grep, Glob, Task +tools: Read, Grep, Glob, Edit, Write, Task --- # lead — Orchestration @@ -14,6 +14,7 @@ You are the lead. You do not do the work yourself; you decide who does and keep - Every item ends with exactly one next action and who owns it (an agent or the human). ## Bounds +- Edit only `fleet/board.md`. Code, tests and fleet files are other agents' work. - Never merge, release, or send anything. Those are human gates. - If two agents disagree, surface both positions with evidence; do not pick silently. - If a tool you need is unavailable, say which tool and which call failed. Do not describe a permission you did not measure. diff --git a/.claude/agents/quality-auditor.md b/.claude/agents/quality-auditor.md index 643c194..24efd61 100644 --- a/.claude/agents/quality-auditor.md +++ b/.claude/agents/quality-auditor.md @@ -1,7 +1,7 @@ --- name: quality-auditor description: Scores a finished session against the fleet's rubric and files the score. Use after any session report, and on a schedule. -tools: Read, Grep, Glob +tools: Read, Grep, Glob, Edit, Write --- # quality-auditor — Quality + Learning @@ -14,4 +14,5 @@ Load the `score-session` skill. - Append the score to `fleet/evals/scorecards.md`. ## Bounds +- Edit only `fleet/evals/scorecards.md`, and only by appending a row. - Score the artifacts, not the narration. A claim without an artifact scores 0 on that line. diff --git a/.claude/agents/reporter.md b/.claude/agents/reporter.md index 41fe750..63ac697 100644 --- a/.claude/agents/reporter.md +++ b/.claude/agents/reporter.md @@ -1,7 +1,7 @@ --- name: reporter description: Writes the end-of-session report in the fixed format: what changed, what was touched, what is next. Use at the end of every session and for status questions. -tools: Read, Grep, Glob +tools: Read, Grep, Glob, Edit, Write --- # reporter — Orchestration @@ -19,5 +19,6 @@ You write one report per session, in this exact shape, and nothing else: Load the `session-report` skill for the rules. A report with "probably" or "should" in it is not finished. ## Bounds +- Edit only `reports/`. Write the report as `reports/YYYY-MM-DD--.md`. - You report; you do not act. If the session left something undone, it goes under **Next action**, not into a fix. - No adjectives. "Changed" lists artifacts; "Blocked" lists errors. diff --git a/.claude/agents/triager.md b/.claude/agents/triager.md index 005e5d7..de64856 100644 --- a/.claude/agents/triager.md +++ b/.claude/agents/triager.md @@ -1,7 +1,7 @@ --- name: triager description: Reproduces a reported problem, classifies it, and hands it on with evidence. Use for bug reports, failing tests, and 'something is wrong' requests. -tools: Read, Grep, Glob, Bash +tools: Read, Grep, Glob, Edit, Write, Bash --- # triager — Delivery @@ -12,8 +12,8 @@ Load the `triage-issue` skill. - Reproduce first. Run the failing test or the reported command and paste the actual output. - Classify: bug / regression / expectation mismatch / environment. One label. - Locate: the smallest file and function the evidence points at. Cite `path:line`. -- Hand on: to `implementer` with the reproduction, the classification, and the location. +- Hand on: to `test-writer` with the reproduction, the classification, and the location, so the failure is pinned as a test before `implementer` changes code. An `expectation` goes to `docs-writer` instead. ## Bounds -- No fix. If you find yourself editing code, stop and hand on. +- Edit only `reports/`, for the session report. No fix: if you find yourself editing code or tests, stop and hand on. - No diagnosis without a reproduction. "Likely" is not a classification. diff --git a/.claude/commands/reinforce.md b/.claude/commands/reinforce.md new file mode 100644 index 0000000..d4c668f --- /dev/null +++ b/.claude/commands/reinforce.md @@ -0,0 +1,6 @@ +--- +description: Find the pattern that repeated across recent sessions and propose one reinforcement. +--- +Use the `pattern-miner` subagent. Run the `reinforce-pattern` skill. Read the last ten session reports in `reports/` and `fleet/evals/scorecards.md`. Name the one pattern that appears at least twice, cite both occurrences, and say which single file it belongs in. Hand the proposal to `skill-smith`, who opens it as a pull request titled `reinforce: `. If nothing repeats, say so and stop. + +Mirrors `.github/prompts/reinforce.prompt.md`. diff --git a/.claude/commands/sweep.md b/.claude/commands/sweep.md new file mode 100644 index 0000000..fb515e4 --- /dev/null +++ b/.claude/commands/sweep.md @@ -0,0 +1,6 @@ +--- +description: Run the master sweep across every open item and rebuild the board. +--- +Use the `lead` subagent. Run the `master-sweep` skill: dispatch one agent per open item in parallel, collect every session report, and rewrite `fleet/board.md`. End with the one thing a human should do next. + +Mirrors `.github/prompts/sweep.prompt.md`. diff --git a/.claude/commands/triage.md b/.claude/commands/triage.md new file mode 100644 index 0000000..6a35527 --- /dev/null +++ b/.claude/commands/triage.md @@ -0,0 +1,9 @@ +--- +description: Reproduce, classify, and locate a reported problem. +argument-hint: +--- +Use the `triager` subagent. Triage this: $ARGUMENTS + +Reproduce it first and paste the actual output. Classify it. Locate it as path:line. Hand on to `test-writer` with a session report in `reports/`. + +Mirrors `.github/prompts/triage.prompt.md`. diff --git a/.claude/settings.json b/.claude/settings.json index a33383c..5cfd46c 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -2,11 +2,11 @@ "hooks": { "PreToolUse": [ { - "matcher": "Bash|Write|Edit", + "matcher": "Bash|PowerShell", "hooks": [ { "type": "command", - "command": "python3 tools/hooks/guard.py" + "command": "python \"$CLAUDE_PROJECT_DIR\"/tools/hooks/guard.py" } ] } @@ -16,7 +16,7 @@ "hooks": [ { "type": "command", - "command": "python3 tools/hooks/require_report.py" + "command": "python \"$CLAUDE_PROJECT_DIR\"/tools/hooks/require_report.py" } ] } diff --git a/.claude/skills/score-session/SKILL.md b/.claude/skills/score-session/SKILL.md index b442a6d..a42c0a8 100644 --- a/.claude/skills/score-session/SKILL.md +++ b/.claude/skills/score-session/SKILL.md @@ -4,7 +4,7 @@ description: Score a finished session against the rubric and record it. Use afte --- # Score a session -Each line 0 (absent), 1 (partial), 2 (present). Cite the artifact for every 2. +Each line 0 (absent), 1 (partial), 2 (present). Cite the artifact for every 2. A line that does not apply is `—` and lowers the maximum by 2. | # | Rubric line | Evidence to look for | |---|-------------|----------------------| @@ -18,7 +18,7 @@ Each line 0 (absent), 1 (partial), 2 (present). Cite the artifact for every 2. Append to `fleet/evals/scorecards.md`: ``` -| | | | 1:_ 2:_ 3:_ 4:_ 5:_ 6:_ | /12 | | +| | | | 1:_ 2:_ 3:_ 4:_ 5:_ 6:_ | / | | ``` ## Rules diff --git a/.claude/skills/triage-issue/SKILL.md b/.claude/skills/triage-issue/SKILL.md index 71564db..c068960 100644 --- a/.claude/skills/triage-issue/SKILL.md +++ b/.claude/skills/triage-issue/SKILL.md @@ -5,10 +5,10 @@ description: Reproduce, classify, and locate a reported problem before anyone fi # Triage an issue ## Steps -1. **Reproduce.** Run the exact command or test from the report. Paste the output verbatim. No reproduction, no triage. +1. **Reproduce.** Run the exact command or test from the report. Paste the output verbatim. No reproduction, no triage. If the test is marked `xfail`, run it with `--runxfail` so the real failure shows. 2. **Classify.** One of: `bug` (wrong output), `regression` (was right before — name the commit), `expectation` (the report expects behavior the code never promised), `environment` (only fails here). 3. **Locate.** The smallest function the evidence points to, as `path:line`. Read it. Say in one sentence why the evidence points there. -4. **Hand on.** To `implementer` (bug, regression) or `docs-writer` (expectation) with steps 1–3 attached. +4. **Hand on.** To `test-writer` (bug, regression), who pins the failure as a test before `implementer` changes code, or to `docs-writer` (expectation). Attach steps 1–3. ## Rules - Do not fix. Triage that edits code has become implementation without a test. diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..46526f6 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +* text=auto eol=lf +*.png binary +*.jpg binary diff --git a/.github/agents/changelog-keeper.agent.md b/.github/agents/changelog-keeper.agent.md index 101d811..435fddf 100644 --- a/.github/agents/changelog-keeper.agent.md +++ b/.github/agents/changelog-keeper.agent.md @@ -6,7 +6,7 @@ tools: ["read", "edit"] # changelog-keeper — Knowledge ## Do -- Add one line under `## [Unreleased]` in `CHANGELOG.md`, in the right section: Added / Changed / Fixed / Removed. +- Add one line under `[Unreleased]` in the part of `CHANGELOG.md` the change belongs to (Fleet or Sample application), in the right section: Added / Changed / Fixed / Removed. - Write it for a user, not a developer: what they can now do or no longer hit. ## Bounds diff --git a/.github/agents/lead.agent.md b/.github/agents/lead.agent.md index c9d5ab5..f4b4a3f 100644 --- a/.github/agents/lead.agent.md +++ b/.github/agents/lead.agent.md @@ -1,7 +1,7 @@ --- name: lead description: Routes incoming work to the right agent, runs the master sweep, and keeps one next action visible per item. Use for any request that does not name an agent. -tools: ["read", "search", "agent"] +tools: ["read", "search", "edit", "agent"] --- # lead — Orchestration @@ -13,6 +13,7 @@ You are the lead. You do not do the work yourself; you decide who does and keep - Every item ends with exactly one next action and who owns it (an agent or the human). ## Bounds +- Edit only `fleet/board.md`. Code, tests and fleet files are other agents' work. - Never merge, release, or send anything. Those are human gates. - If two agents disagree, surface both positions with evidence; do not pick silently. - If a tool you need is unavailable, say which tool and which call failed. Do not describe a permission you did not measure. diff --git a/.github/agents/quality-auditor.agent.md b/.github/agents/quality-auditor.agent.md index 4aba4f2..737f344 100644 --- a/.github/agents/quality-auditor.agent.md +++ b/.github/agents/quality-auditor.agent.md @@ -1,7 +1,7 @@ --- name: quality-auditor description: Scores a finished session against the fleet's rubric and files the score. Use after any session report, and on a schedule. -tools: ["read", "search"] +tools: ["read", "search", "edit"] --- # quality-auditor — Quality + Learning @@ -13,4 +13,5 @@ Load the `score-session` skill. - Append the score to `fleet/evals/scorecards.md`. ## Bounds +- Edit only `fleet/evals/scorecards.md`, and only by appending a row. - Score the artifacts, not the narration. A claim without an artifact scores 0 on that line. diff --git a/.github/agents/reporter.agent.md b/.github/agents/reporter.agent.md index dbbca79..fcb6c7e 100644 --- a/.github/agents/reporter.agent.md +++ b/.github/agents/reporter.agent.md @@ -1,7 +1,7 @@ --- name: reporter description: Writes the end-of-session report in the fixed format: what changed, what was touched, what is next. Use at the end of every session and for status questions. -tools: ["read", "search"] +tools: ["read", "search", "edit"] --- # reporter — Orchestration @@ -18,5 +18,6 @@ You write one report per session, in this exact shape, and nothing else: Load the `session-report` skill for the rules. A report with "probably" or "should" in it is not finished. ## Bounds +- Edit only `reports/`. Write the report as `reports/YYYY-MM-DD--.md`. - You report; you do not act. If the session left something undone, it goes under **Next action**, not into a fix. - No adjectives. "Changed" lists artifacts; "Blocked" lists errors. diff --git a/.github/agents/triager.agent.md b/.github/agents/triager.agent.md index 964042a..5afc5f0 100644 --- a/.github/agents/triager.agent.md +++ b/.github/agents/triager.agent.md @@ -1,7 +1,7 @@ --- name: triager description: Reproduces a reported problem, classifies it, and hands it on with evidence. Use for bug reports, failing tests, and 'something is wrong' requests. -tools: ["read", "search", "execute"] +tools: ["read", "search", "edit", "execute"] --- # triager — Delivery @@ -11,8 +11,8 @@ Load the `triage-issue` skill. - Reproduce first. Run the failing test or the reported command and paste the actual output. - Classify: bug / regression / expectation mismatch / environment. One label. - Locate: the smallest file and function the evidence points at. Cite `path:line`. -- Hand on: to `implementer` with the reproduction, the classification, and the location. +- Hand on: to `test-writer` with the reproduction, the classification, and the location, so the failure is pinned as a test before `implementer` changes code. An `expectation` goes to `docs-writer` instead. ## Bounds -- No fix. If you find yourself editing code, stop and hand on. +- Edit only `reports/`, for the session report. No fix: if you find yourself editing code or tests, stop and hand on. - No diagnosis without a reproduction. "Likely" is not a classification. diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 9f0c087..5db6161 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -19,4 +19,5 @@ This repository is a small agent fleet and the sample application it works on. R - Skills: `.claude/skills//SKILL.md` (read by both Copilot and Claude Code) - Path rules: `.github/instructions/*.instructions.md` - Hooks: `.github/hooks/*.json` (Copilot) and `.claude/settings.json` (Claude Code) — both call `tools/hooks/guard.py` +- Slash prompts: `.github/prompts/*.prompt.md` (Copilot Chat) and `.claude/commands/*.md` (Claude Code), kept in pairs - Corpus, registers, evals, board: `fleet/` diff --git a/.github/hooks/guard.json b/.github/hooks/guard.json new file mode 100644 index 0000000..fdfd3c2 --- /dev/null +++ b/.github/hooks/guard.json @@ -0,0 +1,14 @@ +{ + "version": 1, + "hooks": { + "preToolUse": [ + { + "type": "command", + "bash": "python3 tools/hooks/guard.py", + "powershell": "python tools/hooks/guard.py", + "cwd": ".", + "timeoutSec": 10 + } + ] + } +} diff --git a/.github/mcp.json b/.github/mcp.json index 226e6bf..e249b0e 100644 --- a/.github/mcp.json +++ b/.github/mcp.json @@ -2,7 +2,7 @@ "mcpServers": { "github": { "type": "http", - "url": "https://api.githubcopilot.com/mcp/", + "url": "https://api.githubcopilot.com/mcp/readonly", "tools": [ "*" ] @@ -11,10 +11,18 @@ "type": "local", "command": "npx", "args": [ - "@playwright/mcp@latest" + "@playwright/mcp@0.0.82" ], "tools": [ - "*" + "browser_navigate", + "browser_snapshot", + "browser_take_screenshot", + "browser_console_messages", + "browser_network_requests", + "browser_click", + "browser_type", + "browser_wait_for", + "browser_close" ] } } diff --git a/.github/prompts/reinforce.prompt.md b/.github/prompts/reinforce.prompt.md new file mode 100644 index 0000000..f4ac3e2 --- /dev/null +++ b/.github/prompts/reinforce.prompt.md @@ -0,0 +1,5 @@ +--- +description: Find the pattern that repeated across recent sessions and propose one reinforcement. +agent: pattern-miner +--- +Run the `reinforce-pattern` skill. Read the last ten session reports in `reports/` and `fleet/evals/scorecards.md`. Name the one pattern that appears at least twice, cite both occurrences, and say which single file it belongs in. Hand the proposal to `skill-smith`, who opens it as a pull request titled `reinforce: `. If nothing repeats, say so and stop. diff --git a/.github/prompts/triage.prompt.md b/.github/prompts/triage.prompt.md index 84fcbe3..bc59d4a 100644 --- a/.github/prompts/triage.prompt.md +++ b/.github/prompts/triage.prompt.md @@ -2,4 +2,4 @@ description: Reproduce, classify, and locate a reported problem. agent: triager --- -Triage this: ${input:report:Paste the bug report, failing test name, or command}. Reproduce it first and paste the actual output. Classify it. Locate it as path:line. Hand on with a session report. +Triage this: ${input:report:Paste the bug report, failing test name, or command}. Reproduce it first and paste the actual output. Classify it. Locate it as path:line. Hand on to `test-writer` with a session report in `reports/`. diff --git a/CHANGELOG.md b/CHANGELOG.md index e660b38..e015a84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,49 @@ # Changelog -All notable changes to the sample application. Format: Keep a Changelog. Versioning: semantic. +Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Versioning: semantic. -## [Unreleased] -### Fixed -- (pending) KI-1 — percent discounts were applied as flat amounts. +Two things are versioned here: the **fleet** (the agents, skills, flows, hooks and tools in this repository) and the **sample application** the fleet works on. Each has its own `[Unreleased]` block. -## [0.1.0] — 2026-09-21 -### Added +## Fleet + +### [Unreleased] + +### [0.2.0] — 2026-09-26 +#### Added +- `tools/hooks/guard.py`, the PreToolUse guard the README describes: denies force-push (DENY[0]), `reset --hard` (DENY[1]), test deselection (DENY[2]) and outbound sends (DENY[3]). Reads both Claude Code and GitHub Copilot hook input. +- `.github/hooks/guard.json`, the Copilot hook configuration that calls it. +- `tests/test_guard.py` and `tests/test_require_report.py`. +- `tools/runner.py`, the container entry point. It prints the plan for one run and exits 2, because the image has no agent CLI yet. +- Claude Code commands `/sweep`, `/triage` and `/reinforce` in `.claude/commands/`, and the Copilot prompt `.github/prompts/reinforce.prompt.md`. +- Validator checks: agent name matches filename, loaded skills exist, flow steps and handoffs resolve, `.claude/settings.json` hook scripts exist, each prompt has a matching command, README agent and skill counts match the tree. +- `.gitattributes`, `llms.txt`, `CITATION.cff`, `SECURITY.md`, `CONTRIBUTING.md`. +#### Changed +- Handoff order is `triager` → `test-writer` → `implementer` everywhere (agent, skill, eval case, report, board). +- `lead`, `quality-auditor`, `reporter` and `triager` have the `edit` tool, each bounded to the one file or folder it writes. +- The Stop hook uses the Claude Code contract: blocks once with a reason when no report is dated today, otherwise silent. +- `.claude/settings.json` calls `python "$CLAUDE_PROJECT_DIR"/tools/hooks/...`, so hooks run from any working directory on Windows and Linux. +- `.github/mcp.json`: GitHub server uses the read-only endpoint; `@playwright/mcp` pinned to 0.0.82 with a named tool list. +- `deploy/run-once.sh` runs from the repo root, names runs like `reports/`, and refuses to run with unset or `REPLACE_ME` values. +- The KI-1 demo tells the truth about the `xfail` marker: triage uses `--runxfail` to see the failure. +- The worked reinforcement example and the seed scorecard rows are labelled as illustrations. +#### Fixed +- CI failed on `main`: the validator required `.github/hooks/` files and `guard.py`, which were missing. +- `tools/validate_fleet.py` crashed on Windows paths; `tools/sync_claude.py --check` reported drift on Windows because of CRLF and the default code page. + +### [0.1.0] — 2026-09-21 +- `3be9cac` Reference layout: agents, skills, instructions, corpus, evals, sample app. +- `aa62079` Flows, the fourth part of the model. +- `3ae380f` The deploy template and the hosted-runners design. +- `1367a65` README rewritten around agents, skills, flows and the orchestrator. +- `d6a2100` The validator fails when the hook files are missing. + +## Sample application + +### [Unreleased] + +### [0.1.0] — 2026-09-21 +#### Added - Pricing with flat and percent discounts. - Inventory with reservations. + +Known issue KI-1 (percent discounts applied as flat amounts) is open on purpose for the triage demo; see `fleet/corpus/known-issues.md`. diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000..10b8272 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,18 @@ +cff-version: 1.2.0 +message: "If you use or adapt this layout, please cite it as below." +title: "fleet-kit: a reference layout for a bounded agent fleet for Claude Code and GitHub Copilot" +type: software +authors: + - given-names: Zachary + family-names: Allen + alias: zacha0dev +version: 0.2.0 +date-released: 2026-09-26 +license: MIT +repository-code: "https://github.com/zacha0dev/fleet-kit" +keywords: + - agents + - claude-code + - github-copilot + - mcp + - hooks diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..abf05c6 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,19 @@ +# Contributing + +Issues and pull requests are welcome. + +Before opening a pull request, run the same three checks CI runs: + +```bash +python tools/validate_fleet.py +python tools/sync_claude.py --check +python -m pytest -q +``` + +- **Agents** are edited in `.github/agents/`. Run `python tools/sync_claude.py` afterwards; never edit `.claude/agents/` by hand. +- **Fleet changes** (agents, skills, instructions, hooks, flows) use a title starting `fleet:` or `reinforce:`, as `.github/copilot-instructions.md` asks. +- **Tests first.** A change to `sample_app/` or `tools/` comes with a test in `tests/`. Do not skip or deselect tests to get green. +- **Claims.** Say what you ran. A statement about behavior outside the repo names the command that shows it. +- **KI-1** is a seeded bug kept open for the triage demo. Please do not send a fix for it. + +By contributing you agree your contribution is released under the MIT license in `LICENSE`. diff --git a/README.md b/README.md index cc86955..8fe1dfc 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,30 @@ # fleet-kit -**A reference layout for a bounded, self-improving agent fleet.** One source of truth, loaded by both GitHub Copilot and Claude Code. +**A reference layout for a bounded agent fleet, designed to improve from scored sessions.** One source of truth, loaded by both GitHub Copilot and Claude Code. + +**Why.** For people who use Copilot or Claude Code and want multi-agent work they can repeat and check. The problem it addresses: agents that forget what the last session learned, claim things they did not measure, or act without limits. Here every role, procedure and limit is a file, every session leaves a report, and a hook refuses the few actions that must never happen. Eighteen agents, ten skills, flows that chain them, hooks that hold whatever the prompt says, and a small sample project for the fleet to work on. No custom runtime — every piece is a documented extension point of tools you already have. +## Quickstart (about five minutes) + +```bash +git clone https://github.com/zacha0dev/fleet-kit && cd fleet-kit +pip install pytest +python tools/validate_fleet.py # ok - 18 agents, 10 skills, 4 flows, hooks, prompts and instructions valid +python tools/sync_claude.py --check # ok - the .claude/agents mirror is current +python -m pytest -q # 49 passed, 1 xfailed +python -m pytest -q --runxfail # 1 failed, 49 passed - the seeded bug, KI-1 +``` + +Then open the folder in Claude Code and run `/triage KI-1`, or in VS Code Copilot Chat run the `/triage` prompt with `KI-1`. The triager reproduces the failure, classifies it, locates it at `sample_app/pricing.py:22`, writes a report to `reports/`, and hands on to `test-writer`. It does not fix it. + ## The model, in four parts | | What it is | Where it lives | |---|---|---| | **Agent** | A role. One file: what it is for, which skills it may call, what it must never do. | `.github/agents/*.agent.md` | -| **Skill** | A procedure. The steps, the tool, and what finished looks like. An agent holds one or several. | `.claude/skills//SKILL.md` | +| **Skill** | A procedure. The steps, the tool, and what finished looks like. An agent loads a skill when its role has a written procedure; some roles need none. | `.claude/skills//SKILL.md` | | **Flow** | A path that worked, written down. Agents and skills in the order that gets a job done. Flows chain. | `flows/*.md` | | **Orchestrator** | The agent above the rest. It routes each task and returns finished work to you. | `.github/agents/lead.agent.md` | @@ -18,10 +33,15 @@ Eighteen agents, ten skills, flows that chain them, hooks that hold whatever the No agent carries a whole job. Each one does its part and hands off to the role that comes next. ``` -you ──▶ orchestrator ──▶ triager ──▶ test-writer ──▶ implementer ──▶ reviewer ──▶ you - └── every handoff leaves a written artifact ──┘ +you ──▶ lead ──▶ triager ──▶ test-writer ──▶ implementer ──▶ reviewer ──▶ you + │ (a problem) ▲ + └──▶ planner ──────────┘ + (a change to make) + every handoff leaves a written artifact ``` +`lead` is the orchestrator. A report of something wrong goes to `triager`; a request for a change goes to `planner`, whose plan names the tests `test-writer` writes first. + **Every handoff is written down, not held in context.** The triager leaves a reproduction. The test-writer leaves a failing test. The implementer leaves a diff and a test run. Each agent reads what the last one wrote. That is what makes the fleet debuggable. When a result is wrong, the step that produced it is a file you can open — not a transcript you have to scroll. @@ -30,10 +50,12 @@ That is what makes the fleet debuggable. When a result is wrong, the step that p One prompt sweeps everything open and dispatches one agent per item. You read the reports instead of opening the items. -```bash -sweep # lead dispatches, collects, and rebuilds fleet/board.md +``` +/sweep # lead dispatches, collects, and rebuilds fleet/board.md ``` +`/sweep`, `/triage` and `/reinforce` are slash commands, not shell commands: Claude Code reads them from `.claude/commands/`, and VS Code Copilot Chat reads the same prompts from `.github/prompts/`. + Every session ends with a report in one fixed shape. A session without a report is a failed session, not a quiet one. ## The guardrails @@ -41,50 +63,51 @@ Every session ends with a report in one fixed shape. A session without a report These hold regardless of what a prompt asks for. - **Tools allowlist.** Each agent declares its tools in its own file. It cannot reach one it was not given. -- **Hooks.** `tools/hooks/guard.py` runs before the tool does — called by both `.github/hooks/guard.json` and `.claude/settings.json`. It denies force-pushes, `reset --hard`, test deselection and outbound sends. +- **Hooks.** `tools/hooks/guard.py` runs before a shell command does — called by both `.github/hooks/guard.json` (Copilot) and `.claude/settings.json` (Claude Code). It denies force-pushes, `reset --hard`, test deselection and outbound sends, and `tests/test_guard.py` checks each rule. It matches command lines, so it stops the ordinary way of doing each thing, not every possible workaround. - **Drafts only.** Anything addressed to a person outside the repo goes to `drafts/` for a human to send. Reading is never restricted by this. - **Human gates.** Merge, tag, release, publish. Agents prepare the commands; people run them. - **Fleet changes are pull requests.** Checked in CI. -## It gets better with use +## How it is meant to improve Reports get scored. Repeats get found. The repeat gets written into the agent that should have known it — as a pull request. The next run starts from the better version. -A worked example is in [`fleet/learning/`](fleet/learning/example-reinforcement.md). +That loop is designed, not yet demonstrated: the scorecards hold seed rows, and the example in [`fleet/learning/`](fleet/learning/example-reinforcement.md) is illustrative, not a recorded run. `/reinforce` starts it. ## Try it -```bash -pip install pytest -python tools/validate_fleet.py # 18 agents, 10 skills, hooks, instructions -python tools/sync_claude.py --check # the .claude/agents mirror is current -python -m pytest -q # 6 passed, 1 xfailed — the seeded bug -``` +The seeded bug, KI-1, is a real failing test marked `xfail(strict=True)` so CI stays green. A plain `pytest` run reports it as `xfailed`; `--runxfail` shows the failure the triager works from. -Then, from the repo root in Copilot CLI or Claude Code: +- `/triage KI-1` — the triager reproduces the bug, classifies it, locates it, and hands on to `test-writer`. It does not fix it. That path is [`flows/triage-a-failing-test.md`](flows/triage-a-failing-test.md). +- `/sweep` — the orchestrator rebuilds the board. See [`flows/sweep-the-board.md`](flows/sweep-the-board.md). +- `/reinforce` — the pattern-miner proposes one change and the skill-smith opens it as a PR. -- `triage: tests are failing on pricing` — the triager reproduces the seeded bug, classifies it, locates it, and hands on. It does not fix it. That path is [`flows/triage-a-failing-test.md`](flows/triage-a-failing-test.md). -- `sweep` — the orchestrator rebuilds the board. See [`flows/sweep-the-board.md`](flows/sweep-the-board.md). -- `reinforce` — the pattern-miner proposes one change and the skill-smith opens it as a PR. +In Copilot CLI, which reads the agents but not these prompt files, ask for the agent by name: "use the triager agent to triage KI-1". ## Layout ``` .github/agents/ 18 agents — canonical .github/instructions/ path-scoped rules -.github/hooks/ deny rules, enforced before the tool runs +.github/hooks/ Copilot hook config, calls tools/hooks/guard.py +.github/prompts/ /sweep · /triage · /reinforce for Copilot Chat +.github/mcp.json MCP servers for Copilot CLI (see below) .claude/agents/ generated mirror (do not edit) +.claude/commands/ /sweep · /triage · /reinforce for Claude Code +.claude/settings.json Claude Code hooks: guard.py before shell, require_report.py at stop .claude/skills/ 10 skills — both tools read these flows/ paths that worked, chained by handoff fleet/ corpus · capability register · evals · board -deploy/ the v2 template — hosted runners, federated MCP +deploy/ a template for hosted runners (not deployed) docs/hosted-runners.md how this runs without a laptop -tools/ sync · validate · hooks +tools/ sync · validate · hooks · runner stub sample_app/ tests/ the thing the fleet works on ``` `.github/agents/` is canonical; `tools/sync_claude.py` generates the `.claude/` mirror and CI checks it. Skills live once and both tools read them. +`.github/mcp.json` uses the `mcpServers` format that Copilot CLI reads from that path. The Copilot coding agent takes the same JSON pasted into the repository's Copilot settings, not from this file. VS Code reads `.vscode/mcp.json` with a `servers` key, and Claude Code reads `.mcp.json`; neither is included here. The GitHub server points at its read-only endpoint, and `@playwright/mcp` is pinned (0.0.82, the current version on 2026-09-26) with a named tool list. + ## The teams | Team | Agents | @@ -99,7 +122,7 @@ One-liners for each in [`fleet/README.md`](fleet/README.md). ## Running it without a laptop -The agents, skills, flows and limits do not change. What changes is where they run and how each run proves who it is: one container per agent, a private network, the caller's identity forwarded to every source, a log per run. +The agents, skills, flows and limits do not change. What changes is where they run and how each run proves who it is: one container per agent, a private network, the caller's identity forwarded to every source, a log per run. The container's entry point, `tools/runner.py`, prints the run plan and stops: no agent CLI is installed in the image yet. [`deploy/mcp-hosted.json`](deploy/mcp-hosted.json) is the file worth reading — it is where an agent's declared allowlist stops being a suggestion and becomes a network boundary. The reasoning, and what would have to be true before it ships, is in [`docs/hosted-runners.md`](docs/hosted-runners.md). Neither is deployed anywhere today, and both say so. @@ -111,4 +134,4 @@ The agents, skills, flows and limits do not change. What changes is where they r fleet-kit is a personal side project by [@zacha0dev](https://github.com/zacha0dev), published as an open reference for anyone building their own agent fleet. It is not affiliated with, endorsed by, or connected to any employer, and contains no proprietary, customer or production data. The sample project and the agents around it are a generic software development workflow, written from scratch for this repository. -MIT — clone it, strip it, rebuild it for your own domain. +MIT — clone it, strip it, rebuild it for your own domain. See [CONTRIBUTING.md](CONTRIBUTING.md) to send changes and [SECURITY.md](SECURITY.md) to report a problem privately. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..c53e716 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,9 @@ +# Security + +Please report a security problem privately, through GitHub's private vulnerability reporting: the **Security** tab of this repository, then **Report a vulnerability**. Do not open a public issue for it. + +Useful to include: the file and line, what an agent or hook could be made to do, and the steps to show it. + +In scope: anything that lets an agent get past `tools/hooks/guard.py`, the tools allowlist in an agent file, or the drafts-only rule; and anything in `deploy/` that would weaken the identity or network rules it describes. The guard matches command lines and is documented as not stopping every workaround; a bypass is still worth reporting if it looks like an ordinary command. + +This is a personal project maintained in spare time. Reports are read, but there is no response-time commitment. diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 8e75d38..550049f 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -8,13 +8,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends git ca-certific WORKDIR /fleet # The fleet is files. Copy the definitions, not a runtime. -COPY .github/agents/ .github/agents/ -COPY .github/instructions/ .github/instructions/ -COPY .github/hooks/ .github/hooks/ -COPY .claude/skills/ .claude/skills/ +COPY .github/ .github/ +COPY .claude/ .claude/ COPY flows/ flows/ +COPY fleet/ fleet/ COPY tools/ tools/ -COPY AGENTS.md CLAUDE.md ./ +COPY reports/ reports/ +COPY drafts/ drafts/ +COPY sample_app/ sample_app/ +COPY tests/ tests/ +COPY AGENTS.md CLAUDE.md README.md pyproject.toml ./ +COPY deploy/agents.json deploy/agents.json COPY deploy/mcp-hosted.json /fleet/mcp.json RUN pip install --no-cache-dir pytest @@ -27,4 +31,7 @@ ENV FLEET_IDENTITY_MODE="caller" # Fail closed: refuse to start without an agent name, and verify the fleet # definitions are structurally valid before accepting any work. +# tools/runner.py prints the plan for the run and exits 2: this image has no +# agent CLI in it yet. Install one (Claude Code or GitHub Copilot CLI) above and +# replace the last step of tools/runner.py before treating this as a runner. ENTRYPOINT ["/bin/sh", "-c", "test -n \"$FLEET_AGENT\" || { echo 'FLEET_AGENT is not set; refusing to start' >&2; exit 1; }; python tools/validate_fleet.py && exec python -m tools.runner --agent \"$FLEET_AGENT\""] diff --git a/deploy/README.md b/deploy/README.md index eb96763..a9d1729 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,6 +1,9 @@ # deploy — the v2 template -**Status: a template, not a deployment.** Nothing here is running anywhere. These files are the shape a hosted fleet would take, written out so it can be read, argued with and adapted before anyone provisions a thing. Every value you must supply is marked `REPLACE_ME`. +**Status: a template, not a deployment.** + +"v1" in this repository means the fleet as it runs today, inside a Claude Code or Copilot session on a workstation. "v2" means the same files run as hosted containers, which is what this directory sketches. + Nothing here is running anywhere. These files are the shape a hosted fleet would take, written out so it can be read, argued with and adapted before anyone provisions a thing. Every value you must supply is marked `REPLACE_ME`. The design reasoning is in [`docs/hosted-runners.md`](../docs/hosted-runners.md). This directory is the mechanical half. @@ -20,10 +23,10 @@ The agents, skills, flows and limits in this repository do not change. An agent | File | What it is | |---|---| -| `Dockerfile` | One image for every agent. The agent is chosen at start by `FLEET_AGENT`, so eighteen agents are eighteen apps from one build, not eighteen images. | -| `agents.json` | The roster the deployment loops over: which agents get a runner, and how much each may consume. | -| `mcp-hosted.json` | **The centre of this directory.** The v2 MCP configuration: each source reached through its own governed connection, with the caller's identity forwarded per call rather than a stored credential, and each agent seeing only the servers its role needs. | -| `run-once.sh` | What a trigger actually invokes — one agent, one task, one log line. | +| `Dockerfile` | One image for every agent. The agent is chosen at start by `FLEET_AGENT`, so each agent on the roster is one container app from the same build, not its own image. Today its entry point, `tools/runner.py`, prints the run plan and exits 2: no agent CLI is installed in the image. | +| `agents.json` | The roster the deployment loops over: which agents get a runner, and how much each may consume. It lists 8 of the 18 agents; the rest are not expected to need a hosted runner. | +| `mcp-hosted.json` | **The centre of this directory.** The v2 MCP configuration: each source reached through its own governed connection (one connection per source, with its own allowlist of tools, network scope and write permission, so each can be limited and audited on its own), with the caller's identity forwarded per call rather than a stored credential, and each agent seeing only the servers its role needs. | +| `run-once.sh` | What a trigger actually invokes — one agent, one task, one run id. Requires `FLEET_CALLER_ID` and `FLEET_RESOURCE_GROUP`; refuses to run while either is unset or still `REPLACE_ME`. | ## How it fits together @@ -50,7 +53,7 @@ Start by reading `mcp-hosted.json`. It is the architecture; the rest is packagin docker build -t REPLACE_ME/fleet-kit:latest -f deploy/Dockerfile . # Run one agent once, as a named caller -FLEET_CALLER_ID=someone@example.com ./deploy/run-once.sh triager "reproduce the failing pricing test" +FLEET_CALLER_ID=someone@example.com FLEET_RESOURCE_GROUP=my-group ./deploy/run-once.sh triager "reproduce the failing pricing test" ``` The hosting platform is deliberately not prescribed beyond the one `az` line in `run-once.sh`. Azure Container Apps is what this was written against, but nothing in the design needs it — a container per agent, a private network, a per-call identity and a retained log is the whole requirement. diff --git a/deploy/mcp-hosted.json b/deploy/mcp-hosted.json index 20cd07e..307dfee 100644 --- a/deploy/mcp-hosted.json +++ b/deploy/mcp-hosted.json @@ -45,7 +45,7 @@ "identity": "caller", "writes": true, "deny": ["merge_pull_request", "delete_branch", "force_push"], - "$denyMeaning": "Named here as well as in tools/hooks/guard.py. The hook is the enforcement; this list is so a reader of the config knows the rule exists without reading Python." + "$denyMeaning": "force_push is also denied on a workstation by tools/hooks/guard.py (DENY[0]). merge_pull_request and delete_branch are not in the hook: they rely on this list being enforced by whatever hosts the server, and on the human gate." }, "ci": { @@ -77,7 +77,7 @@ }, "perAgent": { - "$comment": "Each agent sees only the servers its role needs. This mirrors the tools allowlist already declared in .github/agents/.agent.md — the agent file is the source of truth, and this is how that declaration becomes a network boundary rather than a suggestion.", + "$comment": "Each agent sees only the servers its role needs. This list is written by hand from each agent's role in .github/agents/.agent.md; it is not generated from, or checked against, the agent's tools line (those name local tools such as read and edit, not MCP servers). Keeping the two in step is a review task. Once enforced by the platform, this is where an agent's reach becomes a network boundary rather than a suggestion.", "lead": ["issues", "ci"], "triager": ["issues", "repo", "ci", "messaging"], "implementer": ["repo", "ci"], diff --git a/deploy/run-once.sh b/deploy/run-once.sh index 6d1d4a9..3ca8852 100755 --- a/deploy/run-once.sh +++ b/deploy/run-once.sh @@ -1,41 +1,60 @@ #!/usr/bin/env bash # Run one agent, once, with a caller identity. This is what a trigger invokes. # -# ./run-once.sh triager "reproduce the failing pricing test" +# FLEET_CALLER_ID=someone@example.com FLEET_RESOURCE_GROUP=my-group \ +# ./deploy/run-once.sh triager "reproduce the failing pricing test" # -# A template: REPLACE_ME values are yours. The point of the script is the -# order of operations, not the CLI it happens to use. +# A template. The point of the script is the order of operations, not the CLI +# it happens to use. It refuses to run while any required value is unset or +# still REPLACE_ME. set -euo pipefail +# Work from the repository root wherever the script is called from. +cd "$(dirname "$0")/.." + AGENT="${1:?usage: run-once.sh }" TASK="${2:?usage: run-once.sh }" -RUN_ID="$(date -u +%Y%m%dT%H%M%SZ)-${AGENT}" # 1. Establish the caller. No identity, no run — this is the fail-closed rule # from mcp-hosted.json, enforced before anything is contacted. -CALLER_ID="${FLEET_CALLER_ID:-}" -if [ -z "$CALLER_ID" ]; then - echo "refusing to run: no caller identity supplied" >&2 - exit 1 -fi +require() { + local name="$1" value="${!1:-}" + if [ -z "$value" ] || [ "$value" = "REPLACE_ME" ]; then + echo "refusing to run: $name is not set" >&2 + exit 1 + fi +} +require FLEET_CALLER_ID +require FLEET_RESOURCE_GROUP # 2. Check the agent is one we actually deploy, so a typo cannot silently # start something arbitrary. -python - "$AGENT" <<'PY' +PYTHON="" +for candidate in python3 python; do + if "$candidate" -c "" >/dev/null 2>&1; then PYTHON="$candidate"; break; fi +done +[ -n "$PYTHON" ] || { echo "refusing to run: no python found" >&2; exit 1; } +"$PYTHON" - "$AGENT" <<'PY' import json, sys -roster = {a["name"] for a in json.load(open("deploy/agents.json"))["agents"]} +roster = {a["name"] for a in json.load(open("deploy/agents.json", encoding="utf-8"))["agents"]} if sys.argv[1] not in roster: sys.exit(f"refusing to run: {sys.argv[1]} is not in deploy/agents.json") PY -# 3. Dispatch to that agent's container app. One agent, one task, one run id. -echo "run ${RUN_ID} · agent ${AGENT} · caller ${CALLER_ID}" +# 3. Name the run and its report the way reports/README.md does: +# YYYY-MM-DD--.md +DATE="$(date -u +%Y-%m-%d)" +SLUG="$(printf '%s' "$TASK" | tr '[:upper:]' '[:lower:]' | tr -cs 'a-z0-9' '-' | sed 's/^-//; s/-$//' | cut -c1-40)" +RUN_ID="${DATE}-${AGENT}-${SLUG:-task}" + +# 4. Dispatch to that agent's container app job. One agent, one task, one run id. +echo "run ${RUN_ID} · agent ${AGENT} · caller ${FLEET_CALLER_ID}" az containerapp job start \ --name "fleet-${AGENT}" \ - --resource-group REPLACE_ME \ - --env-vars "FLEET_AGENT=${AGENT}" "FLEET_CALLER_ID=${CALLER_ID}" \ + --resource-group "${FLEET_RESOURCE_GROUP}" \ + --env-vars "FLEET_AGENT=${AGENT}" "FLEET_CALLER_ID=${FLEET_CALLER_ID}" \ "FLEET_TASK=${TASK}" "FLEET_RUN_ID=${RUN_ID}" -# 4. The run writes its own session report. Nothing is considered finished -# without one — same rule as v1, same fixed shape. -echo "report will be written to reports/${RUN_ID}.md" +# 5. The run writes its own session report. Nothing is considered finished +# without one — same rule as on a workstation, same fixed shape. +echo "report expected at reports/${RUN_ID}.md" diff --git a/docs/hosted-runners.md b/docs/hosted-runners.md index 5af8a05..b76c57d 100644 --- a/docs/hosted-runners.md +++ b/docs/hosted-runners.md @@ -4,7 +4,7 @@ ## What runs today -The fleet runs inside a Copilot session on a workstation. An engineer prompts, the orchestrator routes the task, agents load their skills, skills reach tools through MCP, and the result comes back. Every call runs as the engineer, using the access they already have. +The fleet runs inside a Claude Code or GitHub Copilot session on a workstation. An engineer prompts, the orchestrator routes the task, agents load their skills, skills reach tools through MCP, and the result comes back. Every call runs as the engineer, using the access they already have. Two things follow from that, and both are limits: @@ -19,7 +19,7 @@ The agent files, the skills, the flows and the limits do not change at all. What | | On a workstation | As a hosted runner | |---|---|---| -| Where it runs | one Copilot session | one container per agent | +| Where it runs | one Claude Code or Copilot session | one container per agent | | What starts it | a person typing | a prompt, a schedule, or an event | | How it is reached | locally | over a private network | | Identity | the session's signed-in user | verified by the container on every call | @@ -33,7 +33,7 @@ The agent files, the skills, the flows and the limits do not change at all. What │ ┌──────────────┴──────────────┐ │ container per agent │ - │ scoper · triager · … │ + │ lead · triager · … │ │ each verifies identity │ │ and auth on every call │ └──────────────┬──────────────┘ @@ -48,6 +48,8 @@ The agent files, the skills, the flows and the limits do not change at all. What results and drafts ──▶ engineer ``` +"Federated MCP" here means there is no single gateway holding every credential: each source is its own MCP server, reached through its own connection, with the caller's identity passed through. A "governed connection" is one of those connections with its own tool allowlist, network scope and write permission, so it can be limited and audited on its own. `deploy/mcp-hosted.json` spells both out. + One container per agent, rather than one container running everything, for three reasons: 1. **The tools allowlist becomes a boundary you can see.** An agent that may not run shell does not have shell in its image. diff --git a/fleet/board.md b/fleet/board.md index 3edb119..871401a 100644 --- a/fleet/board.md +++ b/fleet/board.md @@ -4,6 +4,6 @@ Rebuilt by every sweep. One row per open item. | Item | Owner | State | Next action | Evidence | |------|-------|-------|-------------|----------| -| KI-1 percent discount applied as flat amount | `implementer` | triaged | write the fix after `test_percent_discount` is confirmed failing | `fleet/corpus/known-issues.md` | +| KI-1 percent discount applied as flat amount | `test-writer` | triaged | remove the `xfail` marker from `test_percent_discount` on a fix branch so it fails in the normal run | `reports/2026-09-21-triager-ki-1.md` | **Next for a human:** review the KI-1 pull request when it opens. diff --git a/fleet/corpus/known-issues.md b/fleet/corpus/known-issues.md index 91c6787..6227bc1 100644 --- a/fleet/corpus/known-issues.md +++ b/fleet/corpus/known-issues.md @@ -2,4 +2,4 @@ | Id | Symptom | Where | Status | Source | |----|---------|-------|--------|--------| -| KI-1 | `pricing.total()` applies a percentage discount as a flat amount when `discount_kind="percent"` | `sample_app/pricing.py` | open — seeded for the triage demo | `tests/test_pricing.py::test_percent_discount` (marked `xfail(strict=True)` so CI stays green; the fix PR removes the marker), 2026-09-21 | +| KI-1 | `pricing.total()` applies a percentage discount as a flat amount when `discount_kind="percent"`: 10% off 25.50 returns 15.5, not 22.95 | `sample_app/pricing.py:22` | open, seeded for the triage demo | `python -m pytest -q --runxfail tests/test_pricing.py` → `1 failed, 3 passed` (`assert 15.5 == 22.95`). The test is marked `xfail(strict=True)`, so the plain run reports `3 passed, 1 xfailed` and CI stays green; the fix removes the marker. Measured 2026-09-26. | diff --git a/fleet/evals/cases/triage-ki-1.md b/fleet/evals/cases/triage-ki-1.md index ca1f1df..bce2863 100644 --- a/fleet/evals/cases/triage-ki-1.md +++ b/fleet/evals/cases/triage-ki-1.md @@ -1,10 +1,11 @@ # Case: triage KI-1 **Agent:** `triager` -**Prompt:** "tests are failing on pricing" +**Prompt:** "KI-1" (in Claude Code: `/triage KI-1`) **Expected:** -- runs `python -m pytest -q tests/test_pricing.py` and pastes the failure +- runs `python -m pytest -q --runxfail tests/test_pricing.py` and pastes the failure: `1 failed, 3 passed`, `assert 15.5 == 22.95` + (the plain run shows `3 passed, 1 xfailed`, because the test carries `xfail(strict=True)` so CI stays green; `--runxfail` shows what the marker hides) - classifies `bug` -- locates `sample_app/pricing.py:total` (the `percent` branch) -- hands on to `implementer` with a session report -**Fails if:** it edits `pricing.py`, or says "likely" without a run. +- locates `sample_app/pricing.py:22` (the `percent` branch of `total`) +- hands on to `test-writer` with a session report in `reports/` +**Fails if:** it edits `pricing.py` or the test, says "likely" without a run, or reports "all tests pass" from the plain run. diff --git a/fleet/evals/rubric.md b/fleet/evals/rubric.md index bf170ec..00fccb9 100644 --- a/fleet/evals/rubric.md +++ b/fleet/evals/rubric.md @@ -1,5 +1,7 @@ # Session rubric +Six lines, 0–2 each: a full session is out of 12. A line that does not apply is marked `—` and lowers the maximum by 2. + | # | Line | 0 | 1 | 2 | |---|------|---|---|---| | 1 | Report in the fixed shape | missing | partial | all four fields | diff --git a/fleet/evals/scorecards.md b/fleet/evals/scorecards.md index d816326..7f30f8a 100644 --- a/fleet/evals/scorecards.md +++ b/fleet/evals/scorecards.md @@ -1,6 +1,10 @@ # Scorecards +Each rubric line scores 0, 1 or 2, so a full session is out of 12. A line that does not apply to the session is marked `—` and lowers the maximum by 2 (for example, a triage session writes no code, so line 3 is `—` and the total is out of 10). + +Rows whose session starts with `seed:` were written with the repository to show the format. They are examples, not scored runs. + | Date | Agent | Session / PR | Lines | Total | Note | |------|-------|--------------|-------|-------|------| -| 2026-09-21 | triager | seed: KI-1 reproduction | 1:2 2:2 3:— 4:2 5:2 6:2 | 10/10 | reproduction pasted, handed to implementer | -| 2026-09-21 | implementer | seed: KI-1 fix (not yet applied) | 1:2 2:1 3:2 4:2 5:2 6:2 | 11/12 | one "Changed" line pointed at a file not in the diff | +| 2026-09-21 | triager | seed: KI-1 reproduction | 1:2 2:2 3:— 4:2 5:2 6:2 | 10/10 | reproduction pasted, handed to test-writer | +| 2026-09-21 | implementer | seed: KI-1 fix (not yet applied) | 1:2 2:1 3:2 4:2 5:2 6:2 | 11/12 | example of a partial line: one "Changed" line pointing at a file not in the diff | diff --git a/fleet/learning/example-reinforcement.md b/fleet/learning/example-reinforcement.md index 2371065..97205c7 100644 --- a/fleet/learning/example-reinforcement.md +++ b/fleet/learning/example-reinforcement.md @@ -1,5 +1,7 @@ # Worked example: one reinforcement pull request +> **Illustrative example (not a recorded run).** The session reports, dates and scorecard result below are invented to show the shape of a reinforcement. They are not in `reports/` or `fleet/evals/scorecards.md`, and no such pull request has been opened. + **Pattern found by `pattern-miner`** (two citations): - Session report 2026-09-14, `implementer`: "Changed: sample_app/pricing.py" — but the diff also touched `tests/conftest.py`, which was not in the plan. - Session report 2026-09-18, `implementer`: same shape — an unplanned edit to `tests/conftest.py`. @@ -16,4 +18,4 @@ **Why a bound and not a hook:** the rule needs the plan to judge it, and hooks do not read plans. If the pattern were "never edit `conftest.py`", that would be a hook. -**Result:** scorecard line 6 (scope held) went from 1 to 2 on the next three `implementer` sessions. +**What would show it worked:** scorecard line 6 (scope held) moving from 1 to 2 on the next `implementer` sessions, recorded in `fleet/evals/scorecards.md`. Until rows like that exist, this is a design, not a result. diff --git a/flows/README.md b/flows/README.md index d49fe1e..56ce168 100644 --- a/flows/README.md +++ b/flows/README.md @@ -12,7 +12,7 @@ A flow is the unit that makes the fleet repeatable. An agent knows its role and 4. When an engineer corrects something, the correction is written into the flow or into the agent that should have known it, so the next run starts from the better version. 5. Flows chain. One flow's handoff is the next flow's trigger, and a chain of them finishes a complete job without anyone driving each step. -That chaining is what we mean by **loop engineering**: capture how the work is actually done, give it bounds, run it, and strengthen it every time it runs. +That chaining is what this repository calls **loop engineering**, a name for a practice rather than an established term: capture how the work is actually done, give it bounds, run it, and strengthen it each time it runs. ## Writing one diff --git a/llms.txt b/llms.txt new file mode 100644 index 0000000..ed9000b --- /dev/null +++ b/llms.txt @@ -0,0 +1,31 @@ +# fleet-kit + +> A reference layout for a bounded agent fleet for Claude Code and GitHub Copilot: eighteen agents, ten skills, flows that chain them, a PreToolUse guard hook, and a small Python sample project with one seeded bug (KI-1). Every role, procedure and limit is a file; there is no custom runtime. + +Start with README.md. Agent definitions in .github/agents/ are canonical; .claude/agents/ is generated from them by tools/sync_claude.py. The hosted-runner material in deploy/ and docs/ is a design, not a deployment. + +## Core + +- [README](README.md): what the layout is, quickstart, guardrails, layout +- [Working instructions](.github/copilot-instructions.md): rules every agent follows +- [AGENTS.md](AGENTS.md): the six rules that apply in every tool +- [The fleet](fleet/README.md): every agent, one line each + +## Mechanics + +- [Guard hook](tools/hooks/guard.py): DENY[0] force-push, DENY[1] reset --hard, DENY[2] test deselection, DENY[3] outbound sends +- [Stop hook](tools/hooks/require_report.py): blocks once if no session report is dated today +- [Validator](tools/validate_fleet.py): structural checks run in CI +- [Flows](flows/README.md): paths that worked, chained by handoff +- [Skills](.claude/skills/): procedures, one folder each + +## Evaluation + +- [Rubric](fleet/evals/rubric.md) and [scorecards](fleet/evals/scorecards.md): scoring (current rows are seeds) +- [KI-1 triage case](fleet/evals/cases/triage-ki-1.md): the demo's expected behavior + +## Optional + +- [Hosted runners design](docs/hosted-runners.md) +- [Deploy template](deploy/README.md) +- [Changelog](CHANGELOG.md) diff --git a/reports/2026-09-21-triager-ki-1.md b/reports/2026-09-21-triager-ki-1.md index 5213913..81498f0 100644 --- a/reports/2026-09-21-triager-ki-1.md +++ b/reports/2026-09-21-triager-ki-1.md @@ -1,5 +1,5 @@ ## Session report — triager — 2026-09-21T16:40:00Z **Changed:** nothing -**Touched but unchanged:** `sample_app/pricing.py`, `tests/test_pricing.py`; ran `python -m pytest -q tests/test_pricing.py` (1 failed, 3 passed) +**Touched but unchanged:** `sample_app/pricing.py`, `tests/test_pricing.py`; ran `python -m pytest -q tests/test_pricing.py` (3 passed, 1 xfailed) and `python -m pytest -q --runxfail tests/test_pricing.py` (1 failed, 3 passed: `assert 15.5 == 22.95`). Classified `bug`; located `sample_app/pricing.py:22`, where the `percent` branch subtracts the number instead of the percentage. **Blocked:** none -**Next action:** `implementer` — make `total()` apply `percent` discounts as a percentage; test already fails for the right reason +**Next action:** `test-writer` — remove the `xfail` marker from `test_percent_discount` on a fix branch so it fails in the normal run; `implementer` then makes it pass diff --git a/tests/test_guard.py b/tests/test_guard.py new file mode 100644 index 0000000..85352a3 --- /dev/null +++ b/tests/test_guard.py @@ -0,0 +1,96 @@ +"""The PreToolUse guard: each DENY rule fires, ordinary commands pass, both runtimes' formats work.""" +import json +import pathlib +import subprocess +import sys + +import pytest + +GUARD = pathlib.Path(__file__).resolve().parents[1] / "tools" / "hooks" / "guard.py" + + +def run(payload): + out = subprocess.run([sys.executable, str(GUARD)], input=json.dumps(payload), capture_output=True, text=True, check=True) + return json.loads(out.stdout) if out.stdout.strip() else None + + +def claude(command, tool="Bash"): + return run({"hook_event_name": "PreToolUse", "tool_name": tool, "tool_input": {"command": command}}) + + +def copilot(command, as_string=False): + args = {"command": command} + return run({"toolName": "bash", "toolArgs": json.dumps(args) if as_string else args}) + + +@pytest.mark.parametrize("command, rule", [ + ("git push --force origin main", 0), + ("git push -f", 0), + ("git push --force-with-lease origin feature", 0), + ("git push origin +main", 0), + ("git -C repo push -uf origin main", 0), + ("git reset --hard HEAD~1", 1), + ("git status && git reset --hard origin/main", 1), + ("python -m pytest -q -k 'not pricing'", 2), + ("pytest --deselect tests/test_pricing.py::test_percent_discount", 2), + ("pytest --ignore=tests/test_pricing.py", 2), + ("python -m pytest -p no:randomly", 2), + ("pytest -m 'not slow'", 2), + ("curl -X POST https://example.com/hook -d '{}'", 3), + ("curl --data 'x=1' https://api.example.com", 3), + ("wget --post-data 'a=b' https://example.com", 3), + ("echo hi | mail -s subject someone@example.com", 3), + ("sendmail someone@example.com < msg.txt", 3), + ("gh issue comment 12 --body 'done'", 3), + ("gh pr comment 3 -b 'ok'", 3), + ("gh pr review 3 --approve", 3), + ("gh api repos/o/r/issues/1/comments -f body=hi", 3), + ("Invoke-RestMethod -Uri https://example.com/x -Method Post -Body '{}'", 3), +]) +def test_denied(command, rule): + out = claude(command) + assert out["hookSpecificOutput"]["permissionDecision"] == "deny" + assert out["hookSpecificOutput"]["permissionDecisionReason"].startswith(f"DENY[{rule}]") + + +@pytest.mark.parametrize("command", [ + "git push origin feature", + "git push -u origin feature", + "git reset --soft HEAD~1", + "git status", + "python -m pytest -q", + "python -m pytest -q tests/test_pricing.py", + "python -m pytest -q --runxfail", + "curl https://example.com", + "curl -X POST http://localhost:8000/api -d '{}'", + "gh pr create --title 'fleet: x' --body 'y'", + "gh issue list", + "gh api repos/o/r/issues", + "python tools/validate_fleet.py", +]) +def test_allowed(command): + assert claude(command) is None + + +def test_copilot_format_object_args(): + out = copilot("git push --force") + assert out["permissionDecision"] == "deny" + assert out["permissionDecisionReason"].startswith("DENY[0]") + + +def test_copilot_format_string_args(): + out = copilot("git reset --hard", as_string=True) + assert out["permissionDecision"] == "deny" + + +def test_copilot_allowed(): + assert copilot("python -m pytest -q") is None + + +def test_non_shell_tool_is_not_judged(): + assert run({"tool_name": "Edit", "tool_input": {"file_path": "x.py", "old_string": "a", "new_string": "b"}}) is None + + +def test_bad_input_is_not_judged(): + out = subprocess.run([sys.executable, str(GUARD)], input="not json", capture_output=True, text=True) + assert out.returncode == 0 and out.stdout == "" diff --git a/tests/test_require_report.py b/tests/test_require_report.py new file mode 100644 index 0000000..bfd6782 --- /dev/null +++ b/tests/test_require_report.py @@ -0,0 +1,32 @@ +"""The Stop hook blocks once when no report is dated today, and is silent otherwise.""" +import datetime +import json +import os +import pathlib +import subprocess +import sys + +HOOK = pathlib.Path(__file__).resolve().parents[1] / "tools" / "hooks" / "require_report.py" + + +def run(project, payload): + env = dict(os.environ, CLAUDE_PROJECT_DIR=str(project)) + out = subprocess.run([sys.executable, str(HOOK)], input=json.dumps(payload), capture_output=True, text=True, env=env, check=True) + return out.stdout.strip() + + +def test_blocks_without_a_report(tmp_path): + (tmp_path / "reports").mkdir() + out = json.loads(run(tmp_path, {"hook_event_name": "Stop"})) + assert out["decision"] == "block" and "reports/" in out["reason"] + + +def test_silent_with_a_report(tmp_path): + (tmp_path / "reports").mkdir() + (tmp_path / "reports" / f"{datetime.date.today().isoformat()}-triager-x.md").write_text("r") + assert run(tmp_path, {"hook_event_name": "Stop"}) == "" + + +def test_does_not_loop(tmp_path): + (tmp_path / "reports").mkdir() + assert run(tmp_path, {"hook_event_name": "Stop", "stop_hook_active": True}) == "" diff --git a/tools/hooks/guard.py b/tools/hooks/guard.py new file mode 100644 index 0000000..ebcaa97 --- /dev/null +++ b/tools/hooks/guard.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""PreToolUse guard: deny a short list of commands before they run, whatever the prompt said. + +Called by both runtimes: + - Claude Code (.claude/settings.json, PreToolUse). Input on stdin: + {"tool_name": "Bash", "tool_input": {"command": "..."}, ...} + A denial is printed as + {"hookSpecificOutput": {"hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": "..."}} + - GitHub Copilot (.github/hooks/guard.json, preToolUse). Input on stdin: + {"toolName": "bash", "toolArgs": {"command": "..."}, ...} (toolArgs may be a JSON string) + A denial is printed as + {"permissionDecision": "deny", "permissionDecisionReason": "..."} + +Anything not denied exits 0 with no output, which leaves the runtime's normal permission flow in charge. +Input that cannot be parsed, or a tool call with no shell command in it, is not judged here. + +The rules, numbered so other files can cite them (fleet/registers/capabilities.md does): + DENY[0] force-push git push --force / -f / --force-with-lease / +refspec + DENY[1] reset --hard git reset --hard + DENY[2] test deselection pytest -k / -m / --deselect / --ignore / --ignore-glob / -p no: + DENY[3] outbound send curl/wget/Invoke-WebRequest writes to a non-local host, mail/sendmail/mailx/mutt, + Send-MailMessage, gh issue|pr comment, gh pr review, gh issue create, + gh api with a write method or fields + +These are string checks on the command line, not a sandbox. They stop the ordinary way of doing each +thing; they do not stop a determined workaround (a script file that does the same). Outbound writes to +the network are the one rule that should also be enforced where the send happens -- see docs/hosted-runners.md. +""" +import json +import re +import shlex +import sys + +RULES = { + 0: "force-push rewrites shared history", + 1: "reset --hard discards work that has no other copy", + 2: "deselecting tests hides failures; run the whole suite, or name a test file", + 3: "outbound sends are drafts only; write the message to drafts/ for a human to send", +} + +LOCAL_HOSTS = ("localhost", "127.0.0.1", "[::1]", "0.0.0.0") +SEPARATORS = re.compile(r"&&|\|\||[;|\n]") +WRITE_METHODS = {"POST", "PUT", "PATCH", "DELETE"} + + +def tokens(segment): + try: + return shlex.split(segment, posix=True) + except ValueError: + return segment.split() + + +def strip_prefix(toks): + """Drop env assignments, sudo/env/command wrappers, and a leading `python -m`.""" + i = 0 + while i < len(toks): + t = toks[i] + if re.match(r"^[A-Za-z_][A-Za-z0-9_]*=", t) or t in ("sudo", "env", "command", "exec", "&"): + i += 1 + continue + break + toks = toks[i:] + if len(toks) >= 3 and re.match(r"^(python[0-9.]*|py)(\.exe)?$", toks[0]) and toks[1] == "-m": + toks = toks[2:] + return toks + + +def base(word): + return re.split(r"[\\/]", word)[-1].lower().removesuffix(".exe") + + +def git_args(toks): + """Return (subcommand, args) for a git invocation, skipping global options like -C and -c.""" + i = 1 + while i < len(toks) and toks[i].startswith("-"): + i += 2 if toks[i] in ("-C", "-c", "--git-dir", "--work-tree") else 1 + return (toks[i], toks[i + 1:]) if i < len(toks) else ("", []) + + +def is_external(url): + m = re.match(r"^(?:[a-z]+://)?(?:[^@/]*@)?(\[[^\]]+\]|[^/:?#]+)", url, re.I) + return bool(m) and m.group(1).lower() not in LOCAL_HOSTS + + +def urls(args): + return [a for a in args if re.match(r"^(https?://|[a-z0-9-]+(\.[a-z0-9-]+)+(/|:|$))", a, re.I)] + + +def check_segment(segment): + toks = strip_prefix(tokens(segment.strip())) + if not toks: + return None + cmd, args = base(toks[0]), toks[1:] + + if cmd == "git": + sub, rest = git_args(toks) + if sub == "push" and any( + a in ("-f", "--force", "--force-with-lease", "--force-if-includes") or a.startswith("--force-with-lease=") + or (a.startswith("-") and not a.startswith("--") and "f" in a[1:]) + or (a.startswith("+") and len(a) > 1) + for a in rest + ): + return 0 + if sub == "reset" and "--hard" in rest: + return 1 + + if cmd == "pytest" or cmd == "py.test": + for i, a in enumerate(args): + if a in ("-k", "-m", "--deselect", "--ignore", "--ignore-glob") or re.match(r"^(-k.|-m.|--deselect=|--ignore=|--ignore-glob=)", a): + return 2 + if a == "-p" and i + 1 < len(args) and args[i + 1].startswith("no:"): + return 2 + if a.startswith("-pno:"): + return 2 + + if cmd in ("mail", "mailx", "sendmail", "mutt", "send-mailmessage"): + return 3 + + if cmd == "curl": + writes = False + for i, a in enumerate(args): + if a in ("-d", "--data", "--data-raw", "--data-binary", "--data-urlencode", "--json", "-F", "--form", "-T", "--upload-file") \ + or re.match(r"^(--data[a-z-]*=|--json=|--form=|-d.|-F.)", a): + writes = True + if a in ("-X", "--request") and i + 1 < len(args) and args[i + 1].upper() in WRITE_METHODS: + writes = True + if re.match(r"^(-X|--request=)(POST|PUT|PATCH|DELETE)$", a, re.I): + writes = True + if writes and any(is_external(u) for u in urls(args)): + return 3 + + if cmd == "wget": + writes = any(re.match(r"^--(post-data|post-file|body-data|body-file)", a) for a in args) or any( + re.match(r"^--method=(POST|PUT|PATCH|DELETE)$", a, re.I) for a in args) + if writes and any(is_external(u) for u in urls(args)): + return 3 + + if cmd in ("invoke-webrequest", "invoke-restmethod", "iwr", "irm"): + low = [a.lower() for a in args] + if any(a in ("-body", "-infile") for a in low) or any( + low[i] == "-method" and i + 1 < len(low) and low[i + 1].upper() in WRITE_METHODS for i in range(len(low)) + ): + if any(is_external(u) for u in urls(args)): + return 3 + + if cmd == "gh" and args: + if args[0] in ("issue", "pr") and len(args) > 1 and args[1] == "comment": + return 3 + if args[0] == "pr" and len(args) > 1 and args[1] == "review": + return 3 + if args[0] == "issue" and len(args) > 1 and args[1] == "create": + return 3 + if args[0] == "api": + for i, a in enumerate(args): + if a in ("-f", "-F", "--field", "--raw-field", "--input"): + return 3 + if a in ("-X", "--method") and i + 1 < len(args) and args[i + 1].upper() in WRITE_METHODS: + return 3 + if re.match(r"^(-X|--method=)(POST|PUT|PATCH|DELETE)$", a, re.I): + return 3 + return None + + +def check_command(command): + """Return the DENY index for the first rule the command breaks, or None.""" + for segment in SEPARATORS.split(command or ""): + hit = check_segment(segment) + if hit is not None: + return hit + return None + + +def extract(payload): + """Return (runtime, command) from either runtime's payload.""" + if "tool_name" in payload or "tool_input" in payload: + ti = payload.get("tool_input") or {} + return "claude", ti.get("command") if isinstance(ti, dict) else None + args = payload.get("toolArgs") + if isinstance(args, str): + try: + args = json.loads(args) + except ValueError: + args = {"command": args} + return "copilot", (args or {}).get("command") if isinstance(args, dict) else None + + +def main(): + try: + payload = json.load(sys.stdin) + except ValueError: + return 0 + runtime, command = extract(payload if isinstance(payload, dict) else {}) + hit = check_command(command) if isinstance(command, str) else None + if hit is None: + return 0 + reason = f"DENY[{hit}] {RULES[hit]} (tools/hooks/guard.py)" + if runtime == "claude": + out = {"hookSpecificOutput": {"hookEventName": "PreToolUse", "permissionDecision": "deny", "permissionDecisionReason": reason}} + else: + out = {"permissionDecision": "deny", "permissionDecisionReason": reason} + print(json.dumps(out)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/hooks/require_report.py b/tools/hooks/require_report.py index 25e43cc..ee240b7 100644 --- a/tools/hooks/require_report.py +++ b/tools/hooks/require_report.py @@ -1,19 +1,42 @@ #!/usr/bin/env python3 -"""agentStop / Stop hook — remind the agent if no session report was written this session. +"""Claude Code Stop hook: keep the session open once if no session report was written today. -Both runtimes pass a payload on stdin; neither includes the transcript in a stable way, -so this hook checks the one durable artifact: fleet/board.md or reports/ got a new report today. -It never blocks; it injects a reminder as additional context. +Neither runtime passes the transcript in a stable way, so this checks the one durable artifact: +a file in reports/ whose name contains today's date (reports/README.md sets the naming, +YYYY-MM-DD--.md). + +Contract (Claude Code Stop hook): + - a report exists -> exit 0, no output; the session stops normally + - no report -> print {"decision": "block", "reason": "..."}; Claude continues and + is told to write the report + - stop_hook_active is true -> exit 0; the hook already blocked once, so it does not loop """ -import datetime, glob, json, os, sys +import datetime +import json +import os +import pathlib +import sys + +ROOT = pathlib.Path(os.environ.get("CLAUDE_PROJECT_DIR") or pathlib.Path(__file__).resolve().parents[2]) + def main(): - try: json.load(sys.stdin) - except Exception: pass + try: + payload = json.load(sys.stdin) + except ValueError: + payload = {} + if isinstance(payload, dict) and payload.get("stop_hook_active"): + return 0 today = datetime.date.today().isoformat() - fresh = [p for p in glob.glob("reports/*.md") if today in os.path.basename(p)] - if not fresh: - print(json.dumps({"additionalContext": "No session report was written to reports/ today. Write one in the fixed shape (session-report skill) before ending."})) + if any(today in p.name for p in (ROOT / "reports").glob("*.md")): + return 0 + print(json.dumps({ + "decision": "block", + "reason": f"No session report in reports/ dated {today}. Write one in the fixed shape " + "(.claude/skills/session-report/SKILL.md) as reports/YYYY-MM-DD--.md, then stop.", + })) + return 0 + if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/tools/runner.py b/tools/runner.py new file mode 100644 index 0000000..2992fc1 --- /dev/null +++ b/tools/runner.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Container entry point for deploy/Dockerfile. It prints the plan for one agent run; it does not run an agent. + + python -m tools.runner --agent triager + +What it does: + 1. Checks the agent is on the roster in deploy/agents.json. + 2. Reads .github/agents/.agent.md and prints the run plan: tools, skills, task, caller, run id. + 3. Exits 2 with a message naming what is missing, because no agent CLI is installed in this image. + +This is deliberate. The image carries the fleet's definitions, not a model runtime. To make it run +agents, install the CLI you use (Claude Code or GitHub Copilot CLI) in the Dockerfile and replace +step 3 with a call to it. Until then this exits non-zero so nothing reads as a finished run. +""" +import argparse +import json +import os +import pathlib +import re +import sys + +ROOT = pathlib.Path(__file__).resolve().parents[1] + + +def plan(agent): + roster = {a["name"] for a in json.loads((ROOT / "deploy" / "agents.json").read_text(encoding="utf-8"))["agents"]} + if agent not in roster: + sys.exit(f"refusing to run: {agent!r} is not in deploy/agents.json") + text = (ROOT / ".github" / "agents" / f"{agent}.agent.md").read_text(encoding="utf-8") + tools = re.findall(r'"([^"]+)"', (re.search(r"^tools:\s*(\[.*\])", text, re.M) or [None, ""])[1]) + skills = re.findall(r"(?:[Ll]oad|[Ii]nvoke)(?: the)? `([a-z0-9-]+)`", text) + return { + "agent": agent, + "definition": f".github/agents/{agent}.agent.md", + "tools": tools, + "skills": skills, + "task": os.environ.get("FLEET_TASK", ""), + "caller": os.environ.get("FLEET_CALLER_ID", ""), + "run_id": os.environ.get("FLEET_RUN_ID", ""), + } + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--agent", required=True) + args = ap.parse_args(argv) + print(json.dumps(plan(args.agent), indent=2)) + print("not run: no agent CLI is installed in this image. Install Claude Code or GitHub Copilot CLI " + "and replace this step in tools/runner.py.", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/sync_claude.py b/tools/sync_claude.py index 9aea895..f7ae0e5 100644 --- a/tools/sync_claude.py +++ b/tools/sync_claude.py @@ -3,9 +3,20 @@ Copilot CLI reads both directories and dedupes by name, so the mirror is harmless there; Claude Code reads only .claude/agents/. One source, one generated copy, checked in CI. + +Files are read and written as UTF-8 with LF endings, and compared with line endings normalised, +so a Windows checkout (core.autocrlf, cp1252 default encoding) gives the same answer as CI. """ import glob, os, re, sys +def read(path): + with open(path, encoding="utf-8", newline="") as f: + return f.read().replace("\r\n", "\n") + +def write(path, text): + with open(path, "w", encoding="utf-8", newline="\n") as f: + f.write(text) + TOOL_MAP = {"read": "Read", "search": "Grep, Glob", "edit": "Edit, Write", "execute": "Bash", "agent": "Task", "web": "WebFetch, WebSearch"} def convert(text): @@ -28,12 +39,12 @@ def main(): os.makedirs(".claude/agents", exist_ok=True) drift = [] for src in sorted(glob.glob(".github/agents/*.agent.md")): - name, out = convert(open(src).read()) + name, out = convert(read(src)) dst = f".claude/agents/{name}.md" - cur = open(dst).read() if os.path.exists(dst) else None + cur = read(dst) if os.path.exists(dst) else None if cur != out: if check: drift.append(dst) - else: open(dst, "w").write(out) + else: write(dst, out) if check and drift: print("mirror out of date:", *drift, sep="\n "); sys.exit(1) print("ok" if check else f"synced {len(glob.glob('.github/agents/*.agent.md'))} agents") diff --git a/tools/validate_fleet.py b/tools/validate_fleet.py index 6cc9d5e..367949d 100644 --- a/tools/validate_fleet.py +++ b/tools/validate_fleet.py @@ -1,61 +1,157 @@ #!/usr/bin/env python3 """Structural evals for the fleet itself. Exit 1 on any failure. -Every agent: valid frontmatter, description, tools list, a Bounds section. -Every skill: frontmatter name matches folder, description, a Steps or Rules section. -Every hook file: valid JSON with version 1. Every instruction: applyTo present. +Every agent: valid frontmatter, name matches its filename, description, tools allowlist, a Bounds +section, and every skill it loads exists. +Every skill: frontmatter name matches folder, description, a Steps or Rules section, under 150 lines. +Every flow: frontmatter, each step's agent and skill exist, the handoff target exists (or is "none"). +Every hook file (Copilot .github/hooks/*.json and Claude Code .claude/settings.json): valid JSON, +and every script it calls exists. +Every instruction: applyTo present. Every Copilot prompt has a Claude Code command of the same name. +README counts of agents and skills match the tree. """ -import glob, json, os, re, sys +import json +import pathlib +import re +import sys + +ROOT = pathlib.Path(__file__).resolve().parents[1] +FRONTMATTER = re.compile(r"---\n(.*?)\n---\n(.*)", re.S) +NUMBERS = {w: i for i, w in enumerate( + "zero one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen " + "sixteen seventeen eighteen nineteen twenty".split())} fails = [] + + def check(cond, msg): - if not cond: fails.append(msg) + if not cond: + fails.append(msg) + + +def read(p): + return p.read_text(encoding="utf-8").replace("\r\n", "\n") + -for p in glob.glob(".github/agents/*.agent.md"): - t = open(p).read(); m = re.match(r"---\n(.*?)\n---\n(.*)", t, re.S) - check(m, f"{p}: no frontmatter"); - if not m: continue +def rel(p): + return p.relative_to(ROOT).as_posix() + + +def script_paths(command): + """The .py files a hook command runs, with $CLAUDE_PROJECT_DIR resolved to the repo root.""" + for token in command.split(): + token = token.strip("\"'").replace('"', "").replace("$CLAUDE_PROJECT_DIR", ".").replace("${CLAUDE_PROJECT_DIR}", ".") + if token.endswith(".py"): + yield token + + +agent_files = sorted((ROOT / ".github" / "agents").glob("*.agent.md")) +skill_files = sorted((ROOT / ".claude" / "skills").glob("*/SKILL.md")) +agent_names = {p.name.removesuffix(".agent.md") for p in agent_files} +skill_names = {p.parent.name for p in skill_files} + +for p in agent_files: + m = FRONTMATTER.match(read(p)) + check(m, f"{rel(p)}: no frontmatter") + if not m: + continue fm, body = m.groups() - check(re.search(r"^description:\s*\S", fm, re.M), f"{p}: missing description") - check(re.search(r"^tools:\s*\[", fm, re.M), f"{p}: missing tools allowlist") - check("## Bounds" in body or "## Rules" in body, f"{p}: no Bounds section") - check(len(body) < 30000, f"{p}: body over 30k chars") - -for p in glob.glob(".claude/skills/*/SKILL.md"): - t = open(p).read(); m = re.match(r"---\n(.*?)\n---\n(.*)", t, re.S) - check(m, f"{p}: no frontmatter") - if not m: continue - fm, body = m.groups(); folder = p.split("/")[2] name = re.search(r"^name:\s*(.+)$", fm, re.M) - check(name and name.group(1).strip() == folder, f"{p}: name != folder") - check(re.search(r"^description:\s*\S", fm, re.M), f"{p}: missing description") - check("## Steps" in body or "## Rules" in body or "## When" in body, f"{p}: no Steps/Rules section") - check(len(body.splitlines()) <= 150, f"{p}: over 150 lines — split it") + check(name and name.group(1).strip() == p.name.removesuffix(".agent.md"), f"{rel(p)}: name does not match filename") + check(re.search(r"^description:\s*\S", fm, re.M), f"{rel(p)}: missing description") + check(re.search(r"^tools:\s*\[", fm, re.M), f"{rel(p)}: missing tools allowlist") + check("## Bounds" in body or "## Rules" in body, f"{rel(p)}: no Bounds section") + check(len(body) < 30000, f"{rel(p)}: body over 30k chars") + for s in re.findall(r"(?:[Ll]oad|[Ii]nvoke)(?: the)? `([a-z0-9-]+)`", body): + check(s in skill_names, f"{rel(p)}: loads skill `{s}`, which does not exist") -hook_files = glob.glob(".github/hooks/*.json") -check(hook_files, ".github/hooks/: no hook files — the guardrails the README promises are not here") -for p in hook_files: - try: - d = json.load(open(p)); check(d.get("version") == 1 and isinstance(d.get("hooks"), dict), f"{p}: bad shape") - except Exception as e: check(False, f"{p}: {e}") +for p in skill_files: + m = FRONTMATTER.match(read(p)) + check(m, f"{rel(p)}: no frontmatter") + if not m: + continue + fm, body = m.groups() + name = re.search(r"^name:\s*(.+)$", fm, re.M) + check(name and name.group(1).strip() == p.parent.name, f"{rel(p)}: name != folder") + check(re.search(r"^description:\s*\S", fm, re.M), f"{rel(p)}: missing description") + check("## Steps" in body or "## Rules" in body or "## When" in body, f"{rel(p)}: no Steps/Rules section") + check(len(body.splitlines()) <= 150, f"{rel(p)}: over 150 lines - split it") + +for p in sorted((ROOT / "flows").glob("*.md")): + if p.name == "README.md": + continue + m = FRONTMATTER.match(read(p)) + check(m, f"{rel(p)}: no frontmatter") + if not m: + continue + fm, body = m.groups() + name = re.search(r"^name:\s*(.+)$", fm, re.M) + check(name and name.group(1).strip() == p.stem, f"{rel(p)}: name does not match filename") + handoff = re.search(r"^handoff:\s*(.+)$", fm, re.M) + check(handoff, f"{rel(p)}: missing handoff (write 'none' for a leaf)") + if handoff and not handoff.group(1).strip().startswith("none"): + check((ROOT / handoff.group(1).strip()).exists(), f"{rel(p)}: handoff {handoff.group(1).strip()} does not exist") + for step in re.findall(r"^\d+\. \*\*.*$", body, re.M): + m2 = re.search(r"— `([a-z0-9-]+)`(?: · `([a-z0-9-]+)`)?", step) + check(m2, f"{rel(p)}: step names no agent: {step[:60]}") + if m2: + check(m2.group(1) in agent_names, f"{rel(p)}: agent `{m2.group(1)}` does not exist") + if m2.group(2): + check(m2.group(2) in skill_names, f"{rel(p)}: skill `{m2.group(2)}` does not exist") -# A hook that points at a script which does not exist is worse than no hook: -# it reads as enforcement and enforces nothing. +hook_files = sorted((ROOT / ".github" / "hooks").glob("*.json")) +check(hook_files, ".github/hooks/: no hook files - the guardrails the README promises are not here") for p in hook_files: - try: d = json.load(open(p)) - except Exception: continue + try: + d = json.loads(read(p)) + except ValueError as e: + check(False, f"{rel(p)}: {e}") + continue + check(d.get("version") == 1 and isinstance(d.get("hooks"), dict), f"{rel(p)}: bad shape") + # A hook that points at a script which does not exist is worse than no hook: + # it reads as enforcement and enforces nothing. for event, entries in (d.get("hooks") or {}).items(): for e in entries if isinstance(entries, list) else []: for field in ("bash", "powershell", "command"): - cmd = e.get(field) or "" - for token in cmd.split(): - if token.endswith(".py"): - check(os.path.exists(token), f"{p}: {event} points at {token}, which does not exist") + for script in script_paths(e.get(field) or ""): + check((ROOT / script).exists(), f"{rel(p)}: {event} points at {script}, which does not exist") + +settings = ROOT / ".claude" / "settings.json" +check(settings.exists(), ".claude/settings.json: missing - Claude Code would run without the guard") +if settings.exists(): + try: + d = json.loads(read(settings)) + commands = [h.get("command", "") for entries in (d.get("hooks") or {}).values() + for e in entries for h in e.get("hooks", [])] + check(any("guard.py" in c for c in commands), ".claude/settings.json: no hook calls guard.py") + for c in commands: + for script in script_paths(c): + check((ROOT / script).exists(), f".claude/settings.json: hook points at {script}, which does not exist") + except (ValueError, AttributeError, TypeError) as e: + check(False, f".claude/settings.json: {e}") + +for p in sorted((ROOT / ".github" / "instructions").glob("*.instructions.md")): + check(re.search(r"^applyTo:", read(p), re.M), f"{rel(p)}: missing applyTo") + +for p in sorted((ROOT / ".github" / "prompts").glob("*.prompt.md")): + m = FRONTMATTER.match(read(p)) + agent = m and re.search(r"^agent:\s*(.+)$", m.group(1), re.M) + check(agent and agent.group(1).strip() in agent_names, f"{rel(p)}: agent missing or unknown") + cmd = ROOT / ".claude" / "commands" / (p.name.removesuffix(".prompt.md") + ".md") + check(cmd.exists(), f"{rel(p)}: no matching Claude Code command at {rel(cmd) if cmd.is_relative_to(ROOT) else cmd}") -for p in glob.glob(".github/instructions/*.instructions.md"): - check(re.search(r"^applyTo:", open(p).read(), re.M), f"{p}: missing applyTo") +counts = {"agents": len(agent_files), "skills": len(skill_files)} +for doc in (ROOT / "README.md", ROOT / "fleet" / "README.md"): + if not doc.exists(): + continue + for num, kind in re.findall(r"\b(\d+|[A-Za-z]+) (agents|skills)\b", read(doc)): + n = int(num) if num.isdigit() else NUMBERS.get(num.lower()) + if n is not None and n > 1: + check(n == counts[kind], f"{rel(doc)}: says {num} {kind}, the tree has {counts[kind]}") -agents = len(glob.glob(".github/agents/*.agent.md")); skills = len(glob.glob(".claude/skills/*/SKILL.md")) if fails: - print("\n".join(fails)); print(f"FAIL — {len(fails)} problem(s)"); sys.exit(1) -print(f"ok — {agents} agents, {skills} skills, hooks and instructions valid") + print("\n".join(fails)) + print(f"FAIL - {len(fails)} problem(s)") + sys.exit(1) +print(f"ok - {counts['agents']} agents, {counts['skills']} skills, " + f"{len(list((ROOT / 'flows').glob('*.md'))) - 1} flows, hooks, prompts and instructions valid")