From ddab51e83af63fb5bdc8f1203a9e7405d749ab8e Mon Sep 17 00:00:00 2001 From: Jiri Puc Date: Fri, 17 Jul 2026 20:02:56 +0200 Subject: [PATCH] feat: add orchestrator-owned protocol v3 --- AGENTS.md | 37 +- CHANGELOG.md | 19 + README.md | 376 +++--- ...ty-orchestrator-owned-completion-review.md | 54 + ...avity-protocol-v3-implementation-review.md | 52 + ...de-orchestrator-owned-completion-review.md | 46 + ...-code-protocol-v3-implementation-review.md | 60 + ...rompt-layer-planning-and-handoff-review.md | 929 ++++++++++++++ design/decisions.md | 377 +++--- .../designs/long-running-loop-reliability.md | 48 +- ...ned-completion-and-cross-harness-review.md | 982 +++++++++++++++ .../designs/recursive-loop-layer-contract.md | 388 +++--- .../success-semantics-and-evaluation.md | 387 +++--- .../proposals/state-driven-eval-fast-paths.md | 458 +++++++ docs/http-contract.md | 733 ++++++------ docs/session-layout.md | 882 ++++++-------- pyproject.toml | 4 +- skills/loopy-loop/SKILL.md | 982 ++++++--------- src/loopy_loop/assignments.py | 162 ++- src/loopy_loop/cli.py | 4 - src/loopy_loop/config.py | 165 ++- src/loopy_loop/coordinator_app.py | 1066 ++++++++++++++++- src/loopy_loop/models.py | 456 ++++++- src/loopy_loop/sessions.py | 266 +++- src/loopy_loop/state_store.py | 52 +- .../inner_outer_eval/contract.yaml | 53 +- .../workflows/eval_reviewer/config.yaml | 3 +- .../workflows/eval_reviewer/prompt.txt | 99 +- .../workflows/eval_runner/config.yaml | 9 +- .../workflows/eval_runner/prompt.txt | 146 +-- .../workflows/inner/config.yaml | 4 +- .../workflows/inner/prompt.txt | 122 +- .../workflows/outer/config.yaml | 4 +- .../workflows/outer/prompt.txt | 165 ++- .../inner_outer_eval/loopy_loop_config.yaml | 40 +- .../pm_planner_dispatcher/contract.yaml | 64 +- .../workflows/dispatcher/config.yaml | 4 +- .../workflows/dispatcher/prompt.txt | 131 +- .../workflows/eval_reviewer/config.yaml | 11 - .../workflows/eval_reviewer/prompt.txt | 65 - .../workflows/eval_runner/config.yaml | 9 - .../workflows/eval_runner/prompt.txt | 111 -- .../workflows/planner/config.yaml | 4 +- .../workflows/planner/prompt.txt | 183 ++- .../loopy_loop_config.yaml | 42 +- src/loopy_loop/worker.py | 57 +- src/tests/test_agent_skill_contract.py | 35 +- src/tests/test_cli.py | 15 +- src/tests/test_config.py | 94 +- src/tests/test_examples.py | 18 +- src/tests/test_recursive_loop_contract_v2.py | 53 + src/tests/test_recursive_loop_contract_v3.py | 982 +++++++++++++++ src/tests/test_semantic_state_paths_v3.py | 353 ++++++ src/tests/test_template_contracts.py | 258 ++-- uv.lock | 10 +- 55 files changed, 9143 insertions(+), 2986 deletions(-) create mode 100644 design/analysis/antigravity-orchestrator-owned-completion-review.md create mode 100644 design/analysis/antigravity-protocol-v3-implementation-review.md create mode 100644 design/analysis/claude-code-orchestrator-owned-completion-review.md create mode 100644 design/analysis/claude-code-protocol-v3-implementation-review.md create mode 100644 design/analysis/historical-prompt-layer-planning-and-handoff-review.md create mode 100644 design/designs/orchestrator-owned-completion-and-cross-harness-review.md create mode 100644 design/proposals/state-driven-eval-fast-paths.md delete mode 100644 src/loopy_loop/templates/pm_planner_dispatcher/.loopy_loop/workflow_sets/pm_planner_dispatcher/workflows/eval_reviewer/config.yaml delete mode 100644 src/loopy_loop/templates/pm_planner_dispatcher/.loopy_loop/workflow_sets/pm_planner_dispatcher/workflows/eval_reviewer/prompt.txt delete mode 100644 src/loopy_loop/templates/pm_planner_dispatcher/.loopy_loop/workflow_sets/pm_planner_dispatcher/workflows/eval_runner/config.yaml delete mode 100644 src/loopy_loop/templates/pm_planner_dispatcher/.loopy_loop/workflow_sets/pm_planner_dispatcher/workflows/eval_runner/prompt.txt create mode 100644 src/tests/test_recursive_loop_contract_v3.py create mode 100644 src/tests/test_semantic_state_paths_v3.py diff --git a/AGENTS.md b/AGENTS.md index e591ba2..c26cc7a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,17 +18,27 @@ relevant decision (and its companion design doc): - **D3 — iteration success ≠ "work is good."** `IterationResult.success` means only that the harness ran without erroring. Do not make it consult worker exit codes / infer - semantic success. The eval layer (`control.json` + `goal_check.json`) decides quality. -- **D4 — evaluation is LLM-as-judge; agents don't author deterministic checks.** Do not - "add deterministic checks" to the stock `inner_outer_eval` eval workflows. (A target - repo that owns its own test suite is a different case — see D4.) + semantic success. The session's declared orchestrator (`outer` or `planner` in the + stock sets) decides completion through `control.json`, using optional eval and other + evidence. +- **D4 — when evaluation is used, it is LLM-as-judge; agents don't author deterministic + checks.** Do not "add deterministic checks" to the stock `inner_outer_eval` eval + workflows. A target repo that owns its own test suite is a different case. For + non-trivial eval-check creation, prefer parallel cross-harness criteria analysis and + different-family review, without making them a quota or gate. - **D2 — single worker is deliberate.** Do not add parallel loopy workers as a scaling feature. - **D5 — full autonomy with a last-resort escape hatch.** See Rule 2. - **D8 — constraints are detection, not prevention.** Do not add preventive fences (path-level write enforcement, semantic scheduling vetoes, approval flows, arbitrary - mid-run hard-fails). Express constraints as evaluation-layer checks whose failure - blocks *acceptance* of the work, with repair as the path forward. + mid-run hard-fails, mandatory eval gates). Surface findings to the accountable + orchestrator for repair, rerouting, or a reasoned disposition. Hard enforcement is for + protocol truth—identity, topology, schemas, hashes, and reference containment—not + semantic sufficiency. +- **D11 — the durable orchestrator owns completion; eval roles only produce optional + evidence.** Do not give `goal_met` authority to `eval_runner` in fresh amended + contracts or require a same-attempt passing eval. Preserve already-live sessions' + frozen historical contract. If you believe a decision is genuinely wrong, propose amending `design/decisions.md` (state what changes and why) — do not silently contradict it in code. @@ -41,12 +51,14 @@ involvement is a last resort, never a normal step. - **The one sanctioned escape hatch already exists.** When a workflow hits a *genuinely terminal* blocker — a decision only a human can make, a missing credential, a billable/destructive action it isn't permitted to take — it writes the session - `control.json`. Fresh v2 sessions use the identity-bound form below; the - assignment envelope supplies the exact session, workflow, attempt, control path, - and timestamp context: + `control.json`. Use the protocol version frozen in the assignment envelope; it + supplies the exact session, workflow, attempt, control path, and timestamp context. + The protocol-v3 identity-bound blocker form used by the stock workflow sets is + shown below. Frozen v2 sessions use the same blocker fields with + `schema_version: 2`: ```json { - "schema_version": 2, + "schema_version": 3, "control_id": "", "state": "stopped", "reason": "", @@ -61,8 +73,9 @@ involvement is a last resort, never a normal step. "created_at": "" } ``` - Only an already-running legacy v1 session uses the historical compact - `schema_version: 1` form. Do not emit that form for a v2 assignment: it is + Only an assignment whose frozen contract is v1 (including a conservative + custom set with no explicit contract) uses the historical compact + `schema_version: 1` form. Do not emit that form for a v2/v3 assignment: it is archived as a repairable protocol failure rather than stopping the session. This stops the loop as terminal, with a recorded reason. That is the entire human-in-the-loop mechanism, and it is enough. diff --git a/CHANGELOG.md b/CHANGELOG.md index 068802b..ae01d52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## 0.8.0 + +- Added the protocol-v3 orchestration contract. Each layer now has an + inspectable semantic state spine, frozen workflow roster, attempt-local + scheduler view, rolling handoff, and topology-neutral terminal outcome. +- Moved successful completion authority from eval roles to the declared layer + orchestrator (`outer` or `planner`). Evaluations remain provenance-checked + advisory evidence and may be omitted, non-passing, or cited across attempts. +- Added a frozen four-tier harness capability roster to assignments and + coordinator prompts, including nested Team Harness coordinators and direct + spawn audit records. Requires Team Harness 0.5.4. +- Made every protocol-v3 terminal lifecycle produce the same parent-linkable + outcome, including engine stops without terminal control. Accepted control + and handoff bytes are frozen in state so restart and trace refreshes cannot + rewrite the terminal basis. +- Simplified the stock PM set to planner and dispatcher. The planner dispatches + milestone outcomes; standalone or nested `inner_outer_eval` owns leaf + decomposition and completion within its own scoped goal. + ## 0.7.2 - Corrected the installable `loopy-loop` Agent Skill to teach the released v2 diff --git a/README.md b/README.md index c8fa69b..e9369be 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ `loopy-loop` runs long-running AI agent workflows inside your repository. It turns a goal file in your repository into an inspectable sequence of agent -iterations: plan, implement, evaluate, record evidence, and continue until the -goal is met or the loop hits a terminal blocker. +iterations: plan, implement, review, optionally evaluate, record evidence, and +continue until the goal is met or the loop hits a terminal blocker. The value is control and durability. Instead of asking one agent to solve a large task in one fragile chat, loopy-loop gives each durable goal layer a @@ -48,15 +48,13 @@ For development inside this repository: uv sync --extra dev ``` -Version 0.7's recursive contract spans three owned projects. It requires -`team-harness>=0.5.0` for caller-owned run records, pre-call coordinator input, -spawn assignment envelopes, and canonical stdout/stderr capture; it requires -`eval-banana>=0.3.5` for hermetic `--no-project-config` evaluation, explicit -harness-agent validation, and the public canonical check-definition digest -used to verify eval receipts. Version 0.3.5 also retains the exact judge input -and gives every per-check result, stream, and deterministic evidence directory -a common collision-safe stem inside the caller-owned eval trace. Install all -three companion changes together. +The stock protocol-v3 contract spans three owned projects. It requires +`team-harness>=0.5.4` for caller-owned run records, nested assignment context, +capability-roster propagation, and canonical agent streams. When a workflow +uses evaluation, `eval-banana>=0.3.5` supplies hermetic +`--no-project-config` runs, explicit harness selection, canonical check +digests, and retained judge evidence. Install compatible releases of all three +projects together. For coordinated development across the repositories, install the corresponding team-harness and eval-banana checkouts as editable dependencies: @@ -64,9 +62,9 @@ team-harness and eval-banana checkouts as editable dependencies: uv pip install -e /path/to/team-harness -e /path/to/eval-banana ``` -An older dependency is not a reduced-fidelity v2 mode: a fresh v2 session -fails registration clearly if the worker cannot advertise the required -capabilities. +An older dependency is not a reduced-fidelity mode: a fresh stock session +fails registration clearly if its worker or harness cannot advertise the +required protocol-v3 capabilities. ## Install the Agent Skill @@ -100,6 +98,18 @@ This is the recommended starting template. It creates: - additive `.gitignore` entries for session state, traces, the trace-finalization outbox, repository identity, and root state/lock/archive files +For a double loop, initialize the program-level template instead: + +```bash +loopy init --template pm_planner_dispatcher +``` + +Its planner works in high-level phase or milestone outcomes. Its dispatcher +turns one selected outcome into an `inner_outer_eval` child session, whose +outer role owns the detailed leaf plan. The program layer deliberately has no +scheduled eval roles; the planner may coordinate optional program-level or +goal-required final evaluation itself. + `loopy init` is idempotent. It creates missing files and leaves existing files alone — except `.gitignore`, which is updated additively with all runtime ignore rules. @@ -122,9 +132,10 @@ Completion criteria: - README documents required environment variables. ``` -Keep the goal specific enough that a reviewer or eval workflow can decide -whether the loop is done. For one-off overrides, start the coordinator with -`--goal-file PATH`; the file is copied into the session as `goal.md`. +Keep the goal specific enough that its orchestrator and reviewers can assess +whether the scoped outcome is complete. For one-off overrides, start the +coordinator with `--goal-file PATH`; the file is copied into the session as +`goal.md`. ## Run the Loop @@ -167,47 +178,77 @@ At a high level: 1. `loopy init` writes a root config, a goal file, and workflow files into the target repo. 2. `loopy coordinator` loads `loopy_loop_config.yaml`, freezes the goal and - workflow-set contract, creates a v2 root session under + stock protocol-v3 workflow contract, creates a root session under `.loopy_loop/sessions/`, and exposes `/register` and `/finished`. -3. A v2 worker advertises its protocol/capabilities and repository identity. - The coordinator assigns work only to a matching checkout, freezes the exact - workflow config/prompt/contract and assignment, creates the active attempt - trace, and returns one identity-bound attempt with its assignment hash. -4. Before calling a model, the worker reopens and verifies that same trace, - records the exact task response, verifies the frozen `assignment.json`, and - writes the rendered prompt and git-before evidence. The assignment gives the - harness coordinator absolute paths for its own session layer while durable - receipts continue to use portable logical references. -5. `team-harness` runs the coordinator model. It may dynamically spawn Codex, - Claude Code, Gemini, or other configured agents; each direct spawn receives - an automatic assignment envelope identifying its parent attempt, delegated - task, relevant state paths, and output directory. -6. The worker records the normalized result and compact evidence with the - session and posts a completion bound to the exact worker, repository, - attempt, and assignment hash. The coordinator records the exact observed - completion response (or an explicit unavailable status after interruption) - and then seals detailed observable execution under +3. A protocol-v3 worker advertises its Loopy and team-harness capabilities plus + repository identity. The coordinator dispatches only to a matching checkout + with the required capability-roster context support. +4. For each attempt, the coordinator freezes the exact workflow + config/prompt/contract, an assignment, and a conditional scheduler view. + The assignment gives the harness coordinator absolute paths to its scoped + goal, plan, tasks, decisions, handoff, workflow roster, scheduler view, + capability roster, outputs, and trace directory. Durable receipts still use + portable logical references. +5. Before calling a model, the worker verifies that assignment and its trace, + records the exact task response, and writes rendered prompt and git-before + evidence. +6. `team-harness` runs the coordinator model. It may dynamically spawn Codex, + Claude Code, Gemini, or another enabled family. Each direct spawn receives + the current layer identity, relevant absolute state paths, the frozen + capability roster, and its focused delegated assignment. The workflow + coordinator remains accountable for integration. +7. The worker posts a completion bound to the exact worker, repository, + attempt, and assignment hash. A successful iteration means the harness ran + without an execution error; it does not mean the work was semantically + accepted. The coordinator records and seals the observable execution under `.loopy_loop/traces/`. -7. The coordinator checks structural protocol evidence, session-local eval and - control artifacts, child requests, and stop/budget conditions. Semantic - quality remains the responsibility of the workflow/eval agents (D3/D4). +8. The coordinator enforces protocol facts such as identity, topology, + schemas, hashes, and reference containment. The workflow contract's durable + orchestrator decides semantic acceptance and completion from the available + evidence. A terminal protocol-v3 session receives a topology-neutral + `session_outcome.json` that links its control, handoff, and available + evidence. The `inner_outer_eval` template is organized around four workflows: -- `outer`: reviews implementation evidence, accepts or returns work, maintains - the accepted ledger, and publishes eval-readiness context without closing - the session. -- `inner`: dynamically plans/delegates and implements one focused unit in the - target repo; its harness coordinator integrates all spawned-agent work. -- `eval_reviewer`: creates or refreshes outcome-oriented, session-scoped - eval-banana checks. -- `eval_runner`: runs those checks, publishes the canonical eval receipt and - matching `goal_check.json`, and alone may request successful terminal control - for this layer. - -The loop does not hide state inside a chat transcript. Continuity comes from -git plus compact files in `.loopy_loop/sessions//`; detailed -execution records live separately in `.loopy_loop/traces/`. +- `outer`: owns the layer plan, selects and accepts leaves, maintains compact + resumption state and the upward handoff, and alone decides successful + completion for this layer. +- `inner`: implements and verifies the one leaf selected by `outer`, then + reports evidence upward. It does not create the layer plan or accept its own + work. +- `eval_reviewer`: may create or refresh outcome-oriented, session-scoped + `harness_judge` checks as advisory evidence. +- `eval_runner`: may run those checks and publish provenance-rich receipts for + `outer` to weigh. It does not write `goal_check.json` or successful terminal + control. + +The `pm_planner_dispatcher` template has two durable roles: + +- `planner`: owns a high-level phase/milestone plan, accepts or reroutes child + outcomes, maintains the program handoff, and decides program completion. +- `dispatcher`: faithfully turns the one selected milestone outcome into a + child request. It does not pre-plan the child's leaves or accept the child's + result. + +Evaluation is optional evidence in both sets. The completion owner can run or +delegate checks, wait for a scheduled eval role, or decide from stronger +repository, review, child, test, git, or delivery evidence. Protocol-v3 +`control.json` binds `goal_met` to the exact current orchestrator attempt; +`evidence_refs`, plural `eval_receipt_refs`, and `handoff_ref` are optional, +but any cited reference must validate. `unresolvable_error` remains the +last-resort stop for a genuinely terminal blocker after autonomous routes are +exhausted. + +The loop does not hide state inside a chat transcript. Each layer has a visible +semantic spine in `project_state/`—plan, stable task records, current state, +decisions, accepted-work ledger, optional eval index, and rolling handoff. A +session-frozen workflow roster explains every scheduled role; an +attempt-frozen scheduler view forecasts the next role under explicit +assumptions; and the root capability roster shows every enabled harness family +across the configured strength tiers. Detailed prompts, model turns, tool and +spawn I/O, raw eval reports, and verbose logs live separately in the gitignored +trace tree. ## Repo Layout @@ -231,12 +272,21 @@ target repo/ │ ├── goal_contract.json │ ├── session.json │ ├── workflow_contract.json + │ ├── workflow_roster.json + │ ├── harness_capability_roster.json # root, shared by session tree │ ├── state.json - │ ├── control.json + │ ├── control.json # written only to stop + │ ├── session_outcome.json # terminal projection │ ├── inputs/{user_updates.jsonl,accepted_request.json,artifacts/} │ ├── project_state/ + │ │ ├── plan.md + │ │ ├── tasks/ + │ │ ├── current_state.md + │ │ ├── decisions/ + │ │ ├── finished.md + │ │ ├── eval_state.md + │ │ └── handoff.json │ ├── eval_checks/ - │ ├── eval_readiness/ │ ├── eval_receipts/ │ ├── child_requests/{pending,accepted,rejected}/ │ ├── child_outcomes/ @@ -244,7 +294,9 @@ target repo/ │ ├── git_receipts/ │ ├── delivery_receipts/ │ ├── trace_seals/ - │ ├── iterations/ + │ ├── iterations///workflow_snapshot// + │ │ ├── assignment.json + │ │ └── scheduler_view.json │ └── children//... ├── traces//sessions//attempts// └── trace_finalization_outbox/ @@ -290,28 +342,42 @@ Important rules: - `max_turns` is the maximum number of completed workflow iterations. - `team_harness_model` controls the team-harness coordinator model. - `team_harness_agent_models` controls default models for worker subprocesses. -- `model_tiers` (optional) declares named worker tiers — tier name → agent → - `{model, effort}` — as the single source of truth for this repo's model - ids. Loopy renders the table into the harness system prompt so coordinators - can pass `spawn_agent(model=…, effort=…)` to select one bundle for a task - (guidance, not enforcement — see D8/D9 in `design/decisions.md`). The - per-spawn `effort` argument was introduced in team-harness 0.4.0. Loopy 0.7 - requires team-harness 0.5.0 for the wider caller/trace contract, so current - installs always have it. With `default_tier` set, the named tier - derives `team_harness_agent_models` and - `team_harness_agent_reasoning_efforts` (the tier must cover every - configured agent); setting those mappings explicitly alongside - `default_tier` is a config error. +- `model_tiers` (optional) maps semantic strength → enabled harness family → + `{model, effort}`. The four stock tiers are `frontier` for maximum-capability + and highest-stakes work, `strong` for complex work, `standard` for the + balanced default, and `economy` for bounded low-risk work. At root-session + creation, Loopy freezes every family/tier cell—including unavailable cells— + into `harness_capability_roster.json`; the assignment and team-harness caller + context carry that roster to coordinators and nested delegates. Agents choose + proportionately and are encouraged to use another enabled family for useful + independent review, especially during eval-check design. This remains + guidance, not a quota or semantic gate (D8/D9). + + With `default_tier` set, that tier derives + `team_harness_agent_models` and + `team_harness_agent_reasoning_efforts`; it must cover every configured + family, and the two flat mappings must be omitted. Without `default_tier`, + flat per-family model settings appear as configured `standard` bundles. ```yaml model_tiers: + frontier: + codex: {model: "", effort: ""} + claude: {model: "", effort: ""} + gemini: {model: "", effort: ""} strong: - codex: {model: "gpt-5.6-sol", effort: "xhigh"} - claude: {model: "claude-fable-5", effort: "max"} + codex: {model: "", effort: ""} + claude: {model: "", effort: ""} + gemini: {model: "", effort: ""} + standard: + codex: {model: "", effort: ""} + claude: {model: "", effort: ""} + gemini: {model: "", effort: ""} economy: - codex: {model: "gpt-5.6-terra", effort: "low"} - claude: {model: "claude-haiku-4-5"} - default_tier: "economy" + codex: {model: "", effort: ""} + claude: {model: "", effort: ""} + gemini: {model: "", effort: ""} + default_tier: "standard" ``` - `team_harness_api_base` is normalized by loopy-loop: trailing slash stripped, `/v1` appended when missing. @@ -350,7 +416,6 @@ must_follow: null not_before_iteration: 0 run_on_start: false run_after_successes: null -emits_goal_check: false description: "" ``` @@ -373,36 +438,35 @@ run_after_successes: every: 10 ``` -- `emits_goal_check=true` lets a non-`goal_check` workflow write - `goal_check.json` as an eval artifact. Stopping still requires updating - session `control.json`. - -Each workflow set may declare `contract.yaml`. It names the layer kind, every -workflow role's responsibility, accountable state paths, eval author/runner/ -goal-control roles, task-acceptance owner, terminal-blocker reporting roles, -and whether the set uses the recursive child interface. This is accountability -metadata and prompt context, not a filesystem ACL or semantic scheduler gate -(D8). All built-in templates declare `session_protocol_version: 2`; an explicit -contract that omits the field also selects v2. An older -custom set without a contract receives a conservative derived protocol-v1 role -contract and remains executable; add and validate an explicit v2 contract -before expecting evidence-bound terminal control and child requests. +Each workflow set declares `contract.yaml`. It names the layer kind, every +scheduled role's responsibility and accountable state paths, the one +orchestrator that owns planning/handoff/completion, optional advisory eval +authors and runners, terminal-blocker reporting roles, and whether the set uses +the recursive child interface. This is accountability metadata and prompt +context, not a filesystem ACL or semantic scheduler gate (D8). The stock sets +declare `session_protocol_version: 3`; the coordinator validates that the same +orchestration role owns plan, handoff, and successful completion. Historical +custom v1/v2 contracts remain resumable under the version frozen in their +session. ## Output and Logging Each coordinator run creates a root under `.loopy_loop/sessions/`. Recursive workflow sets may create child sessions beneath it, but one worker still advances only the deepest active session. Session files hold compact durable -truth: scoped goals, progress and decisions, task/recovery state, eval and -control receipts, child handoffs, and git/delivery evidence. +truth: scoped goals, the semantic state spine, workflow and capability rosters, +attempt scheduler views, task/recovery state, optional eval receipts, child +handoffs, control, terminal outcomes, and git/delivery evidence. Each attempt freezes its workflow sources and `assignment.json`. The assignment identifies the exact repository/session/workflow/attempt and gives -the harness coordinator absolute paths to its own state and outputs. Durable -receipts use logical `session:/`, `parent:/`, `root:/`, and `trace:` references -so a stopped checkout can move. Team-harness gives every direct spawn its own -absolute assignment/output paths and dynamic delegated task; the original -coordinator remains accountable for integrating the result. +the harness coordinator absolute paths to its own layer state and outputs, +along with compact roster and scheduler context. Durable receipts use logical +`session:/`, `parent:/`, `root:/`, and `trace:` references so a stopped checkout +can move. Team-harness gives every direct spawn its own absolute +assignment/output paths, current layer identity, dynamic delegated task, and +inherited capability roster; the original coordinator remains accountable for +integrating the result. Detailed observable execution goes to the separately gitignored `.loopy_loop/traces//sessions//attempts//` tree. It @@ -415,24 +479,52 @@ known; it does not mean the attempt succeeded semantically. Use `loopy status` and `loopy events` for compact progress, and `loopy traces list` plus `loopy traces inspect` for attempt detail. The complete artifact and writer/reader reference is [docs/session-layout.md](docs/session-layout.md). -Legacy v1 files remain readable on resume but are not the fresh-session write -contract. +Fresh stock workflow sets use protocol v3. ## Control and Completion Each session evaluates its own scoped goal. A delivery child may prove its task while its parent still needs integration or release work. In the packaged -`inner_outer_eval` set, `outer` accepts task evidence, `eval_reviewer` authors -LLM-as-judge checks, and only `eval_runner` may publish successful terminal -control after a matching same-session eval receipt. Readiness informs prompts; -it does not become a semantic scheduler gate. +`inner_outer_eval` set, `outer` accepts task evidence and owns completion. In +`pm_planner_dispatcher`, `planner` independently accepts child outcomes and +owns program completion. Eval roles and dynamically spawned delegates report +evidence to those orchestrators; they cannot complete another role's layer. + +Successful protocol-v3 control is identity-bound to the exact current attempt +and must come from the completion role frozen in the workflow contract: + +```json +{ + "schema_version": 3, + "control_id": "control-goal-met-id", + "state": "stopped", + "reason": "Why this scoped goal is complete", + "stop_reason": "goal_met", + "producer": { + "session_id": "session-id", + "workflow_id": "outer", + "attempt_id": "attempt-id" + }, + "evidence_refs": [], + "eval_receipt_refs": [], + "handoff_ref": "session:/project_state/handoff.json", + "created_at": "2026-07-17T12:00:00Z" +} +``` -The one D5 escape hatch is an identity-bound v2 `unresolvable_error`. It is for +The three evidence fields are optional. An orchestrator does not need an eval +to complete the goal. A missing, malformed, stale, non-passing, or conflicting +advisory eval from an otherwise completed harness attempt is recorded as a +diagnostic; it does not consume workflow-failure budget or become an engine +veto. If control cites an artifact, however, its reference, hash, subject, and +accepted provenance must be truthful. + +The one D5 escape hatch is an identity-bound v3 `unresolvable_error`. It is for a genuinely terminal blocker after autonomous alternatives are exhausted: ```json { - "schema_version": 2, + "schema_version": 3, "control_id": "control-blocker-id", "state": "stopped", "reason": "specific terminal blocker", @@ -449,14 +541,14 @@ a genuinely terminal blocker after autonomous alternatives are exhausted: ``` Both successful and blocker control must identify the exact current -session/workflow/attempt. A delegate reports its conclusion to the harness -coordinator; it cannot publish a durable decision for another layer or a later -attempt. Invalid v2 control is archived with repair diagnostics instead of -being treated as semantic failure. Repeated broken protocol or workflow -execution is bounded by the configured failure caps. +session/workflow/attempt. A blocker producer must be authorized by the frozen +contract and list the autonomous routes already tried. Invalid control is +archived with repair diagnostics instead of being treated as semantic truth. +Repeated broken protocol or workflow execution is bounded by the configured +failure caps. -The exact eval receipt, goal-check projection, successful control, blocker, and -rejection rules are in [docs/http-contract.md](docs/http-contract.md). +The exact eval-receipt acceptance, successful control, blocker, and rejection +rules are in [docs/http-contract.md](docs/http-contract.md). ## Workflow Sets and Child Sessions @@ -474,34 +566,40 @@ unique `*.json` file under the assignment's absolute planner/dispatcher, and deeper trees use one state machine. Only the deepest session runs an assignment; every ancestor is suspended on one child. -The v2 request carries parent provenance, a child-scoped goal contract, and -hashed input references. The coordinator archives the accepted request and -copies each verified input into the child's immutable `inputs/` area. Child -attempts receive those logical references, hashes, and absolute local paths; -later parent edits cannot change accepted work. +The child-request payload currently uses schema version 2. It carries parent +provenance, a child-scoped outcome and completion criteria, and hashed input +references. The coordinator archives the accepted request and copies each +verified input into the child's immutable `inputs/` area. Child attempts +receive those logical references, hashes, and absolute local paths; later +parent edits cannot change accepted work. -The stock PM dispatcher first freezes the selected work item and its planning +The stock PM dispatcher first freezes the selected milestone and its planning evidence under `project_state/dispatch_inputs/.json`, then hashes that immutable snapshot into the request. Only after the request is atomically -published does it update the mutable `work_items.md` ledger. Hashing the ledger -itself would invalidate the request when that required status update occurs. +published does it add the factual request/lifecycle link to the stable task +record. Mutable plan and task prose are never used as hashed child inputs. -When the child stops, the engine writes a factual outcome. The parent then -writes a separate acceptance, rework, or reroute decision after reviewing the -evidence. Terminal descendants unwind iteratively, so the same edge supports -three or more active depths without a depth-specific scheduler. Invalid -requests are archived with reasons and can be repaired autonomously. +When the child stops, the engine writes its topology-neutral +`session_outcome.json` and links that result from the parent's factual child +outcome. The parent then writes a separate acceptance, rework, or reroute +decision after reviewing the child's handoff and evidence. Terminal +descendants unwind iteratively, so the same edge supports three or more active +depths without a depth-specific scheduler. Invalid requests are archived with +reasons and can be repaired autonomously. The packaged `pm_planner_dispatcher` workflow set uses this contract for PM orchestration: -- `planner` maintains PM state, selects one work item, reviews terminal child - evidence, and owns parent-acceptance/eval-readiness receipts. -- `dispatcher` freezes the selected item into an immutable request input, - publishes the v2 child assignment, and tracks factual lifecycle evidence +- `planner` maintains the high-level phase/milestone plan, selects one outcome, + reviews terminal child evidence, owns parent acceptance and the rolling + handoff, and decides completion. +- `dispatcher` freezes the selected outcome into an immutable request input, + publishes the schema-v2 child request, and tracks factual lifecycle evidence without deciding acceptance. -- `eval_reviewer` and `eval_runner` evaluate the PM layer's own broader goal; - only that layer's `eval_runner` may request successful terminal control. + +The planner can run prepared final evaluations named by the target goal or +delegate other program-level review when useful. These are planner inputs, not +additional scheduled PM roles. ## HTTP Contract @@ -512,10 +610,11 @@ The coordinator exposes exactly two endpoints: Both return a `TaskResponse` with `action` equal to `"run"` or `"stop"`. -Fresh v2 registration binds worker protocol/capabilities to the absolute -checkout and stable repository ID. A missing capability returns HTTP 426 -without advancing state. A run response identifies the frozen -session/workflow/attempt assignment; completion must echo its worker, +Fresh stock registration requires worker protocol 3 and the declared Loopy and +team-harness capabilities, then binds the worker to the absolute checkout and +stable repository ID. A missing capability returns HTTP 426 without advancing +state. A run response identifies the frozen session/workflow/attempt assignment +and its absolute protocol-v3 context; completion must echo its worker, repository, attempt, and assignment hash. Stale or mismatched completion cannot mutate current work. Durable local result and pending-completion records allow the next registration to recover an interrupted acknowledgement exactly once. @@ -531,11 +630,12 @@ and compatibility rules are in [docs/http-contract.md](docs/http-contract.md). loopy init [--template default|inner_outer_eval|pm_planner_dispatcher] ``` -Scaffolds loopy-loop files. The default template creates only the reserved -`goal_check` workflow. `inner_outer_eval` creates the recommended outer/inner/eval -workflow set. `pm_planner_dispatcher` creates planner/dispatcher workflows for -child-session orchestration — and also ships the `inner_outer_eval` child set -its dispatcher spawns, so a clean init is executable end to end. +Scaffolds loopy-loop files. The compatibility `default` template creates only +the historical `goal_check` workflow. `inner_outer_eval` creates the +recommended protocol-v3 outer/inner/advisory-eval workflow set. +`pm_planner_dispatcher` creates the two-role planner/dispatcher program layer +and also ships the `inner_outer_eval` child set its dispatcher spawns, so a +clean init is executable end to end. ```bash loopy coordinator --host 0.0.0.0 --port 8080 [--resume] [--workflow-set NAME] [--goal-file PATH] diff --git a/design/analysis/antigravity-orchestrator-owned-completion-review.md b/design/analysis/antigravity-orchestrator-owned-completion-review.md new file mode 100644 index 0000000..9186c36 --- /dev/null +++ b/design/analysis/antigravity-orchestrator-owned-completion-review.md @@ -0,0 +1,54 @@ +# Antigravity review: orchestrator-owned completion design + +**Reviewer:** Antigravity AI Coding Assistant + +**Initial review:** 2026-07-17 + +**Final re-review:** 2026-07-17 + +**Scope:** D3/D4/D6/D8–D12 amendment, protocol-v3 compatibility, +standalone/nested role ownership, semantic state/handoff, absolute paths, +optional eval provenance, and cross-harness collaboration + +**Status:** historical design review; the binding designs and decision log are +canonical + +## Final verdict + +**PASS.** The final re-review found no unresolved blocker or correction. It +confirmed orchestrator-owned completion, optional eval evidence, exact v1/v2/v3 +migration, standalone outer plan ownership, phase/milestone PM dispatch, upward +handoff/fallback, absolute path context, concrete schedule/capability rosters, +standardized strength tiers, non-enforced cross-harness review guidance, +trace-independent accepted receipts, and unchanged D5 autonomy. + +## Initial findings and disposition + +| Finding | Final disposition | +| --- | --- | +| Child immutable inputs had no named absolute Assignment path. | Added `layer_inputs` plus stable optional parent/request keys. | +| The control example made `handoff_ref` look mandatory despite missing handoff being diagnostic. | Made handoff/eval/evidence refs explicitly optional and defined terminal fallback/completeness states. | +| Receipt validation could remain hard-coded to `eval_runner`. | Required validation against frozen `check_runner_roles`, including outer/planner direct coordination. | +| Terminal outcomes could lose delivery evidence from earlier attempts. | Required session-wide delivery resolution with original attempt provenance. | +| Protocol v3 lacked explicit worker capability negotiation. | Added named v3 capabilities and HTTP 426 fail-fast before mutation/dispatch. | + +The review also suggested making malformed handoff non-fatal, naming parent +context keys, and showing rejected-control/protocol-failure directories; those +changes were incorporated. + +## Post-review tier amendment + +During the review, Antigravity suggested automatically mapping a custom +`strong` tier to `frontier`. That suggestion was not adopted: configuration is +still never silently rewritten. After the PASS, the owner clarified that +`strong` is not a custom compatibility alias but one of four canonical strength +tiers. The binding design now uses `frontier`, `strong`, `standard`, and +`economy`; an illustrative Anthropic mapping is Fable-, Opus-, Sonnet-, and +Haiku-class respectively. Sparse or provider-specific configuration remains +visible as configured rather than being reinterpreted. + +## Verdict boundary + +This PASS covers the binding documentation only. It does not approve the +future protocol-v3 implementation or coordinated loopy-loop/team-harness +release. diff --git a/design/analysis/antigravity-protocol-v3-implementation-review.md b/design/analysis/antigravity-protocol-v3-implementation-review.md new file mode 100644 index 0000000..749df60 --- /dev/null +++ b/design/analysis/antigravity-protocol-v3-implementation-review.md @@ -0,0 +1,52 @@ +# Antigravity review of the protocol-v3 implementation + +Status: external implementation review, resolved before release + +Date: 2026-07-17 + +Reviewer: Antigravity CLI, multi-hour print timeout + +## Review scope + +Antigravity performed a read-only adversarial review of the protocol-v3 Loopy +changes and Team Harness capability-roster transport. It inspected the full +working tree and ran tests while the terminal-outcome repair was still being +edited, so one report describes a transient intermediate state. + +## Findings and disposition + +### Protocol-3 stale completion was mistaken for a legacy worker — fixed + +In `_finish_assignment_locked()`, the no-current-task retry branch required the +remembered worker contract to equal version 2. A correctly registered +protocol-3 worker therefore failed the same safety check intended to reject +unregistered legacy workers. + +The check now accepts any validated version 2 or newer contract and continues +to reject missing or version-1 handshakes. The error message is version-neutral, +and `test_v3_stale_finished_accepts_valid_v3_worker_handshake` exercises the +previously rejected path. + +### `JsonValue` resolution — transient import issue fixed; rebuild unnecessary + +Antigravity observed a transient tree in which +`AcceptedTerminalControlSnapshot` referenced Pydantic's `JsonValue` without an +import, producing schema-resolution failures. The final implementation imports +`JsonValue`. An explicit `LoopState.model_rebuild()` was suggested as well, +but is unnecessary once the type is present in module globals: +`LoopState.model_json_schema()` and the focused suite both complete without it. +Adding a redundant global rebuild would obscure rather than strengthen the +model contract, so it was not adopted. + +### Keyword-only helper style — fixed + +`_is_full_sha256()` now declares `value` as keyword-only, matching its existing +call sites and the repository's named-argument convention. + +## Final assessment + +The real compatibility and style findings were fixed and regression-tested. +The schema finding was resolved at its cause; the extra rebuild suggestion was +rejected based on direct schema generation and test evidence. No additional +architecture or feature work was introduced. + diff --git a/design/analysis/claude-code-orchestrator-owned-completion-review.md b/design/analysis/claude-code-orchestrator-owned-completion-review.md new file mode 100644 index 0000000..7c0c045 --- /dev/null +++ b/design/analysis/claude-code-orchestrator-owned-completion-review.md @@ -0,0 +1,46 @@ +# Claude Code review: orchestrator-owned completion design + +**Reviewer:** Claude Code (`claude-opus-4-8`, `xhigh`) + +**Initial review:** 2026-07-17 + +**Final re-review:** 2026-07-17 + +**Scope:** current code anchors and the binding D3/D4/D6/D8–D12, +protocol-v3, state/handoff, schedule/capability, prompt, and compatibility +amendments + +**Status:** historical design review; the binding designs and decision log are +canonical + +## Final verdict + +**PASS.** Claude found the architecture coherent and confirmed all requested +properties against the settled documents and current code anchors. The final +pass reported no blocking architecture issue. + +## Findings that shaped the final design + +| Finding | Final disposition | +| --- | --- | +| The older fast-path proposal asserted the eval-owned D11 invariant that the amendment replaces. | Marked the proposal superseded and historical without deleting its analysis. | +| Outer appeared able to author, run, and cite its own checks without explaining D4 independence. | Clarified that role identity is durable accountability; delegate/judge provenance and cross-family review provide independence as guidance, not an engine role-name gate. | +| Existing `session_protocol_version >= 2` branches would accidentally route v3 into v2 eval-gate validation. | Required exact version dispatch and a test that v3 never enters v2-only validation. | +| Inner still had a prompt path to create the layer plan; flat state accountability could not express owner versus contributor. | Required removing inner plan bootstrap and adding per-artifact owner/contributor metadata. | +| The fate of `eval_readiness/` was unspecified. | Retired it for v3 into `eval_state.md` plus scheduler context while preserving frozen v2 readers. | +| Durable plan state referred to an absolute task path. | Corrected it to a portable logical task ref resolved to an absolute attempt path. | +| The D5 blocker field was accidentally renamed and PM blocker roles were omitted. | Retained `terminal_blocker_reporting_roles` and declared `[planner, dispatcher]` for PM. | +| A contract file omitting a protocol version was an undocumented middle migration case. | Permanently pinned that compatibility default to v2; only explicit v3 opts in. | +| One recursive-design sentence could imply non-empty evidence refs. | Made evidence refs optional/empty everywhere; only asserted refs are validated. | + +Additional accepted improvements include a single accountable writer for +`eval_state.md`, monotonic handoff revisions as diagnostic structure, complete +canonical tier examples, explicit v1/v2 singular versus v3 plural eval receipt +fields, session-wide delivery lineage, and accepted-receipt validation that +does not depend on later retention of raw trace bytes. + +## Verdict boundary + +This PASS covers the revised architecture documents. It does not certify the +future implementation, migrations, tests, releases, or the Ultimate Memory +workflow upgrade. diff --git a/design/analysis/claude-code-protocol-v3-implementation-review.md b/design/analysis/claude-code-protocol-v3-implementation-review.md new file mode 100644 index 0000000..a4bc965 --- /dev/null +++ b/design/analysis/claude-code-protocol-v3-implementation-review.md @@ -0,0 +1,60 @@ +# Claude Code review of the protocol-v3 implementation + +Status: external implementation review, resolved before release + +Date: 2026-07-17 + +Reviewer: Claude Code, `claude-opus-4-8`, `xhigh` + +## Review scope + +Claude reviewed the protocol-v3 Loopy implementation and the capability-roster +transport in Team Harness against the binding design in +`design/designs/orchestrator-owned-completion-and-cross-harness-review.md`. +The review was read-only and asked for concrete blocker, high, and material +medium findings rather than new product scope. + +## Findings and disposition + +### Every v3 terminal child needs one outcome shape — fixed + +Claude found that `coordinator_app.py` originally built +`session_outcome.json` only from an accepted terminal `control.json`. A child +that ended through an engine lifecycle reason such as `max_turns`, +`workflow_failure_cap`, or `stop_requested` therefore could not be projected +to its parent. Stack unwind could raise instead of handing the factual child +result upward. + +The implementation now gives every v3 terminal lifecycle the same +`SessionOutcome` shape. Control-owned stops include the accepted control hash; +engine-owned stops use `control: null` and a factual `engine_stop_reason` +fallback. `_ensure_session_outcome()` is called before child projection and +parent unwind. `test_v3_non_control_child_stop_writes_outcome_and_resumes_parent` +exercises the workflow-failure path through a real child ledger. + +### Negative provenance coverage was too thin — fixed + +Claude asked for adversarial coverage showing that plausible-looking files do +not become trusted evidence merely because they exist. The focused v3 suite +now covers false completion authority, stale attempts, unaccepted or changed +eval evidence, false handoff attempt provenance, exact child-outcome linkage, +and restoration from engine-owned terminal snapshots after mutable files are +changed. + +These checks validate identity and provenance only. They do not add semantic +gates or make evaluation mandatory. + +### Claimed missing assignment-builder docstring — no change + +The review reported that `build_attempt_assignment()` lacked a docstring. The +reviewed tree already contained the docstring “Build the identity-bound, +absolute-path envelope for an attempt.” No code change was needed. + +## Final assessment + +The blocker and material coverage gap were accepted and repaired. The docstring +finding was not reproducible. The resulting change remains within the accepted +design: durable orchestrators own completion, evaluation is advisory, the +engine validates provenance, and parent acceptance stays separate from child +completion. + diff --git a/design/analysis/historical-prompt-layer-planning-and-handoff-review.md b/design/analysis/historical-prompt-layer-planning-and-handoff-review.md new file mode 100644 index 0000000..c2b1903 --- /dev/null +++ b/design/analysis/historical-prompt-layer-planning-and-handoff-review.md @@ -0,0 +1,929 @@ +# Historical prompt review: layer planning, eval autonomy, goal granularity, and handoff + +**Status:** architecture analysis; non-binding working notes + +**Date:** 2026-07-17 + +**Scope:** the historical and current `inner_outer_eval` and +`pm_planner_dispatcher` prompt contracts, the live Ultimate Memory double-loop +session, and the requirement that `inner_outer_eval` remain independently +executable as a one-layer system + +**Accepted resolution:** The binding design chose an explicit protocol v3 for +new sessions so live v2 sessions keep their frozen authority. References below +to “changing v2” or removing a v2 prerequisite describe the semantic change +that v3 must implement; they do not authorize reinterpreting an existing v2 +session in place. + +## Executive conclusion + +The current run did not independently choose the wrong abstraction level. It +faithfully followed prompts that told the PM layer to dispatch one work package +or a similarly narrow maintenance/closeout item, then gave the child delivery +layer a nearly leaf-level assignment. That left too little meaningful planning +for the child's `outer` role to do. + +The original one-layer prompts were clearer about the important semantic split: + +- `outer` owned the durable high-level plan, decomposition, task lifecycle, and + acceptance for its session; +- `inner` implemented exactly one available leaf and returned it for review; +- the plan and leaf specifications had named, visible paths and stable status + transitions; and +- the same workflow could take a broad root goal and drive it to completion + without a PM parent. + +The recursive v2 work correctly strengthened session identity, absolute paths, +parent/child provenance, eval provenance, and trace separation. In compressing +the old prompts, however, it made the semantic planning contract implicit and +also overcorrected evaluation ownership. The stock `inner_outer_eval` outer +still claims planning ownership, while its inner is also told to create a plan +when one is absent. Its workflow contract does not declare a plan, task tree, +eval-status index, or semantic handoff as accountable state. + +V2 completion is now unnecessarily eval-gated. `eval_runner` is the only role +allowed to publish `goal_met`, and the engine requires a passing same-attempt +goal-check projection and eval receipt. That is the wrong semantic boundary. +The persistent layer orchestrator—`outer` for `inner_outer_eval`, `planner` for +`pm_planner_dispatcher`—should decide when its goal is complete. Evals are +optional, provenance-rich observations that the orchestrator may run directly, +return so a scheduled eval role can produce for its next attempt, or decide are +unnecessary. “Optional” here means optional to the Loopy protocol; a repository +goal can still instruct the orchestrator to run a particular prepared eval +before declaring completion. + +There is also a concrete upward-communication defect. The coordinator already +looks for `project_state/handoff.json` and places its reference in the factual +parent-side child outcome, but no packaged prompt or workflow contract requires +any role to create that file. The live Ultimate Memory child therefore has +accepted work, decisions, eval evidence, and delivery evidence, but no standard +semantic result that tells its parent what was achieved, what was learned, what +remains, and what disposition it recommends. + +The recommended direction is to give every durable session layer the same +small, obvious semantic spine: a layer plan, current state, task details, +decisions, accepted-work ledger, eval-status summary, and semantic handoff. The +engine should scaffold and expose absolute paths to those artifacts and validate +their structural provenance. Agents should continue to decide the plan and its +meaning. This preserves D8: the engine makes state visible and accountable; it +does not parse the plan to decide what work is wise or prevent agents from +changing the repository. + +Most importantly, this contract must be topology-neutral. A root +`inner_outer_eval` session and the same workflow used as a child should have the +same planning, completion, and optional-evidence semantics. Nesting adds a +parent that consumes the session's outcome; it must not turn the workflow into +a different species. + +## Terminology used here + +The distinction in D10 remains essential: + +| Term | Meaning | +| --- | --- | +| Durable session layer | One loopy session with its own goal, plan, state, decisions, evals, attempts, and optional child | +| Workflow role | A scheduled role inside that session, such as `planner`, `outer`, `inner`, or `eval_runner` | +| Harness delegate | A dynamic, attempt-local agent spawned by a Team Harness coordinator | + +An `inner` workflow role is not a nested durable loop. A spawned Codex or Claude +agent is also not a durable child session. Only a typed child-session request +creates another durable layer. + +## Evidence reviewed + +This review compared: + +- the first packaged `inner_outer_eval` prompt at loopy-loop commit `17f72b8`; +- the mature pre-v2 prompt lineage through commits `0f662b2`, `ff8fa1c`, + `9efdb14`, `bc6d9ee`, `f6e2fa5`, and `b1e4fe8`; +- the first packaged PM workflow at commit `8259343`; +- the recursive layer contract introduced at commit `01a560d` and refined on + current `main`; +- Ultimate Memory's original standalone whole-roadmap setup at commit + `04a480e` and its double-loop conversion at commit `7a65bfe`; +- current stock workflow contracts and prompts under + `src/loopy_loop/templates/`; +- `build_attempt_assignment()` in `src/loopy_loop/assignments.py`; +- `_terminal_evidence_projection()` and child-outcome publication in + `src/loopy_loop/coordinator_app.py`; +- the workflow-owned state and child-outcome descriptions in + `docs/session-layout.md`; and +- the live Ultimate Memory parent session + `20260717_025626_91f946163870_495df8c0` and child session + `20260717_044051_85af1686ad7c_78755280`. + +The live session is evidence of the contract's consequences, not a claim that +one model's private reasoning should be exposed. The desired observability is +the plan, decisions, accepted facts, and handoffs that affect future work—not +hidden chain-of-thought. + +## Follow-up correction: evals inform the orchestrator; they do not own completion + +### The history confirms the distinction + +The first `inner_outer_eval` template at `17f72b8` did include outer, inner, +eval reviewer, and eval runner. Evals were sparse scheduled roles: the reviewer +ran at startup and after every ten successful inner runs; the runner could not +run before iteration ten and followed the reviewer every ten successful inner +runs. At first, its `goal_check.json` was the semantic stop signal. + +Commit `d84942d` later gave outer a second path to stop when accepted work +satisfied the full goal. That created ambiguous dual authority: outer could +decide from accepted implementation evidence, while eval runner could decide +from its check run. The correct repair is one completion owner, but it need not +be the evaluator. + +The first `pm_planner_dispatcher` template at `8259343` had exactly two roles: +`planner` and `dispatcher`. Planner owned the full-goal stop decision. Child +eval and `goal_check` artifacts were review evidence “when present,” and +dispatcher explicitly said planner owned goal-level stop. The user's memory is +therefore correct. + +Commit `01a560d` added parent eval reviewer/runner roles, changed both stock +contracts so `eval_runner` alone owned `goal_met`, and made an eval receipt a +protocol-v2 prerequisite for success. It also made evals much more frequent: +the stock reviewer runs before ordinary work and the reviewer/runner pair recurs +after every three successful inner or planner runs. + +### The current mandate is engine behavior, not merely prompt advice + +Several independent mechanisms now enforce evaluation: + +1. `ControlSignal.validate_stop_reason()` in `src/loopy_loop/models.py` rejects + every v2 `goal_met` without `eval_receipt_ref`. +2. `CoordinatorService._validate_v2_control()` requires the producer to equal + `contract.eval.goal_control_role`, requires a passing same-attempt + `goal_check.json`, and validates a matching passing same-attempt eval receipt + and report/git provenance. +3. Both packaged contracts assign that role to `eval_runner`; outer and planner + prompts explicitly forbid successful control. +4. `emits_goal_check` turns missing or invalid evaluator output into a failed + iteration and can eventually terminate the session as `goal_check_broken`. +5. The PM template now schedules parent eval roles even though each child has + its own eval capability and the planner can run program-level prepared evals + directly when useful. + +This is stronger than “evaluation is available.” It makes one evaluation shape +the mandatory arbiter of semantic completion and can turn a broken advisory +observation into a terminal failure of otherwise recoverable orchestration. + +### Revised ownership + +Each workflow set should name one durable orchestration role whose continuity +lives in session state rather than a persistent model process: + +- `inner_outer_eval`: `outer` owns the layer plan, leaf acceptance, handoff, + and completion decision; +- `pm_planner_dispatcher`: `planner` owns the program plan, child acceptance, + final evidence review, handoff, and completion decision; and +- future recursive layers: the contract names the analogous orchestration role + rather than assuming a role whose name contains `eval`. + +The workflow schema should represent this directly. Completion ownership does +not belong under `eval.goal_control_role`; use a top-level orchestration or +completion owner. The eval sub-contract then names only optional author/runner +roles. V2 `goal_met` keeps exact current session/workflow/attempt identity and a +nonblank rationale, while eval receipt references become optional evidence. + +Eval reviewer/runner roles may still be useful in `inner_outer_eval`. They +author and run independent observations on a schedule, update compact eval +state, and publish receipts for a later outer attempt to consume. They never +publish terminal control. Harness coordinators remain attempt-local: outer does +not stay alive waiting across scheduled roles. Its current attempt returns, the +scheduler runs any due evidence role, and a later outer attempt reads the +durable result. Outer can: + +- run an eval itself or delegate it when immediate evidence is useful; +- avoid duplicating that work when schedule context says an eval role is due; +- return so the scheduled role can run, then use its result in a later attempt; +- repair work after a meaningful failure; +- decide that a check is stale or inapplicable and record why; or +- conclude the goal from other evidence without running an eval. + +Eval provenance remains strict when an eval is accepted or cited as trustworthy +evidence. Provider, model, check definitions, report hashes, git subject, and +producer identity should still be recorded. Those facts make the observation +trustworthy; they do not make it a hard terminal gate. Unrelated malformed eval +output remains a diagnostic and cannot block completion. + +For UGM, the PM workflow should return to planner plus dispatcher. Its goal and +planner prompt should point at the prepared final checks under +`plan/implementation_evals/` and instruct planner to run them near overall +completion, directly or through a dynamic harness delegate. The result is +important program evidence because UGM's goal says so, not because Loopy refuses +to accept `goal_met` without a particular receipt. + +## What the original one-layer contract got right + +### 1. It named one durable planning owner + +The earliest packaged outer prompt opened with a plain contract: outer owns +high-level planning, the overall plan, current state, and review; inner +implements leaf tasks. The matching inner prompt said it implements exactly one +available leaf and must not invent a broad plan when no leaf exists. + +That division remained clear in the mature pre-v2 prompts: + +- outer created and refined the next useful slice of the plan; +- outer wrote acceptance criteria for available leaves; +- inner selected one available or failed leaf; +- inner could move only that leaf through implementation states; and +- outer alone accepted the leaf, updated dependencies, and added it to the + finished ledger. + +This did not constrain the Team Harness coordinator's delegation. Inner could +still choose a dynamic team for its selected leaf. The contract constrained +durable session ownership, not the shape of the ephemeral agent graph. + +### 2. It made the plan inspectable + +The old state layout explicitly named: + +```text +project_state/ +├── current_state.md +├── memory.md +├── what_we_have.md +├── decisions.md +├── eval_results.md +├── finished.md +└── what_we_should_do/ + ├── plan.md + └── tasks//README.md +``` + +The root plan stayed concise. Detailed context and acceptance criteria lived in +task files. Stable task markers made the handoff visible: + +```text +available +in progress +inner complete, waiting for outer +failed / needs repair +accepted +``` + +The exact vocabulary can improve, but the lifecycle itself is valuable. A +reviewer could tell what outer intended before implementation, what inner was +doing, and what outer had actually accepted. + +### 3. It used progressive disclosure rather than duplicating the whole roadmap + +The historical prompt did not require fully expanding every future task. It +kept the root plan readable, expanded only the next useful slice, and put +detailed criteria in leaf files. That is compatible with a repository roadmap +remaining authoritative. + +A session plan can be a compact execution projection containing stable IDs, +status, dependencies, current selection, and source references. It need not +copy the roadmap's prose or become a competing source of truth. + +### 4. It worked with a broad standalone goal + +The first template goal was a complete product outcome, not a preselected leaf. +Ultimate Memory's first one-layer setup similarly gave `inner_outer_eval` the +whole phase 0–8 roadmap and was configured for outer to decompose that goal and +inner to implement leaves. + +This shows that decomposition is part of the intended capability of +`inner_outer_eval`, not something that exists only in a PM parent. + +## What should not be restored from the old prompts + +The historical prompts also accumulated substantial material that should stay +retired: + +- a fixed mandatory Codex/Claude/Gemini team recipe; +- provider-specific orchestration boilerplate repeated in every role; +- an outer role that sometimes orchestrated implementation despite assigning + implementation to inner; +- dual successful-control authority split between outer and eval runner; one + declared orchestration role should own completion instead; +- `waiting_for_human`, contrary to D5; +- v1 control payloads and repo-root-relative state-path assumptions; +- truncation of the old user-update inbox; and +- stock agent-authored deterministic checks, contrary to D4. + +The lesson is to recover the concise planning and ownership protocol, not to +restore the old prompt wholesale. + +## Confirmed gaps in the current contract + +### F1. Durable planning ownership is ambiguous + +The stock outer prompt says it owns planning and `what_we_should_do/`. The stock +inner prompt then says: + +> If the state does not yet have a useful plan, create the smallest maintainable +> plan needed for this attempt. + +The stock `contract.yaml` compounds the ambiguity: it declares +`current_state.md`, `decisions/`, `finished.md`, and eval directories, but does +not declare `what_we_should_do/`, a plan file, or task specifications. + +This creates two possible planning owners: + +- outer can maintain durable cross-attempt decomposition; or +- inner can create an attempt-driven plan when it feels one is missing. + +The second behavior is useful as tactical planning but wrong as durable layer +ownership. Inner should be free to make an execution checklist for the +selected leaf. It should not silently become the session backlog owner. + +### F2. The plan path is inconsistent and not an Assignment-level contract + +Current PM sessions use `project_state/work_items.md`. The Ultimate Memory child +uses `project_state/what_we_should_do/plan.md`. The stock delivery contract does +not require either exact path. `build_attempt_assignment()` exposes the broad +absolute `project_state` directory but no named absolute layer-plan, task-tree, +eval-status, or handoff path. Session creation in `src/loopy_loop/sessions.py` +creates the project-state directory and `finished.md`, but does not scaffold a +plan, current-state file, eval index, or handoff. + +The live child eventually created +`project_state/what_we_should_do/plan.md`, but only after delivery work had +already completed. It is mostly a retrospective list of accepted steps plus the +remaining eval step. It did not show the child's intended decomposition before +the work happened. The fact that a capable agent later created a useful file +does not make the file a reliable protocol surface. + +### F3. The current Ultimate Memory goal deliberately dispatches at leaf scale + +Ultimate Memory's current root goal says the PM selects one work package at a +time and that each child implements exactly one WP or explicitly coupled WP +group. Its planner and dispatcher prompts reinforce this rule. The specific +child goal—“Execute exactly one evidence-backed `PLAN-RECONCILIATION` for Phase +0 WP-0.1”—is therefore the expected result of the prompt, not agent drift. + +At that granularity, the child outer can only break one prescribed edit into +minor execution steps. Its durable planning role is largely ceremonial. + +The repository phase files should remain planning authorities, but that does +not imply that one repository WP must equal one durable child session. A parent +can select a coherent milestone and reference the relevant phase/WP authority; +the child outer can then decide which leaves and PRs are needed to achieve it. + +### F4. The engine has an orphaned semantic-handoff reference, not a producer contract + +When a child becomes terminal, the coordinator writes +`child_outcomes/.json`. That factual record correctly separates +child lifecycle evidence from parent acceptance. + +`CoordinatorService._terminal_evidence_projection()` contains an optional +discovery hook for: + +```text +project_state/handoff.json +``` + +and, when present, places a logical reference to it in +`child_outcome.evidence_refs.handoff`. + +However, no current packaged `inner_outer_eval` role is required to create the +file; it is absent from its `contract.yaml`; no exact absolute handoff path is +provided in the Assignment; and the live Ultimate Memory child has no such +file. At review time the child had written successful control intent, but its +engine state and parent edge were still `running` because the coordinator had +been deliberately stopped before finalizing the transition; no child outcome +had been published. When that transition is finalized, the current projection +will contain `handoff: null` unless the artifact is repaired first. This is +also a useful distinction: workflow control intent and committed engine +lifecycle are not the same fact. + +The parent can reconstruct meaning by reading `current_state.md`, +`finished.md`, decisions, eval receipts, delivery receipts, and perhaps traces. +That is recovery by archaeology, not a solid inter-layer contract. + +There is a related cross-attempt delivery-lineage gap. The current +`_delivery_ref_for_attempt()` accepts only a delivery receipt whose attempt ID +equals the terminal control producer attempt. Normal delivery is performed by +an earlier inner/outer attempt, while successful terminal control is produced +by a later `eval_runner` attempt. The live delivery receipt and terminal eval +attempt have exactly that shape, so the projected delivery reference would +also be null. A correct terminal outcome must bind delivery through the +outer-accepted work/handoff lineage, not assume delivery and terminal eval were +performed by the same attempt. + +### F5. Ultimate Memory's customized delivery workflow is not standalone-safe + +The stock template mostly refers to “this session's scoped goal,” which is +correct at any depth. Ultimate Memory's customized copy instead assumes: + +- this layer owns one dispatched WP or maintenance item; +- the repo-root goal belongs to a PM parent; +- an accepted child request and frozen parent inputs exist; and +- eval must never substitute the PM/root goal for “this child.” + +Those statements are false when the same workflow set is launched as a root +one-layer session. In particular, the customized eval reviewer expects an +accepted request and dispatcher-selected curated inventory that a root session +does not have. + +This is not merely awkward wording. It makes the workflow's evidence inputs +conditional on a parent while its declared `child_interface: none` correctly +says the workflow itself needs no durable child. + +### F6. Suspended parent projections are expected, but inspection is poor + +The live parent `current_state.md` and `child_sessions.md` remained stale while +the child ran, whereas engine-owned `children.json` correctly showed the live +child. With the deliberate depth-first single-worker model, the suspended +parent agent cannot continuously rewrite its semantic projection. That is not +evidence that the engine lost the child. + +The inspection problem is still real: a reviewer opening the parent session +does not get one obvious route to the active layer's goal, plan, current state, +and eventual handoff. The remedy is a topology-aware status/index view that +links canonical artifacts for every session on the active stack—not a second +concurrent parent worker and not engine parsing of plan semantics. + +## Required standalone invariant for `inner_outer_eval` + +`inner_outer_eval` must be runnable both as: + +1. a root, one-layer system with `parent_session_id: null`; and +2. a child delivery session inside a double- or deeper-loop tree. + +The following invariants should hold in both cases: + +1. The session's own immutable `goal_contract` is always authoritative. +2. The workflow never requires a parent, accepted child request, PM backlog, or + parent-owned curated-check selection to exist. +3. If an accepted request and frozen inputs do exist, they provide scoped + origin context; they do not replace the session goal. +4. Outer bootstraps and maintains the durable layer plan before ordinary inner + work. +5. Inner consumes one selected leaf. It may make an attempt-local tactical + checklist and dynamically delegate, but it does not own broad layer + decomposition. +6. Outer reviews each leaf and alone marks it accepted in the layer plan and + finished ledger. +7. Eval reviewer and eval runner, when scheduled or invoked, evaluate this same + session goal and publish evidence for outer. Outer remains the only + successful terminal-control owner. +8. For a normal completion, outer publishes the same semantic handoff artifact. + For a child, the engine exposes it to the parent. For a root, + status/inspection presents it with terminal control and any eval facts as the + operator-facing result. Missing handoff is visible incompleteness for later + review, not an engine veto over the orchestrator's completion decision. A + crash, failure-cap stop, or legitimate D5 blocker may also terminate without + an outer-authored handoff. +9. `child_interface: none` remains valid. Dynamic Team Harness delegates are + attempt-local and do not require another durable session. +10. Prompt language is topology-neutral: “if this session has a parent,” never + “the PM parent” as an unconditional fact. + +This invariant deserves an end-to-end contract test. Existing initialization +and parent-dispatch tests do not prove that a fresh root `inner_outer_eval` +session can bootstrap a plan, implement and accept leaves, decide completion of +its own broad goal with or without eval evidence, publish its outcome, and close +successfully without any parent artifacts. + +## Recommended semantic state contract + +Every durable session should expose the same obvious state surface, regardless +of depth or workflow-set names: + +```text +project_state/ +├── plan.md +├── tasks/ +│ └── .md +├── current_state.md +├── memory.md # optional, concise durable facts only +├── decisions/ +├── eval_state.md +├── finished.md +└── handoff.json +``` + +`plan.md` is the layer's execution projection. It should contain or link: + +- the layer goal reference and hash, without restating a competing goal; +- stable item IDs and outcome-oriented descriptions; +- authority/source references into repository plans and designs; +- dependencies and observable acceptance criteria; +- status and the currently selected leaf; +- enough near-term lookahead to understand the intended direction; and +- the next expected role/action. + +It should not duplicate an entire repository roadmap. Detail belongs in +`tasks/.md`, and speculative distant work can remain unexpanded. + +`eval_state.md` is a compact human-readable evidence index maintained by eval +roles or by the orchestrator when it runs an eval directly. It links the active +check inventory, latest validation/run/receipt, important coverage gaps, and +next eval action. It does not replace canonical checks or receipts and does not +turn an eval result into terminal authority. + +`handoff.json` is the layer's semantic outcome. A minimum useful shape would +identify: + +- session and goal identity; +- the plan and accepted-work ledger references; +- achieved outcomes and accepted item IDs; +- important findings and decisions; +- unachieved scope and remaining risks; +- PR, commit, CI, eval, and delivery evidence references when present; and +- the producing role/attempt plus a recommended consumer disposition such as + `accept`, `rework`, or `reroute`. + +The recommendation is advice, not parent acceptance. The parent planner still +reviews the child goal, repository state, eval evidence, and handoff before +publishing a separate `parent_acceptance` receipt. + +During a running session, outer may atomically update `handoff.json` as a +rolling semantic draft. At terminalization, the engine should snapshot or hash +the exact observed bytes into the immutable terminal session outcome. The +parent then consumes that bound terminal version rather than an unversioned file +that could later drift. If no handoff exists, the outcome records that factual +absence without blocking the orchestrator's decision. + +The handoff alone is not the complete terminal result. Terminal control and any +scheduled eval observation may be produced in different attempts. The engine +should produce a compact terminal session-outcome projection for every +session—root or child—that combines lifecycle/control facts, the semantic +handoff reference, optional eval and git evidence, validated cross-attempt +delivery lineage, usage, trace reference, and explicit completeness flags. A +parent's `child_outcome` can link or project that same record; root status can +display it directly. This makes standalone and nested completion the same +contract without making eval presence a completeness requirement. + +The semantic handoff also must not manufacture a new terminal state. If a +design-conflict report is a valid alternate child outcome, that alternative +must be explicit in the child goal contract. Otherwise an unfinished task is +rework, and a genuinely terminal human-only blocker remains D5's rare +`unresolvable_error` path. + +This gives a cold reviewer a different, appropriate view at each level: + +- the PM parent's `plan.md` shows which program outcomes or milestones it + expects to complete and why one is selected; +- the delivery session's `plan.md` shows how its outer role decomposed that + outcome into leaves and which leaves are accepted or next; +- `eval_state.md` shows what the eval roles intend to judge, what has run, and + what gap remains; and +- attempt traces show the inner coordinator's dynamic delegate graph and + tactical execution details without turning those transient choices into the + durable layer plan. + +## Role ownership within one session + +| Artifact or decision | Accountable owner | Allowed contribution | +| --- | --- | --- | +| Immutable layer goal | Engine/session creation | Roles read it; none rewrites it | +| Durable layer decomposition | `outer` or PM `planner` | Inner may propose follow-ups in its handoff | +| Selected-leaf lifecycle | `outer`/`planner` accountable | Inner may mark only its selected leaf in progress and ready for review | +| Tactical execution checklist | `inner` | Attempt-local; promote only durable facts or decisions | +| Leaf implementation and delegate integration | `inner` | Dynamic delegates report to inner | +| Leaf/child acceptance | `outer` or PM `planner` | Review delegates may advise but not decide | +| Eval check policy/inventory, when configured | `eval_reviewer` | Delegates may inspect evidence | +| Eval run/receipt evidence, when configured or directly invoked | `eval_runner` or the orchestrator | Evidence only; never owns terminal control | +| Session completion and `goal_met` | `outer` or PM `planner` | May cite eval evidence but does not require it at protocol level | +| Semantic layer handoff | `outer` or PM `planner` | Normal upward/operator communication; missing state remains visible | +| Factual terminal session outcome | Engine | Combines handoff with terminal control/git, optional eval, and cross-attempt delivery facts | +| Factual child outcome | Engine | Links/projects the terminal session outcome; makes no acceptance decision | +| Parent acceptance | Parent `planner`/acceptance role | Child recommendation is evidence only | + +The current workflow-contract schema may need more than one undifferentiated +`accountable_roles` list to express “outer owns decomposition; inner may change +only the selected leaf's lifecycle.” This should remain an accountability and +review contract, not a path-level write fence. + +## Correct goal granularity across layers + +There is no universally correct number of files, tasks, WPs, PRs, or iterations +per child. A useful boundary is semantic: + +> A child-session goal should describe a coherent, independently evaluable +> outcome that admits more than one reasonable leaf decomposition. If it +> already dictates the exact single edit or reconciliation step, it probably +> belongs in the child's plan as an inner leaf. + +For Ultimate Memory, the current scopes look roughly like this: + +| Layer | Better responsibility example | +| --- | --- | +| PM parent | Implement the complete design-backed Ultimate Memory program | +| Dispatched delivery session | Bring Phase 0 Foundations to an honestly development-ready state and satisfy its applicable exit criteria with merged evidence | +| Child outer plan | Reconcile convention authority, resolve remaining foundation gaps, deliver coherent PRs, verify CI, and close phase evidence | +| Child inner leaf | Reconcile the WP-0.1 stack-conventions authorities against merged scaffold evidence | +| Harness delegates | Inspect specific authorities, implement one bounded change, run checks, or review evidence for that leaf | + +UGM already provides the right rough parent projection in its phase spine +(phases 0 through 8). Those phase outcomes—not individual WP rows—are the +natural initial PM work items. This should remain planner judgment rather than +an engine law: the planner may split a phase whose outcome is too broad, combine +coupled outcomes, or route around dependency gates, provided it records the +reason. The dispatcher transports the chosen phase/milestone outcome; the +child outer owns its WP/leaf/PR decomposition. + +The example delivery goal still needs explicit observable completion criteria, +constraints, source references, and required evidence. “High-level” must not +mean vague. It means outcome-oriented rather than pre-decomposed into the one +edit the child outer should have selected itself. + +A future three-layer topology uses the same rule recursively. The root can +dispatch a program milestone to a middle coordinator; that coordinator can +dispatch an integrated feature outcome to a delivery session; the delivery +outer can decompose it into inner leaves. No fixed agent graph or semantic +scheduler is required. + +## Recommended upward flow + +The intended parent/child exchange should be legible as this sequence: + +1. Parent planner selects one milestone/outcome in its visible layer plan. +2. Dispatcher freezes that outcome contract and its exact input references; it + does not narrow the outcome into a leaf. +3. Child outer creates the child's visible plan and selects its first leaf. +4. Child inner dynamically executes one leaf and returns evidence. +5. Child outer accepts, repairs, or re-scopes; it updates the plan, finished + ledger, current state, and rolling semantic handoff. +6. Steps 3–5 repeat until outer judges that the child goal may be complete or + that more independent evidence would be useful. +7. Using the frozen schedule context, outer chooses whether to run an eval now, + leave the work for an imminent scheduled eval role, or decide without one. +8. Any scheduled eval role publishes an observation/receipt and yields the next + semantic turn to outer; it never closes the session. Missing or malformed + evidence is also returned as a diagnostic rather than retried until outer is + starved. +9. Outer integrates the available evidence, publishes its semantic handoff, + and alone decides whether to continue or write `goal_met` control. +10. At terminal completion, the engine writes the factual terminal session + outcome. For a child, it also exposes that outcome to the parent. +11. Parent planner independently accepts, requests rework, or reroutes and then + updates its own layer plan. + +For a standalone one-layer run, steps 1–2 and 11 have no parent. The same outer +plan, inner leaves, optional eval evidence, completion decision, and semantic +handoff remain meaningful. Root status/inspection presents the engine's +terminal session outcome—semantic handoff plus control/git/delivery and any +eval facts—to the operator instead of an ancestor agent. + +## Absolute-path and engine responsibilities + +The Assignment should expose exact absolute paths for the canonical semantic +artifacts, not only the enclosing `project_state` directory. At minimum: + +```text +layer_plan +layer_tasks +layer_current_state +layer_eval_state +layer_finished_ledger +layer_decisions +layer_handoff +session_state +workflow_roster +scheduler_view +``` + +These can be generated from exact entries in the frozen workflow contract. A +generic `contract_state_paths` map may scale better than adding one hard-coded +field for every future workflow artifact, provided the keys and paths are +stable and prompts name them unambiguously. + +The current Assignment names the selected workflow and its current config +snapshot. The workflow contract lists role responsibilities, but it does not +tell the coordinator which roles are enabled, their priorities and cadence, +their `must_follow`/`run_after_successes` relationships, recent schedule state, +or which role would probably follow this attempt. A child Assignment also lacks +a named absolute path to that child's own `state.json`. + +`workflow_roster` should therefore be a session-frozen full-set projection +containing every scheduled workflow role, responsibility, enabled state, +cadence, dependencies, and expected outputs. `scheduler_view` should be an +attempt-frozen projection containing recent factual history and an explicitly +conditional forecast such as: + +```text +would_select_next_if_this_attempt_completes_mechanically_without_control_or_child_request +``` + +The forecast is context, not a promise: terminal control, child dispatch, +failure, stop/budget/max-turn/failure-cap conditions, or new durable state can +preempt it. It lets a smart coordinator avoid running an expensive eval itself +when an eval role is already due. + +These are scheduled workflow roles, not all future “agents.” Loopy can know its +own roster. It cannot predict the dynamic harness delegates that the current +coordinator has not yet chosen, and it should not try. + +The engine may: + +- scaffold the files/directories; +- resolve and validate their absolute containment; +- freeze the workflow contract that declares role accountability; +- expose their paths in Assignments and status output; +- validate handoff identity/schema/hash structurally; +- link the handoff into the factual terminal session outcome and its child + projection; +- validate the current orchestrator's control identity and any evidence refs it + elects to cite; and +- surface absent/incomplete handoff, eval, and delivery lineage as factual + completeness information without vetoing the orchestrator's semantic + decision. + +The engine should not: + +- parse plan prose to select the next task; +- reject an iteration merely because plan prose is missing or semantically + weak—the outer/eval/parent review path owns that repair; +- require an eval receipt, passing goal-check projection, or handoff as a + protocol prerequisite for `goal_met`; +- turn an invalid advisory eval into terminal `goal_check_broken`; +- flip mechanical `HistoryEntry.success` or increment generic workflow-failure + counters merely because advisory eval output is missing or invalid; +- veto a semantically surprising but valid decomposition; +- infer acceptance from task status text; +- turn the plan into a path-level ACL; +- run a parallel parent agent merely to keep its Markdown fresh; or +- replace the parent's independent acceptance with the child's recommendation. + +This boundary preserves the core of D3, D4, D8, D10, and D12. D3 and D8 need +wording corrections described below, and D11's current mandatory-eval conclusion +must be replaced rather than cited as support. + +## Follow-up: preserve cross-harness review without restoring a fixed graph + +The historical prompts repeatedly used a useful causal pattern: one harness +performed research or implementation, then another harness reviewed the stable +artifact. Some versions also sent independent analysis to several harnesses in +parallel. This provided real diversity of tools and failure modes. + +The part not worth restoring is the vendor-specific choreography. Hard-coding +“Codex implements, Claude reviews, Gemini reviews next” makes availability and +model churn part of every workflow prompt, requires review even for trivial +work, and prevents a coordinator from adapting the team to live evidence. + +The replacement should expose two independent dimensions to every harness +coordinator: + +1. a session-tree-frozen roster of all enabled harness families; and +2. each family's configured models/efforts through semantic strength tiers, + using the stock vocabulary `frontier`, `strong`, `standard`, and `economy`. + +Prompts can then state a judgment preference rather than a graph: parallelize +genuinely independent analyses; for consequential artifacts, prefer review by +a different enabled harness family than the primary author; use a frontier +tier when the confidence gain justifies the cost; and let the accountable +coordinator synthesize disagreements. Parallel delegates should write separate +findings, not race to edit one canonical file. + +Eval-check creation deserves the strongest form of this guidance. Check design +shapes every later observation, and the historical eval-reviewer prompt still +assigned it to one hard-coded harness rather than using diversity. For a +non-trivial set, different families should independently analyze goal coverage +and failure modes in parallel, one integrator should draft the checks, and +different-family reviewers should attack the stable draft for gaps, +false-positive/negative paths, implementation coupling, gameability, ambiguity, +and evidence discoverability. The coordinator then publishes one coherent set. + +This is an upgrade over both the old and current prompts. It must remain +guidance: no required family, agent count, all-provider quorum, review receipt, +or model-tier gate. If only one family is available or the work is trivial, the +coordinator proceeds autonomously with a proportionate review shape. + +## State versus traces + +The desired visibility belongs in compact semantic state: + +- plan and selected work; +- current progress and blockers; +- decisions and alternatives; +- accepted work; +- eval inventory/headline and receipt links; +- semantic handoff; and +- factual child/parent acceptance records. + +Detailed coordinator turns, dynamic spawn plans, full prompts, tool I/O, raw +eval output, and verbose reviews remain in gitignored traces under D12. If an +attempt-local thought changes what future attempts should do, the responsible +role promotes the resulting decision or plan change into compact state. The +system should not copy entire harness transcripts into the session plan merely +to make reasoning observable. + +## Decision-log implications + +This direction conflicts with current binding text and therefore cannot be +implemented as a quiet prompt edit. + +### D3: retain the mechanical-success boundary; change the semantic arbiter + +`IterationResult.success` should continue to mean that the harness assignment +ran without a Loopy/Team Harness execution error. It must not infer work quality +from worker process exits. + +What changes is the next sentence in the architecture: the declared +orchestration role is the semantic arbiter and `control.json` is its stop +decision. Eval results are evidence available to that role; they are not the +only “real” semantic success mechanism. + +### D4: retain check policy, make its scope conditional on evaluation being used + +D4 can keep the stock LLM-as-judge policy and the distinction between +agent-authored checks and repository-owned suites. It governs how a stock eval +is authored and interpreted when an eval is run. It should not imply that every +session must run an eval or that an eval verdict owns terminal control. + +UGM's prepared repository evals remain legitimate evidence. Its planner is +explicitly instructed to run them near the end because they are part of UGM's +completion method. + +### D8: detection is evidence, not an unconditional semantic veto + +D8's core remains right: do not build path fences, semantic scheduling vetoes, +or approval gates. Its current claim that a detected check failure necessarily +blocks acceptance is too strong under orchestrator-owned completion. + +A failed or malformed eval should become visible evidence. The orchestrator +normally repairs the work or the check; it may also record that the observation +is stale, invalid, or irrelevant. The engine validates identities, schemas, +hashes, and topology, but it does not decide whether the semantic evidence is +sufficient. This preserves accountability without replacing model judgment +with an arbitrary programmatic rule. + +### D10 and D12: retain their boundaries and extend the compact context + +The durable-session versus dynamic-delegate distinction remains correct. So +does the state/evidence versus trace boundary. The schedule projection and eval +headline are compact state; raw evaluator prompts, transcripts, and reports +remain traces. D10 should additionally require the frozen workflow roster, +conditional scheduler view, absolute semantic-state paths, and enabled harness +capability roster. D12 should name those projections and the synthesized +cross-harness conclusions as compact state while leaving full parallel analyses +and reviewer transcripts in traces. + +### D11: replace the conclusion while retaining its useful provenance rules + +D11 should become approximately: + +> Every session owns its scoped goal and names one orchestration role that owns +> its completion decision. Eval roles are optional evidence producers. A child +> result never proves its parent's broader goal complete. When eval evidence is +> produced or cited, its session/goal/check/judge/git provenance is validated, +> but successful terminal control does not require it. + +For stock workflows, the completion owners are: + +- `inner_outer_eval`: `outer`; +- `pm_planner_dispatcher`: `planner`. + +Terminal-blocker producer identity and the D5 last-resort rules remain. A +spawned delegate still cannot publish durable control for its coordinator. + +## Suggested implementation sequence after design agreement + +This analysis is not itself a binding implementation decision. If the direction +is accepted, the smallest coherent follow-up is: + +1. Amend D3/D8/D11 and the binding recursive-layer design: name the orchestrator + as completion owner, make eval evidence optional, and add the universal + layer-plan, topology-neutral standalone, and semantic-handoff invariants. +2. Update workflow contracts: `outer` owns completion in `inner_outer_eval`; + restore `pm_planner_dispatcher` to planner plus dispatcher with planner as + completion owner; keep inner eval roles as optional evidence producers. +3. Remove the v2 same-attempt eval-receipt/goal-check prerequisite for + `goal_met`. Validate optional cited eval evidence, and turn invalid advisory + eval output into non-failing diagnostics/events. Remove `emits_goal_check` + from advisory runners or otherwise ensure bad advisory output neither flips + mechanical history success nor increments generic failure counters that can + starve outer and reach `workflow_failure_cap`. +4. Declare and scaffold canonical state paths, then expose those plus the + current session state, frozen full workflow roster, and conditional scheduler + view through every Assignment. +5. Restore the concise outer/inner planning lifecycle to the stock prompts, + removing inner's authority to invent a durable broad plan and teaching outer + how to use upcoming schedule context without duplicating evaluator work. +6. Add the engine-owned terminal session-outcome projection, bind handoff and + delivery through accepted cross-attempt evidence, and preserve explicit + completeness facts without making semantic artifacts terminal gates. +7. Add contract tests for standalone root `inner_outer_eval`, nested use, + orchestrator completion with and without eval evidence, advisory eval + failure, schedule awareness, terminal handoff, and independent parent + acceptance. +8. Update Ultimate Memory's parent plan to phase-sized outcomes, make its + delivery workflow topology-neutral, and instruct planner to run the prepared + final eval inventory near program completion before making its own decision. +9. Improve status/session inspection so it displays the active stack and the + absolute goal/plan/current-state/eval-state/handoff paths for every layer + without interpreting their semantics. + +## Final assessment + +The recursive state machine is not fundamentally pointed in the wrong +direction. Its identity, path, evidence-provenance, and trace contracts are a +strong base. The regression is narrower and important: in making recursion +precise, the implementation stopped making each layer's semantic plan and +outcome equally precise, then made one optional evidence mechanism the mandatory +completion authority. + +Recover the original outer/inner planning split, standardize visible state and +handoff paths at every depth, give the orchestrator full schedule context, make +evals advisory, and move parent dispatch goals up one abstraction level. Do that +without recovering the old prompt bloat or moving semantic judgment into the +engine. That produces a system in which agents remain autonomous, every layer +can be reviewed cold, a child can communicate upward without transcript +archaeology, and `inner_outer_eval` remains a complete one-layer system in its +own right. diff --git a/design/decisions.md b/design/decisions.md index 9a517cd..8c0b40a 100644 --- a/design/decisions.md +++ b/design/decisions.md @@ -10,9 +10,13 @@ Companion docs: - `design/designs/long-running-loop-reliability.md` — the self-contained description of the crash recovery, telemetry, failure containment, and model-tier mechanisms already implemented under several decisions below. -- `design/designs/recursive-loop-layer-contract.md` — the implemented contract - for recursive session layers, dynamic harness delegation, layer-scoped - evaluation, and the state/evidence-versus-trace boundary (**D10–D12**). +- `design/designs/recursive-loop-layer-contract.md` — the implemented v2 recursive, + provenance, delegation, and state/trace baseline plus its accepted v3 amendment + boundary (**D10–D12**). +- `design/designs/orchestrator-owned-completion-and-cross-harness-review.md` — the + accepted v3 contract for orchestration ownership, semantic state/handoff, optional + evaluation, schedule/capability context, phase-sized PM dispatch, and cross-harness + review. It is implemented in loopy-loop 0.8.0 and team-harness 0.5.4. - `design/proposals/` — forward-looking changes we are *considering* (not decided; do not treat them as binding or as descriptions of current behavior). - `design/analysis/` — the July 2026 review that produced several of these decisions @@ -81,9 +85,11 @@ one at a time (see D6). **Decision.** `IterationResult.success` is `True` whenever a `team-harness` run returns normally, and `False` only when the harness itself raises. It is **not** a judgment about whether the requested work was accomplished, and the per-worker exit codes in -`TeamHarnessResult.agents` are intentionally not consulted. Whether the work was any -*good* is decided by the evaluation layer via `control.json` (the stop switch) and -`goal_check.json` (evidence). +`TeamHarnessResult.agents` are intentionally not consulted. Whether the session has +enough evidence to stop is decided by the durable layer's declared orchestration role +through `control.json`. Reviews, tests, child outcomes, delivery facts, and optional +evaluation results are evidence for that decision; none is inferred from the harness +return value. **Context.** `team-harness`'s coordinator is an orchestrator, not a build system: it can legitimately return a normal result after a worker failed (synthesize an answer, route @@ -93,18 +99,30 @@ useless one. Mapping them to a boolean would manufacture false precision. This h the behavior since the first commit of `harness_runner.py`; it is original intent, not drift. -**Consequences.** The evaluation layer, not the harness return value, is the real -arbiter of completion. The scheduler keys cadence off mechanical success, so a -harness-completed-but-worker-failed run still advances `run_every`/`must_follow` -counters — an accepted, bounded inaccuracy, because `control.json`/`goal_check.json` -remain the true gates. Full reasoning and alternatives: +**Consequences.** The declared orchestration role, not the harness return value or a +scheduled evaluator, is the semantic arbiter of completion. The scheduler keys cadence +off mechanical success, so a harness-completed-but-worker-failed run still advances +`run_every`/`must_follow` counters. An absent, failing, or malformed advisory eval is +recorded as evidence/diagnostics; it does not retroactively flip mechanical success, +increment the workflow's harness-failure counter, or disable the orchestration role's +ability to decide. `control.json` remains the explicit stop switch. Full reasoning and +alternatives: `design/designs/success-semantics-and-evaluation.md` (Decision 1). -## D4. Evaluation is LLM-as-judge; agents do not author deterministic checks +## D4. When evaluation is used, stock checks are LLM-as-judge; agents do not author deterministic checks **Decision.** In the packaged `inner_outer_eval` workflow set, the eval workflows create **only** `harness_judge` (LLM-as-judge) checks that describe desired *outcomes*. -Authoring deterministic checks is explicitly forbidden in the stock template. +Authoring deterministic checks is explicitly forbidden in the stock template. This +decision governs the form and trust boundary of evaluation **when an orchestrator uses +it**; it does not require an eval run and does not give an eval role terminal authority. + +Designing non-trivial eval checks is itself consequential reasoning. The eval-check +authoring prompt should therefore prefer parallel, independent coverage and failure-mode +analyses from different enabled harness families, followed by review of the proposed +checks by a family other than the primary author when practical. The accountable +coordinator synthesizes those views into one coherent check set. This is a strong +judgment default, especially for high-stakes checks, not a fixed agent graph or quorum. **Context.** This is a lesson from experience, not theory. When agents were allowed to *author* deterministic checks, they produced brittle, wrong-target, gameable ones — the @@ -116,11 +134,13 @@ deterministic but is not that failure mode. **Consequences.** The stock "deterministic forbidden" rule is correct for generic target repos, where the only deterministic checks would be agent-invented. For a target that -already owns a trustworthy contract-test suite, the right configuration is *both* — the -judge for qualitative outcomes, plus a deterministic backstop that shells out to the -repo's own suite — via a dedicated child workflow set, not by loosening the stock -template. Judge with a different model family than the implementer where practical. A -single judge pass is evidence, not a hard gate for a high-stakes stop. Full reasoning: +already owns a trustworthy contract-test suite, the orchestrator may use both the judge +for qualitative outcomes and the repo's own deterministic suite as evidence, via a +dedicated workflow set rather than by loosening the stock template. Prefer a judge from +a different harness/model family than the primary implementer and check author where +practical. Prefer the `frontier` tier for difficult eval-policy/check design and +high-stakes judging when the session's capability roster offers it. A single judge pass +is evidence, never an engine-owned completion gate. Full reasoning: `design/designs/success-semantics-and-evaluation.md` (Decision 2). ## D5. Full autonomy, with `unresolvable_error` as the only, last-resort human escape hatch @@ -131,9 +151,10 @@ sanctioned escape hatch is already built: a workflow that hits a genuinely termi blocker writes `control.json` with `stop_reason: "unresolvable_error"`, which stops the session as terminal and leaves a recorded reason. We deliberately **do not** build a preferred, resumable "pause and wait for a human to answer" gate. -For a fresh v2 session, D11's identity-bound form applies: the producer must be the +For an identity-bound v2 or v3 session, D11's form applies: the producer must be the exact current session/workflow/attempt, its role must be declared for terminal-blocker -reporting, and the record must list autonomous routes already tried. +reporting, and the record must list autonomous routes already tried. The frozen +assignment supplies the applicable schema version. **Context.** An autonomous long-horizon loop will occasionally hit something it truly cannot do alone — a decision only a human can make, a credential it lacks, a @@ -163,15 +184,20 @@ considered and rejected. This entry is the canonical disposition and the mechani **Decision.** For a large, multi-phase target project, the intended execution shape is the **planner/dispatcher double loop** (the `pm_planner_dispatcher` workflow set) from -day one — a parent "planner" session that maintains PM state and selects one unit of -work, a "dispatcher" that spawns a child implementation session per unit, and a review -of the child's evidence — rather than starting with a single flat loop and adding the -double loop later. +day one — a parent `planner` session that maintains program state and chooses a coherent +milestone or phase outcome, and a `dispatcher` that transports that outcome into a child +implementation session — rather than starting with a single flat loop and adding the +double loop later. The parent deliberately stays above leaf-task level. The child +`inner_outer_eval` orchestrator owns decomposition into work packages, tasks, reviews, +commits, and PRs. **Context.** A single `inner_outer_eval` loop pointed at "build the whole thing" drowns -in context; the double loop keeps each unit small (a fresh child context scoped to one -work package) while the parent carries durable cross-cutting state. Committing to it from -the start avoids re-architecting mid-project. +in context; the double loop gives each substantial phase or milestone a fresh child +planning context while the parent carries durable cross-cutting state. Sending an exact +leaf such as "execute WP-0.1" down from the parent defeats that boundary: it hoards the +real plan in the PM layer and leaves the child outer role unable to adapt. Milestone size +is a semantic judgment, not an engine limit; the planner may split or combine phases when +the live evidence warrants it. **Consequences.** The parent/child machinery is on the critical path from day one, so it must be hardened *first* — durable active-child crash recovery and a PM template that is @@ -181,10 +207,15 @@ reconstructs session/child *state* from files; it does not re-adopt a crashed wo agent subprocesses. A hard worker crash is handled by the D7 drain/reap cleanup path. Child sessions remain depth-first and one-at-a-time (consistent with D2). The planner drives the target's *own* authoritative plan; it does not invent a parallel backlog. +The stock PM workflow set contains `planner` and `dispatcher`; it does not duplicate the +child layer's scheduled eval roles. A target goal may point the planner to prepared +program-level evals and ask it to run them near the end, but that is semantic goal +context, not a generic protocol gate. -**Refined by D10 and D11.** The same depth-first session edge now recurses beyond one -child level; three-depth dispatch, recovery, budget, and unwind tests guard that -behavior. Every layer still performs its own scoped evaluation before it can close. +**Refined by D10 and D11.** The same depth-first session edge recurses beyond one child +level; three-depth dispatch, recovery, budget, and unwind tests guard that behavior. +Every layer makes its own scoped completion decision; child evidence, including optional +eval evidence, flows upward but cannot make that decision for an ancestor. ## D7. Process-lifecycle ownership is split: team-harness owns agent processes, loopy-loop owns the worker @@ -237,89 +268,112 @@ verify-dead-before-reclaim rather than optimistic; legacy and remote identities documented limitation above. The implemented protocol and salvage boundary are described in `designs/long-running-loop-reliability.md`; the process-group mechanism is team-harness TH-D5. -## D8. Constraints on agents are fail-closed detection with a repair path, never hard prevention +## D8. Semantic constraints are visible detection with accountable repair or disposition, never hard prevention + +**Decision.** The system handles semantic constraints by **detecting** relevant facts and +making them visible to the accountable orchestration role, which repairs the issue, +reroutes the work, or records why the finding does not apply. It never prevents the action +up front and never turns one generic observation into an unconditional engine veto. No +path-level write sandboxes, semantic scheduling vetoes ("this workflow may not run until X +is proven"), approval gates, arbitrary mid-run hard-fails, or mandatory eval gates. -**Decision.** The system constrains agent behavior by **detecting** violations in evidence -and blocking *acceptance* of the work until they are repaired — never by **preventing** the -action up front. No preventive fences: no path-level write sandboxes, no semantic scheduling -vetoes ("this workflow may not run until X is proven"), no approval gates, no arbitrary -mid-run hard-fails. Every constraint must be expressed as something the agent can see, -contest, and repair against — an evaluation check or a recorded disposition — and the only -hard stops are the evaluation gates that decide whether work is *accepted* -(`goal_check.json` / `control.json`), not whether it may be *attempted*. +This decision does not weaken structural protocol integrity. The engine still validates +schemas, identity, current-attempt ownership, hashes, path confinement, explicit budgets, +and state-machine topology. Those checks protect the durable machine from corrupt or stale +input; they do not decide whether the work product is semantically good enough. **Context.** Stated as a general principle by the author (July 2026, during the design-loop work in writeit-loops-and-standards): agents should have enough freedom to decide; "fail-closed detection, not prevention" is the correct mental approach. It generalizes what three existing decisions already do individually: D3 keeps mid-run "success" mechanical and -moves all quality judgment into after-the-fact evidence; D4 bans *agent-authored* pass/fail -criteria while keeping repo/set-owned checks as detection backstops; D5 rejects a preferred -human gate in favor of evidence-in (`updates_from_user`) and a last-resort terminal stop. +makes quality a separate, explicit orchestration decision; D4 bans *agent-authored* +pass/fail criteria while keeping repo/set-owned checks as useful evidence; D5 rejects a +preferred human gate in favor of evidence-in (`updates_from_user`) and a last-resort +terminal stop. The reasoning: prevention encodes today's guess about what agents shouldn't do and hides its own mistakes, while detection publishes every constraint as a visible, arguable check failure with a defined relaxation route — a wrongly-scoped check gets repaired with a counterexample and independent review instead of being silently obeyed forever. Concrete -shape (from the design-loop): a workflow set ships a deterministic "write barrier" check -that diffs protected paths against the session-start digest; a child session can physically -write anywhere, but cannot terminate successfully while the barrier fails — fail-closed -detection, not a sandbox. +shape: a workflow set may report that protected paths differ from their session-start +digest. The orchestrator sees the exact diff, normally repairs it, and can explain a +legitimate exception. The engine neither hides the action with a sandbox nor silently +promotes the report to terminal authority. **Consequences.** New engine features and workflow sets must not introduce preventive mechanisms: no coordinator-enforced path permissions, no eligibility gates keyed to semantic state, no paused/waiting-for-human states (already banned by D5). Where discipline over -files is needed (a research workflow must not touch binding docs), express it as a shipped -deterministic check over the diff — consistent with D4's boundary (set-owned, not -agent-authored) — whose failure blocks the session's goal check until the write is undone. -The accepted cost: a violating action can occur and must be detected and repaired after the -fact; that inefficiency buys inspectability and reversibility of the constraint itself. - -## D9. Coordinators are uniformly strong; worker model choice is per-spawn, prompt-guided, and audited — never enforced - -**Decision.** Every harness coordinator in a session tree — the root PM loop, child -implementation loops, any deeper level — runs the **same strong coordinator model** -(`team_harness_model`, one value per repo). Cost control comes from the **workers**: -the root config may declare **named model tiers** (`model_tiers`: tier name → agent → -`{model, effort}`), and each coordinator chooses a tier per spawned agent via -team-harness's per-spawn `spawn_agent(model=…, effort=…)` overrides. Tier selection is -**guidance rendered into the system prompt plus an audit trail** (team-harness records -requested/effective model and effort per agent in `run.json`) — the engine never -validates or blocks a coordinator's model choice (D8). With `default_tier` set, the -named tier *derives* `team_harness_agent_models` / `team_harness_agent_reasoning_efforts` -(setting both is a config error), so a model id lives in exactly one place. - -**Context.** The obvious alternative — differentiating whole sessions ("strong parent -session, cheap child session", per-session execution profiles carried on -`ChildSessionRequest`) — was analyzed (July 2026) and rejected for now, consistent with -the withdrawal of P0.3. Uniform strong coordinators dissolve that design's two hardest -problems at once: the cost ledger stays correct (loopy only meters the coordinator -model, so one repo-global `model_prices` remains valid), and a child's planning/eval -reasoning is never downgraded along with its implementation muscle (the D4 concern of a -weak session judging its own work). The coordinator context is also cheaper than it -looks: it orchestrates on bounded log tails and status polls while worker CLIs — billed -to their own accounts — chew the bulk tokens. Tier names are deliberately -capability-semantic bundles of model + effort ("strong", "economy"), not raw -model-id/effort axes, so prompts reason about one word and model churn stays a one-line -config edit (the P2.1 drift concern). - -**Consequences.** Workflow prompts should name **tiers**, never model ids; the rendered -guidance block (`render_model_tier_guidance`, `config.py`) is the only place tiers -expand to models. Adherence is probabilistic by design: a coordinator can forget to -escalate a review — the remedy is the audit trail (an outer reviewer or an eval check -verifies `requested_model`/`effective_model` on the child's agent records), never an -engine fence (D8). Do not add per-session/per-depth model allowlists, "children may not -request expensive tiers" vetoes, or coordinator-model differentiation per loop level; if -per-session coordinator profiles ever become genuinely needed, they compose with tiers -(profiles set session defaults, tiers guide per-spawn choice) and require amending this -decision. Effort-as-spawn-argument lives in team-harness (0.4.0+, TH-D6); on older -installed versions the tier guidance still works for `model`, and effort escalation -falls back to raw CLI `flags`. +files is needed (a research workflow should not touch binding docs), express it as visible +diff evidence and clear prompt responsibility. A failing test, review, or eval is important +input, and ignoring it should demand an explicit rationale, but its existence alone does not +rewrite `HistoryEntry.success` or prohibit orchestrator-owned `control.json`. The accepted +cost is that a violating action can occur and must be detected and repaired after the fact; +that inefficiency buys inspectability and reversibility of the constraint itself. + +## D9. Strong coordinators receive a frozen harness/tier roster; delegation and review remain prompt-guided + +**Decision.** Every harness coordinator in a session tree — root PM, child +implementation, and any deeper layer — runs the same strong coordinator model +(`team_harness_model`, one value per repo). Every attempt also receives a frozen roster +of all enabled **harness families** and their configured `{model, effort}` bundles by +semantic strength tier. The canonical stock vocabulary is: + +- `frontier`: the maximum-capability configured bundle for the hardest planning, + architecture, adversarial review, and eval-policy/check design; +- `strong`: a high-capability bundle for complex reasoning, implementation, and review + that does not require the maximum tier; +- `standard`: a balanced default for ordinary implementation, analysis, and review; and +- `economy`: a lower-cost/lower-latency bundle for bounded mechanical work, broad + reconnaissance, and low-risk checks. + +For orientation only, an Anthropic-family roster could map these tiers to +Fable, Opus, Sonnet, and Haiku respectively. Prompts use the semantic tier +names, never those provider-specific examples. + +Harness family and strength tier are independent choices. A different family provides +diversity of tools and failure modes; a stronger tier provides more capability within a +family. The roster states unavailable family/tier combinations explicitly. Every +coordinator receives its absolute path and a rendered summary, then chooses family, +tier, concurrency, retries, and review shape dynamically via team-harness per-spawn +overrides. + +For consequential planning, design, uncertain analysis, review, and especially eval +check creation, prompts should prefer independent analyses from different enabled +harness families in parallel when separable, followed by review by a family other than +the primary author. The accountable coordinator synthesizes disagreements. This is +guidance plus an audit trail, never a required number of agents, a fixed graph, a vendor +rule, or an engine gate (D8). + +**Context.** Differentiating whole sessions ("strong parent, cheap child") was analyzed +and rejected: it complicates cost accounting and can downgrade the very coordinator +that must plan and judge the layer. Uniform strong coordinators avoid both problems, +while per-spawn tiers control the bulk worker cost. Earlier vendor-specific chains did +produce useful independent review, but hard-coded one graph and repeated volatile model +names in prompts. The frozen roster preserves that diversity benefit while allowing the +coordinator to adapt to availability, task coupling, cost, and live evidence. + +**Consequences.** Stock prompts name semantic tiers and roster entries, never model IDs +or required vendors. Repository config may omit unavailable canonical mappings and may +add a clearly explained project-local tier; the frozen roster still renders every +canonical cell for every enabled family and marks missing mappings unavailable. Stock +prompts use `frontier`/`strong`/`standard`/`economy` and inspect that availability. +With `default_tier` set, the tier derives `team_harness_agent_models` and +`team_harness_agent_reasoning_efforts`, so concrete model IDs live in one place. The +engine validates roster shape and records requested/effective harness, model, and +effort; it does not judge whether a model deserves its label or reject a coordinator's +choice. Parallel delegates write separate findings or trace artifacts rather than +racing to edit one canonical plan or check set. If only one family is usable, the +coordinator proceeds autonomously and records the limitation when material. Do not add +per-depth model allowlists, spend vetoes, review quotas, or coordinator-model +differentiation per layer. ## D10. Durable loop layers recurse; harness subagents remain dynamic delegations **Decision.** A durable loop layer is one loopy session with a scoped goal, state, -decisions, evals, attempts, and optional child. One-layer, planner/dispatcher, and -deeper systems compose the same session node and parent→child protocol. Only the -deepest session owns a live loopy assignment. +plan, decisions, accepted-work ledger, semantic handoff, attempts, optional eval +evidence, and optional child. One-layer, planner/dispatcher, and deeper systems compose +the same session node and parent→child protocol. Only the deepest session owns a live +loopy assignment. The workflow contract names one persistent orchestration role that +owns the layer's plan, handoff, and completion decision. Inside that assignment, the team-harness coordinator remains free to choose a dynamic team, roles, models, ordering, retries, and follow-ups. Spawned agents—including a @@ -327,10 +381,19 @@ nested `type=harness` coordinator—are delegates in the current layer unless th owning workflow explicitly publishes a child-session request. Every attempt receives a frozen assignment with loop identity, role, responsibility, -and worker-local absolute state/output paths. Team-harness derives a smaller absolute -assignment for each direct spawn. Durable records use validated logical references so -they survive a moved checkout. Child request/input bytes are copied into the child's -immutable `inputs/` area so later parent edits cannot change accepted work. +and worker-local absolute state/output paths. It also receives the complete frozen +workflow roster, an attempt-frozen scheduler view with recent mechanical history and a +clearly conditional next-workflow forecast, and D9's frozen harness/model capability +roster. The forecast says what would run next if the current attempt returns normally +without terminal control, child dispatch, stop, or failure; it is context, not a +promise or eligibility gate. This lets an orchestrator avoid duplicating work that a +scheduled reviewer or evaluator is already due to perform. + +Team-harness derives a smaller absolute assignment for each direct spawn. Durable +records use validated logical references so they survive a moved checkout. Child +request/input bytes are copied into the child's immutable `inputs/` area so later +parent edits cannot change accepted work. The workflow roster describes scheduled +Loopy roles, not a predicted harness-subagent graph; those delegates remain dynamic. **Context.** “Inner loop,” “child,” and “subagent” blurred durable session depth, workflow roles, and short-lived processes. Encoding a fixed agent graph would weaken @@ -339,63 +402,97 @@ delegates unsure of their layer and paths. **Consequences.** D2 still permits parallel harness agents inside one assignment but not parallel loopy workers. Ownership metadata is accountability, not an ACL or model -allowlist (D8/D9). The same edge is tested through three active depths. Provider-native -nested actors are recorded only when observable. See the +allowlist (D8/D9). Prompts should encourage independent parallel analysis and review by +other enabled harness families while leaving team shape to the coordinator. The same +edge is tested through three active depths. Provider-native nested actors are recorded +only when observable. See the [binding design](designs/recursive-loop-layer-contract.md) for the full contract and legacy boundary. -## D11. Every session evaluates its own goal and names its terminal-control owner - -**Decision.** Every durable session evaluates its own scoped goal. A child verdict is -evidence for its parent, never proof that the parent's broader goal is complete. Each -workflow contract names the check author, runner, task-acceptance owner, terminal -`goal_met` owner, and terminal-blocker reporters. In `inner_outer_eval`, `outer` -records task acceptance/readiness and only `eval_runner` may request success. - -The canonical eval receipt binds the verdict to session/goal identity, exact check -definitions, producer/harness identity, judge settings, raw and canonical report -hashes, and evaluated git state. `goal_check.json` is a matching iteration projection; -successful v2 control cites the same receipt. The engine validates provenance and -all-pass mechanics but does not reinterpret the LLM judge's semantic conclusion. - -Both success and D5 blocker control must identify the exact current -session/workflow/attempt and come from a role named by the frozen workflow contract. -A spawned agent reports to its coordinator; it cannot leave durable control for -another layer or later attempt. - -**Context.** Before D11, the one-layer prompts gave both `outer` and `eval_runner` -paths to close the session, so outer could stop before eval ran. The PM set could -similarly close a root after reviewing child evidence without a separate root-goal -evaluation. -That ambiguity becomes more dangerous at three depths: a leaf may correctly finish -its task while the feature integration or release goal remains incomplete. Existing -`goal_check.json` also lacks enough provenance to prove exactly what goal, checks, -judge, and repository state it evaluated. - -**Consequences.** D3 mechanical success and D4 LLM-as-judge semantics remain -unchanged. Receipt checks are structural validation with a repair path, not a new -semantic scheduler gate (D8). Readiness remains prompt context under mechanical eval -cadence. `unresolvable_error` remains D5's last resort and needs no passing eval. -Malformed v2 control is archived and bounded as repairable protocol failure. Legacy -sessions retain their historical provenance. See the -[binding design](designs/recursive-loop-layer-contract.md). +## D11. Every session's orchestrator owns completion; evals are optional evidence + +**Decision.** Every durable session owns a scoped goal and names exactly one persistent +orchestration role that owns the layer plan, integrates evidence, publishes the semantic +handoff, and decides when that goal is complete. In the stock sets: + +- `inner_outer_eval`: `outer` owns task selection and acceptance, the layer handoff, + and terminal `goal_met`; and +- `pm_planner_dispatcher`: `planner` owns the program plan, child acceptance, root + handoff, and terminal `goal_met`. + +A child outcome is evidence for its parent, never proof that the parent's broader goal +is complete. Eval reviewer/runner roles, when configured or dynamically invoked, are +evidence producers. They may author or run checks on their schedule and publish +provenance-rich observations for a later orchestrator attempt. The orchestrator may +also run or delegate an eval directly, wait for a due scheduled eval role shown in its +scheduler view, rerun or supersede an observation, or decide that evaluation is +unnecessary. The stock PM workflow therefore needs only planner and dispatcher; +program-level prepared evals run near completion when the target goal asks the planner +to run them. + +Successful control must identify the exact current session, workflow, and attempt; come +from the completion role frozen in the workflow contract; and contain a non-empty +reasoned completion disposition. It may cite the evidence considered, but evidence +references are not required to be non-empty. An eval receipt is optional and +need not come from the same attempt. When cited, the engine strictly validates its +session/goal identity, check definitions, producer/harness identity, judge settings, +declared runner role, raw and canonical report hashes, and evaluated git state. It +validates provenance, not the weight the orchestrator gives the verdict. An absent or +malformed advisory eval is +a visible diagnostic; it does not turn a mechanically completed harness invocation into +failure or starve the orchestration role. + +D5 terminal-blocker control retains exact identity, allowed-role, attempted-route, and +evidence requirements. A spawned delegate reports upward to its harness coordinator; it +cannot publish durable control for another role, attempt, or layer. + +**Context.** The first one-layer prompt made `outer` the persistent planner and used +scheduled evaluators as occasional independent observations. The first PM workflow had +only `planner` and `dispatcher`, with planner owning the program decision. The v2 +provenance work fixed real stale/sibling/goal ambiguity, but overcorrected by moving +completion authority to `eval_runner` and requiring a same-attempt passing eval. That +made one advisory mechanism mandatory, duplicated evaluation at PM and child layers, +and could prevent a smart orchestrator from finishing despite stronger direct evidence. +The required boundary is exact identity and truthful provenance, not semantic deference +to a particular scheduled role. + +**Consequences.** Remove terminal control from the eval sub-contract and represent the +orchestration/completion role at top level. Remove the universal same-attempt passing +receipt and `goal_check.json` prerequisite. Invalid advisory eval output becomes a +durable diagnostic rather than `goal_check_broken` or a generic workflow failure. +`inner_outer_eval` remains fully standalone: root and nested sessions use identical +planning, optional-evidence, handoff, and completion semantics. Fresh stock or explicitly +amended-v3 sessions use the new contract; custom sets retain the version they declare, +a contract file with no version remains pinned to the historical v2 default, and +already-live sessions retain their frozen historical contract rather than changing +authority midway. See the +[binding design](designs/orchestrator-owned-completion-and-cross-harness-review.md). ## D12. Correctness state/evidence and exhaustive execution traces have separate retention contracts **Decision.** Compact facts required to schedule, recover, or justify acceptance stay -with the session: topology, goals, assignments, progress, decisions, child handoffs, -normalized results, and eval/git/delivery/recovery receipts. Detailed observable +with the session: topology, goals, assignments, layer plans and accepted-work ledgers, +progress, decisions, semantic handoffs, workflow/scheduler/harness rosters, normalized +results, orchestration completion rationale, and optional eval plus +git/delivery/recovery receipts. Detailed observable execution lives under separately gitignored `.loopy_loop/traces/`: prompts, visible turns, tool/spawn I/O, process/provider identity, streams, raw eval output, verbose git evidence, timing, and usage. Inputs are persisted before their provider calls. +Independent analyses and reviewer transcripts stay in the trace plane. The accountable +coordinator promotes conclusions that future attempts need into compact plan, decision, +eval-definition, accepted-work, or handoff artifacts; continuity never depends on +re-reading raw conversations. Raw eval bytes are validated when a canonical receipt is +accepted and sealed; later citations trust that compact accepted receipt and subject +identity rather than requiring retained trace bytes. + Each attempt has one caller-owned, completeness-aware trace manifest. The coordinator creates it during dispatch; the worker and team-harness populate that same canonical tree. Matching completion or crash abandonment uses a write-ahead finalization record, then a hashed manifest and compact session-side seal receipt. Startup retries only after durable history proves the transition committed; an unavailable HTTP response -is recorded as unavailable rather than invented. Trace I/O failure is observable but -never becomes a semantic acceptance gate (D3/D8). +is recorded as unavailable rather than invented. Trace I/O failure and advisory-eval +failure are observable but never become semantic acceptance gates (D3/D8/D11). **Context.** Before this decision, execution records were split inconsistently: session-local worker artifacts lived under harness outputs, while team-harness 0.4.0 diff --git a/design/designs/long-running-loop-reliability.md b/design/designs/long-running-loop-reliability.md index 765e735..c9341d9 100644 --- a/design/designs/long-running-loop-reliability.md +++ b/design/designs/long-running-loop-reliability.md @@ -22,12 +22,18 @@ limit, flat child requests, the global team-harness run-record mismatch, or missing grandchildren describe the shipped 0.3–0.6 baseline and legacy v1 resume behavior, not fresh 0.7 sessions. +The later accepted protocol-v3 completion/state/roster amendment is implemented +in loopy-loop 0.8.0 and team-harness 0.5.4. It remains binding in +[`orchestrator-owned-completion-and-cross-harness-review.md`](./orchestrator-owned-completion-and-cross-harness-review.md). +It does not reinterpret v1/v2 sessions. + The design is driven by three existing decisions: - files and git are the durable source of truth (D1); - exactly one loopy worker owns an assignment at a time (D2); and -- evaluation artifacts decide work quality; transport and recovery code must - not manufacture semantic success (D3/D4). +- the declared durable orchestrator decides semantic completion from available + evidence, while transport/recovery code must not manufacture it and optional + eval evidence follows D4's trust boundary (D3/D4/D11). Together, the mechanisms below let the planner/dispatcher double loop run for a long time without losing its place, duplicating verifiably live local work after @@ -378,26 +384,38 @@ API surface: confidently generating a removed layout is worse than omitting a feature. The clean-init tests validate the templates themselves, although the prose Skill is not mechanically compared with them. -### Named model tiers centralize project-local worker choice +### A frozen harness/tier roster centralizes project-local worker choice D9 keeps every harness coordinator in a session tree on the same strong `team_harness_model`. Cost control happens per spawned worker. A root config may -declare `model_tiers` as tier name → agent → `{model, effort}` and an optional -`default_tier`. +declare `model_tiers` as tier name → harness family → `{model, effort}` and an +optional `default_tier`. The canonical stock vocabulary is `frontier` for the +maximum-capability configured bundle, `strong` for complex high-capability +work, `standard` for balanced ordinary work, and `economy` for bounded +lower-cost work. In an Anthropic-family roster these correspond, for example, +to Fable, Opus, Sonnet, and Haiku. Missing family/tier cells are allowed in +config and rendered explicitly as unavailable in the frozen roster; prompts +must not assume availability. `load_root_config()` in `src/loopy_loop/config.py` rejects a `default_tier` combined with duplicate explicit mappings. `resolve_model_tiers()` then derives the concrete per-agent defaults from the named default, and -`render_model_tier_guidance()` appends the tier table to every harness -coordinator's system prompt. Workflow prompts can request `economy` or `strong` -without embedding model ids. Children inherit the parent's frozen resolved -config snapshot; editing root YAML mid-session does not silently change their -model policy. - -Tier selection is guidance plus audit evidence, never an engine veto (D8). The -harness records requested and effective model/effort per spawn; review or eval -can detect a poor choice and repair it. The engine does not enforce a tier -allowlist or weaken child coordinators by depth. +`render_model_tier_guidance()` currently appends tier prose to every harness +coordinator's system prompt. The accepted protocol-v3 amendment additionally +freezes a structured, session-tree-wide capability roster, gives every +coordinator its absolute path plus a summary, and preserves the full matrix +rather than only resolved defaults. Workflow prompts request semantic tiers +without embedding model IDs. Children inherit the frozen roster; editing root +YAML mid-session does not silently change their available delegate catalog. + +Harness-family diversity and tier strength are separate choices. Prompts should +prefer parallel independent analyses and different-family review for +consequential work, especially eval-check creation, while one coordinator +integrates the result. Tier/family selection is guidance plus audit evidence, +never an engine veto (D8). The harness records requested and effective +model/effort per spawn; later review can use those facts. The engine does not +enforce a tier allowlist, vendor graph, review quota, or weaker child +coordinator by depth. --- diff --git a/design/designs/orchestrator-owned-completion-and-cross-harness-review.md b/design/designs/orchestrator-owned-completion-and-cross-harness-review.md new file mode 100644 index 0000000..f37d417 --- /dev/null +++ b/design/designs/orchestrator-owned-completion-and-cross-harness-review.md @@ -0,0 +1,982 @@ +# Design: Orchestrator-Owned Completion, Layer Handoffs, and Cross-Harness Review + +**Status:** Accepted and implemented + +**Introduced in:** loopy-loop 0.8.0 and team-harness 0.5.4 + +**Date accepted:** 2026-07-17 + +**Applies to:** standalone `inner_outer_eval`, the +`pm_planner_dispatcher` double loop, future recursive loop layers, assignment +context, semantic state and handoff, optional evaluation, and team-harness +delegation. + +This is the binding companion design for amended D3, D4, D6, and D8–D12 in +[`design/decisions.md`](../decisions.md). It supersedes the mandatory-eval and +eval-owned terminal-control parts of +[`recursive-loop-layer-contract.md`](./recursive-loop-layer-contract.md) and +[`success-semantics-and-evaluation.md`](./success-semantics-and-evaluation.md). +The shipped protocol-v2 behavior remains documented for compatibility. The +loopy-loop 0.8.0 stock contracts explicitly select the v3 implementation; +the 0.7 stock contracts remain v2. Custom sets retain the protocol version they explicitly declare, +contract files that omit `session_protocol_version` remain pinned to the +historical v2 default, and sets with no contract remain derived v1. Only an +explicit v3 declaration selects v3. + +## Summary + +Each durable loop layer has one persistent **orchestrator**. That role owns the +layer's plan, integrates the work and reviews performed inside the layer, +communicates a semantic handoff upward, and decides when its own goal is +complete. For the stock workflow sets, the owner is `outer` in +`inner_outer_eval` and `planner` in `pm_planner_dispatcher`. + +Evaluation is useful information, not a mandatory ceremony and not a second +orchestrator. An eval role may author or run checks on a schedule; the +orchestrator may invoke an eval directly, wait for a scheduled evaluator, ask +for another opinion, or decide from other evidence. When eval evidence is +produced or cited, its provenance remains strict. Its absence or failure does +not programmatically prevent the orchestrator from completing the session. + +The system helps capable models make those decisions by giving every attempt: + +1. absolute paths to the layer's canonical plan, tasks, current state, + decisions, accepted-work ledger, optional eval index, and handoff; +2. a frozen roster of scheduled workflow roles and an attempt-frozen, + conditional scheduler forecast; and +3. a frozen capability roster of all enabled harness families and their models + through the semantic strength tiers `frontier`, `strong`, `standard`, and + `economy`. + +Prompts should prefer independent parallel analyses and review by a different +enabled harness family when that materially improves confidence. This +preference is strongest for eval-check creation, because weak or gameable +checks distort every later judgment. It is guidance, not a fixed graph, review +quota, model gate, or vendor requirement. + +## Why the amendment is necessary + +Protocol v2 solved real problems: it bound control and eval artifacts to the +correct session/goal/attempt, froze workflow contracts, separated durable state +from traces, and made recursive parent/child recovery explicit. Those +structural guarantees remain. + +It also made one semantic mechanism universal. The current +`coordinator_app.py` terminal-control path requires a same-attempt passing eval +receipt and `goal_check.json`; both stock workflow contracts assign +`goal_met` to `eval_runner`; and `emits_goal_check` can turn missing evaluator +output into workflow failure. The PM set consequently carries another eval +reviewer/runner even though each child can already produce scoped evaluation. + +That is too rigid for an agentic orchestrator: + +- a direct review, repo-owned test suite, child handoff, delivery receipt, or + prepared target-specific evaluation may be stronger evidence than the stock + scheduled judge; +- a malformed advisory observation can starve the `outer` or `planner` role + that actually understands the accumulated plan; +- a planner cannot avoid duplicate eval work even when it knows an eval role is + about to run; and +- the parent has been pushed toward leaf-level dispatch because the child + orchestration layer is not visibly carrying its own plan and handoff. + +The correction is not to remove evaluation or provenance. It is to restore the +semantic decision to the durable orchestrator while giving that orchestrator +better state, schedule, capability, and review context. + +## Hard protocol truth versus semantic judgment + +The engine continues to enforce facts required for a trustworthy state machine: + +- repository, root, session, workflow, and exact current-attempt identity; +- parent/child topology and single-deepest-assignment invariants; +- atomic transition and crash-recovery rules; +- schema validity, reference containment, and content hashes; +- truthful provenance for any cited evidence; +- explicit turn/cost/stop limits; and +- D5's identity-bound terminal blocker contract. + +The engine does **not** decide: + +- whether the current plan is the best plan; +- whether a milestone should be split or combined; +- whether an eval is needed or deserves more weight than another review; +- which enabled harness or strength tier must be used; +- how many reviewers constitute enough review; or +- whether the session's accumulated semantic evidence is sufficient. + +Those are responsibilities of the named orchestration role. Detection stays +visible: a failing test, review, eval, or constraint observation should normally +lead to repair, rerouting, or a written disposition. It does not silently become +an engine-owned semantic veto. + +## Layer roles and completion ownership + +### Stock role contract + +| Workflow set | Role | Durable responsibility | +| --- | --- | --- | +| `inner_outer_eval` | `outer` | Own the layer plan, select and accept leaves, integrate evidence, maintain handoff, and decide `goal_met` | +| `inner_outer_eval` | `inner` | Execute one bounded leaf selected from the layer plan and report evidence upward | +| `inner_outer_eval` | `eval_reviewer` | Optionally author/revise outcome-oriented eval checks and update the eval evidence index | +| `inner_outer_eval` | `eval_runner` | Optionally run checks and publish provenance-rich observations for the next outer attempt | +| `pm_planner_dispatcher` | `planner` | Own the high-level program plan, accept/reroute child outcomes, maintain root handoff, run target-requested final evidence, and decide `goal_met` | +| `pm_planner_dispatcher` | `dispatcher` | Faithfully turn one planner-selected milestone outcome into a child-session request | + +`inner` never bootstraps or rewrites the layer plan when selection is absent. +It reports the missing/ambiguous leaf to `outer` and returns useful evidence; +`outer` repairs the plan on its next attempt. Workflow state accountability is +therefore declared per artifact with one owner and optional contributing roles, +not one flat `accountable_roles` list that makes every role appear to own every +file. + +The stock PM set contains only `planner` and `dispatcher`. It does not repeat +the child's scheduled eval roles. The planner can dynamically delegate a +program-level review or evaluation through team-harness when useful, and a +target goal can explicitly ask it to run prepared final evaluations. + +Future workflow sets declare the analogous orchestration role explicitly. The +engine must not infer completion ownership from a role name containing +`outer`, `planner`, or `eval`. + +### `inner_outer_eval` is topology-neutral + +The one-layer set must run unchanged in both forms: + +- as a root session given a broad goal; and +- as a child session given a scoped parent request. + +In both cases, its own `goal_contract.json` is authoritative, `outer` creates +and maintains its plan, inner attempts execute leaves, optional evaluators +produce observations, and outer writes the handoff and terminal decision. +Parent request/input artifacts are useful origin context when present; no +prompt may assume they exist. This invariant prevents the double loop from +quietly becoming a special mode that the one-layer system cannot reproduce. + +### Child evidence never completes an ancestor + +A child owns only its scoped goal. Its semantic handoff and terminal outcome +flow to the parent, which independently chooses `accepted`, `rework`, or +`reroute`. A child may correctly finish a phase while integration, release, or +later phases remain. The engine therefore records a factual child outcome but +never projects child `goal_met` into parent `goal_met`. + +## Correct planning granularity in the double loop + +The PM planner maintains a high-level program plan and dispatches a bounded, +coherent **outcome** that leaves meaningful planning work to the child outer +role. A phase, milestone, or integrated feature is usually the right unit. A +leaf instruction such as “execute exactly WP-0.1” usually is not. + +For example: + +| Layer | Appropriate goal | +| --- | --- | +| PM planner | “Make the development foundations ready and evidenced.” | +| Child outer plan | Reconcile the relevant design authorities, identify the remaining foundation work packages, sequence them, deliver coherent PRs, and verify the phase outcome | +| Inner leaf | Implement one selected schema migration or repair one failing contract test | + +For Ultimate Memory, the rough phase spine (phases 0 through 8) is the natural +initial PM projection. The planner may split a phase that proves too broad, +combine tightly coupled outcomes, or reorder them when dependencies demand it. +That is planner judgment, not an engine-enforced size rule. The dispatcher must +preserve the selected outcome and its observable completion criteria instead of +pre-solving the child's decomposition. + +Near overall completion, the Ultimate Memory goal can point the planner to +`plan/implementation_evals/` and instruct it to run the prepared final suite. +That requirement belongs to the target goal. It does not make evals mandatory +for every Loopy session or duplicate them in the PM scheduler. + +## Canonical semantic state spine + +Every fresh protocol-v3 session receives the same engine-created project-state +skeleton. +The files are compact, durable semantic state; raw agent conversations remain +in traces. + +```text +project_state/ +├── plan.md +├── tasks/ +├── current_state.md +├── decisions/ +├── finished.md +├── eval_state.md +└── handoff.json +``` + +| Path | Accountable writer | Meaning | +| --- | --- | --- | +| `plan.md` | layer orchestrator | Current decomposition, ordering, dependencies, and completion reasoning for this layer only | +| `tasks/` | orchestrator; inner contributes evidence | Stable per-leaf selections/status/evidence so plan prose is not the only ledger | +| `current_state.md` | layer orchestrator | Short resumption view: current outcome, active leaf/child, blockers, risks, and next decision | +| `decisions/` | layer orchestrator | Meaningful choices and rationale that later attempts must not rediscover | +| `finished.md` | layer orchestrator | Append-only accepted-work ledger with commit/PR/test/review references | +| `eval_state.md` | layer orchestrator; eval roles contribute observations | Optional index of check intent, observations, provenance, disagreements, and possible next eval action | +| `handoff.json` | layer orchestrator | Rolling semantic summary that can be consumed by a parent or an operator | + +### Minimum inspectable plan and task shape + +The engine scaffolds stable headings so an operator can find the plan without +teaching every prompt a new format. It does not parse the prose to choose work +or validate whether the plan is wise. `plan.md` starts with: + +```markdown +# Layer Plan + +- Revision: +- Layer goal: +- Current milestone: + +## Outcomes + +| ID | Outcome | Status | Dependencies | Evidence | +| --- | --- | --- | --- | --- | + +## Active selection + + + +## Risks, assumptions, and replanning triggers +``` + +Each `tasks/.md` records the parent outcome ID, objective, current +status, dependencies, completion evidence expected, accepted evidence, and +remaining questions. Suggested statuses (`proposed`, `ready`, `active`, +`accepted`, `rework`, `superseded`) are shared language for agents and humans, +not engine states or scheduler gates. Only the layer orchestrator changes plan +revision, active selection, and semantic task status. Inner/delegate attempts +produce evidence for outer to integrate rather than racing those canonical +fields. The attempt assignment resolves the selected logical task reference to +the absolute path that `inner` reads. + +### Prompt-level state lifecycle + +The stock prompts make the file contract operational without asking the engine +to interpret it: + +1. At attempt start, `outer`/`planner` reads its absolute goal, plan, + `current_state.md`, accepted-work ledger, decisions relevant to the current + milestone, rolling handoff, workflow roster, and scheduler view. +2. It inspects this attempt's child/delegate/review/test/eval/delivery evidence + and makes the next semantic decision. +3. When facts changed, it updates the task ledger and plan revision, then writes + a compact current-state resumption view and advances the rolling handoff + before yielding. Atomic file replacement prevents half-written durable + artifacts; semantic consistency remains its responsibility. +4. `inner` reads only the absolute selected-task path plus relevant goal/state + context, performs that leaf, and reports evidence upward. It does not repair + an absent plan itself. +5. `dispatcher` reads the planner-selected outcome, freezes the dispatch input, + and transports it without inventing leaf decomposition. + +No missing or weak plan heading makes a workflow engine-ineligible. The next +outer/planner attempt sees the omission and repairs it. + +`eval_state.md` always has a path so prompts do not need topology branches, but +it may state that no evaluation has been created or run. Its emptiness is not a +protocol error. Protocol-v3 contracts retire the separate `eval_readiness/` +channel: any useful readiness/eval headline moves into `eval_state.md`, while +the conditional scheduler view explains when a scheduled eval role is due. +Frozen v2 sessions retain their historical readiness files and readers. + +### Handoff contract + +`handoff.json` is updated throughout the session, not improvised after the last +attempt. Its structurally required fields are identity and revision metadata; +its semantic lists remain flexible: + +```json +{ + "schema_version": 1, + "session_id": "", + "goal_sha256": "", + "revision": 7, + "producer": { + "workflow_id": "outer", + "attempt_id": "" + }, + "summary": "What this layer now believes and why.", + "accepted_outcomes": [], + "open_work": [], + "risks": [], + "decision_refs": [], + "evidence_refs": [], + "delivery_refs": [], + "eval_refs": [], + "updated_at": "" +} +``` + +After each attempt, the engine retains the last provenance-valid handoff bytes +and hash. At any terminal transition it writes a topology-neutral +`session_outcome.json` together with the frozen terminal identity, optional +accepted control, goal, delivery, trace-seal, and optional eval references. A parent's +`child_outcomes/` record links that same outcome instead of synthesizing a +different story. A root operator sees the same result shape. Missing or stale +handoff is surfaced as factual incompleteness for review; it is not a hidden +semantic gate that overrides the orchestrator. + +The terminal outcome resolves delivery evidence across the whole session, not +only the terminal-control attempt: implementation, PR creation, merge, and +completion synthesis normally occur in different attempts. Cited receipts and +the accepted-work ledger guide selection, and every projected receipt retains +its original attempt identity. A syntactically invalid `handoff.json` is +quarantined/diagnosed and represented as unavailable or invalid in the outcome; +it must not crash stack reconstruction or turn handoff into a control gate. + +The orchestration role increments `handoff.json.revision` monotonically on each +accepted rewrite. The engine records the observed revision/hash and reports a +missing, malformed, or non-monotonic revision as completeness diagnostics; it +does not infer semantic staleness by parsing the summary or veto authentic +control. + +The engine-owned terminal projection has a topology-neutral minimum shape: + +```json +{ + "schema_version": 1, + "session_id": "", + "root_session_id": "", + "goal_sha256": "", + "lifecycle": "terminal", + "terminal_status": "stopped | goal_met | failed | max_turns", + "stop_reason": "", + "terminal_state_revision": 13, + "control": {"ref": "session:/control.json", "sha256": ""}, + "handoff": { + "status": "valid | missing | invalid | non_monotonic", + "ref": "", + "sha256": "", + "revision": 7 + }, + "fallback_summary": { + "source": "control_reason", + "text": "" + }, + "evidence_refs": [], + "delivery_refs": [], + "eval_refs": [], + "trace_seal_refs": [], + "created_at": "" +} +``` + +`control` is null for engine lifecycle stops such as `max_turns`, +`workflow_failure_cap`, or `stop_requested`. `fallback_summary` is not a +programmatic semantic synthesis. It copies the authenticated orchestrator's +control reason, or the factual engine stop reason when no terminal control +exists, so a parent is never handed a blank outcome when handoff is +unavailable. A valid handoff remains the richer semantic channel. Every +referenced artifact retains its own producer/attempt identity and hash. + +The engine stores the accepted control and handoff bytes in durable state and +uses those snapshots whenever it regenerates the outcome. Mutable artifact +edits after terminal acceptance therefore cannot rewrite parent-visible +meaning. A later trace-finalization pass may only add trace-seal references. + +## Absolute assignment paths + +Durable records continue to use portable logical references, but every running +coordinator receives worker-local absolute paths. The assignment path map must +name at least: + +```text +layer_goal +layer_goal_contract +layer_inputs +layer_plan +layer_tasks +layer_current_state +layer_decisions +layer_finished_ledger +layer_eval_state +layer_handoff +session_state +session_outcome +workflow_contract +workflow_roster +scheduler_view +harness_capability_roster +user_inputs +child_requests +children_index +child_outcomes +parent_acceptance +git_receipts +delivery_receipts +session_control +attempt_root +trace_root +``` + +For a child, `layer_inputs` is the immutable child-local copy, never the +parent's mutable source. Optional origin context uses stable keys +`parent_goal`, `parent_goal_contract`, `parent_handoff`, and +`accepted_child_request`; parent/child-only keys are explicitly null when the +topology makes them inapplicable. Active-child identity is read from the named +`session_state`/`children_index`, never reconstructed from directory names. +Prompts refer to Assignment keys and their rendered absolute values, never +infer state from cwd and never concatenate `.loopy_loop/sessions/...` +themselves. +Team-harness direct-spawn envelopes carry the delegate's own assignment/output +paths plus the relevant layer-state paths. A nested harness coordinator also +inherits the full capability roster and current layer identity; it remains an +attempt-local delegate rather than a durable child session. + +## Workflow and scheduler awareness + +The user must be able to inspect what the outer roles think will happen, and an +orchestrator must know which scheduled roles may produce evidence next. Two +different artifacts provide that context. + +### Session-frozen workflow roster + +`workflow_roster.json` is derived from the frozen workflow set and contains all +enabled scheduled roles, including: + +- role ID and plain-language responsibility; +- cadence and ordering configuration (`run_every`, `must_follow`, and related + mechanical settings); +- expected durable outputs and state accountability; +- orchestration completion and terminal-blocker authority; and +- optional eval author/runner responsibilities. + +This roster is stable for the session. It is not the list of agents that a +team-harness coordinator will dynamically spawn. + +Its minimum structural shape is: + +```json +{ + "schema_version": 1, + "session_id": "", + "workflow_contract_sha256": "", + "created_at": "", + "completion_role": "outer", + "roles": [ + { + "workflow_id": "outer", + "responsibility": "", + "cadence": {}, + "expected_outputs": [], + "authorities": [] + } + ] +} +``` + +### Attempt-frozen scheduler view + +`scheduler_view.json` captures the current session phase, recent mechanical +history, cadence counters, and a conditional forecast: + +> If this attempt returns normally and produces no terminal control, child +> request, stop condition, or harness failure, workflow X would be selected +> next for reasons Y. + +The forecast states its assumptions and may include later due roles. It is not +a reservation or semantic eligibility rule. A new control signal, child +request, failure, user update, or recovery event can change the real next +selection. This information lets an outer orchestrator decide, for example, +not to duplicate an expensive eval when `eval_runner` is already due, while +leaving it free to run one immediately when evidence is urgent. + +Its minimum structural shape is: + +```json +{ + "schema_version": 1, + "session_id": "", + "state_revision": 12, + "attempt_id": "", + "workflow_roster_sha256": "", + "history_watermark": 19, + "captured_at": "", + "conditional_forecast": { + "next_workflow_id": "eval_runner", + "reasons": [""], + "assumptions": [ + "current attempt returns normally", + "no terminal control or child request is accepted", + "no stop, failure, user update, or recovery changes state" + ] + } +} +``` + +The scheduler computes this projection from the same frozen config and history +used for real selection. A null `next_workflow_id` is honest when no conditional +selection is currently derivable. + +## Harness capability roster and strength tiers + +`harness_capability_roster.json` is frozen from root execution configuration +for the session tree and rendered into every harness coordinator's prompt. It +contains no credentials. It records the common harness-coordinator +provider/model separately from the delegate catalog, then separates two worker +choices: + +- **harness family**: a distinct enabled agent CLI/provider/tool ecosystem, + useful for different perspectives and failure modes; and +- **strength tier**: the relative capability/cost bundle selected within that + family. + +The canonical stock tiers are: + +| Tier | Intended use | +| --- | --- | +| `frontier` | Maximum-capability work: the hardest architecture/planning, adversarial review, ambiguous debugging, eval-policy/check design, and highest-stakes judging | +| `strong` | Complex reasoning, implementation, and review where high capability matters but the maximum tier is unnecessary | +| `standard` | Balanced default for ordinary implementation, analysis, and review | +| `economy` | Bounded mechanical work, broad reconnaissance, formatting, and low-risk checks | + +The names describe relative strength across providers, not particular models. +For example, an Anthropic-family mapping could be `frontier` → Fable, +`strong` → Opus, `standard` → Sonnet, and `economy` → Haiku. Other harness +families map their corresponding models without changing stock prompts. + +Config may omit an unavailable family/tier mapping, but the frozen roster +materializes every canonical tier for every enabled family and marks missing +cells explicitly as unavailable. A project may add a documented local tier. +The engine validates that configured mappings are well-formed; it does not +certify that a concrete model deserves its semantic label. Stock prompts use +canonical tier names and inspect availability. Model IDs and effort strings +occur only in config and the generated roster, never in stock role prompts. + +An illustrative shape is: + +```json +{ + "schema_version": 1, + "root_session_id": "", + "root_execution_config_sha256": "", + "created_at": "", + "coordinator": { + "provider": "", + "model": "" + }, + "tiers": { + "frontier": "maximum-capability configured bundle", + "strong": "complex high-capability bundle", + "standard": "balanced default", + "economy": "lower cost and latency" + }, + "harnesses": { + "": { + "frontier": {"model": "", "effort": ""}, + "strong": {"model": "", "effort": ""}, + "standard": {"model": "", "effort": ""}, + "economy": {"available": false} + }, + "": { + "frontier": {"model": "", "effort": ""}, + "strong": {"model": "", "effort": ""}, + "standard": {"model": "", "effort": ""}, + "economy": {"model": ""} + } + }, + "default_tier": "standard" +} +``` + +Requested and effective family/model/effort stay in team-harness's audit and +trace records. They inform later review but never become a model-policy gate. + +## Prompt contract for cross-harness collaboration + +Every coordinator prompt should receive the following meaning, rendered +against the actual enabled roster: + +> For consequential planning, design, uncertain analysis, review, or eval-check +> authoring, prefer independent delegates from different enabled harness +> families when that materially improves confidence. Run independent analyses +> in parallel when they do not depend on one another. After a primary artifact +> exists, prefer review by a family different from its primary author. Use the +> `frontier` tier for high-stakes review and eval-policy/check design when it is +> available and worth the cost. You own synthesis and the final layer artifact. +> These are judgment defaults, not quotas or completion gates; do not spawn +> agents merely to satisfy a count. + +Important operational boundaries: + +- parallel delegates write separate findings or trace outputs; they do not + concurrently edit one canonical plan, check set, or implementation file; +- independent pre-draft analyses may run in parallel; +- reviewers run after a stable draft exists, though several reviewers can then + review that draft in parallel; +- one implementation/integration owner incorporates changes and resolves + disagreements; and +- if only one harness family is usable, proceed autonomously and record that + limitation when it materially affects confidence. + +This revives the useful part of the old “one harness works, another reviews” +pattern without restoring a hard-coded Codex/Claude/Gemini chain. The roster, +not vendor names in prompt text, determines what is available today. + +### Role-specific guidance + +| Role | Prompt preference | +| --- | --- | +| `inner` | Parallelize independent research; keep one integration owner; for meaningful changes prefer post-diff review by another enabled family | +| `outer` | Use cross-family analysis/review for high-impact plan changes, architecture choices, leaf acceptance, and completion synthesis when useful | +| `planner` | Seek independent views on program sequencing, cross-phase risk, child acceptance, and final completion when the confidence gain justifies it | +| `dispatcher` | Preserve planner intent; optionally cross-review a consequential child goal, but do not turn dispatch into a mandatory review ceremony | +| `eval_reviewer` | Apply the stronger eval-check authoring protocol below | +| `eval_runner` | Choose judge family/tier deliberately; where practical avoid the primary implementer's and check author's family, without inventing a quorum | + +## Stronger collaboration for eval-check creation + +Eval checks define what later evidence will notice and what it will miss. For a +non-trivial check set, the eval-reviewer coordinator should normally: + +1. ask independent delegates from different available harness families to + analyze goal coverage and likely failure modes in parallel; +2. have one accountable author/integrator draft a single coherent set of + outcome-oriented `harness_judge` checks; +3. give the stable draft to different-family reviewers, in parallel where + useful, asking them to attack: + - missing goal coverage and redundant checks; + - false-positive and false-negative paths; + - implementation coupling and self-grading bias; + - gameability and wording ambiguity; + - evidence discoverability; and + - whether a repo-owned prepared test/eval should also be cited; +4. reconcile disagreements, revise the canonical checks, and record the + important rationale in `eval_state.md` or `decisions/`; and +5. leave candidate analyses and full reviewer transcripts in traces. + +Use `frontier` delegates for this work when the goal is high-stakes or subtle +and the roster provides them. For a trivial check, unavailable second family, +or disproportionate cost, the coordinator may use fewer passes and say why +when that limitation matters. No exact agent count, all-family review, or +review receipt is required for check publication or session completion. + +D4 remains unchanged at its trust boundary: the generic stock set authors +outcome-oriented LLM-judge checks, not agent-invented deterministic checks. +Repo-owned suites and prepared evaluations are legitimate additional evidence +because their criteria were not invented by the current implementer. + +## Optional evaluation and terminal control + +### Evaluation lifecycle + +Eval roles may create checks and observations on mechanical cadence. Their +outputs update `eval_state.md` and, when valid, canonical eval receipts. The +outer orchestrator sees those facts on its next attempt. It may: + +- repair the implementation in response; +- repair or supersede a weak/stale check; +- ask another family to review or rerun it; +- explain why the result does not apply; +- wait for an imminent scheduled evaluator rather than duplicate work; or +- decide from other evidence without running an eval. + +Provenance validation remains strict whenever a receipt exists or is cited: +the receipt must bind the correct session and goal, exact check definitions, +producer and harness, judge settings, raw/canonical report hashes, and +evaluated git state. Its producing workflow must be one of the frozen +contract's `check_runner_roles`; validation is not hard-coded to a role named +`eval_runner`, because outer/planner may invoke an eval directly. Strict +provenance answers “what exactly produced this observation?” It does not answer +“must the orchestrator obey it?” + +Raw-report validation happens when the engine accepts the canonical +session-side receipt: it verifies the available raw/canonical bytes and hashes, +records subject/evaluated-git identity, and seals the compact receipt. A later +cross-attempt control citation validates that accepted receipt, its subject, +producer role, evaluated git identity, and seal; it does not require the +gitignored raw trace bytes still to be retained. An unaccepted workflow-authored +receipt is not promoted merely because control cites it. This preserves D12's +independent trace-retention boundary. + +`goal_check.json` remains readable as a legacy or optional iteration +projection. It is not the session stop switch. An absent, non-passing, stale, +or malformed observation is recorded as an eval diagnostic; it must not: + +- change a normally returned `IterationResult.success` to false; +- consume a generic workflow-harness failure budget; +- increment a terminal-control protocol-failure counter; +- produce `goal_check_broken`; or +- prevent the completion owner from receiving another turn or writing control. + +### Protocol-v3 successful control + +Changing completion authority mid-session would make frozen provenance +ambiguous, so fresh amended workflow contracts use +`session_protocol_version: 3`. Every v2 session—already live or explicitly +created from a custom v2 contract—continues with its frozen v2 owners and +requirements. + +A v3 `goal_met` signal has exact identity and a reasoned disposition. The +example shows the normal evidence and handoff references, but +`evidence_refs`, `eval_receipt_refs`, and `handoff_ref` are optional schema +fields; lists may be empty: + +```json +{ + "schema_version": 3, + "control_id": "", + "state": "stopped", + "reason": "Why this session's own goal is now complete.", + "stop_reason": "goal_met", + "producer": { + "session_id": "", + "workflow_id": "", + "attempt_id": "" + }, + "evidence_refs": [], + "eval_receipt_refs": [""], + "handoff_ref": "session:/project_state/handoff.json", + "created_at": "" +} +``` + +The engine rejects stale identity, a producer other than the frozen completion +role, malformed references, or false provenance. It does not require an eval +reference, a passing verdict, or a same-attempt eval. If receipts are cited, +their bytes and subject identity must validate; the orchestrator's rationale +may explain how conflicting evidence was resolved. `handoff_ref` is expected +and validated when present, but its absence or staleness is reported as outcome +completeness information rather than used to invalidate otherwise authentic +control. + +V1/v2 keep the historical singular `eval_receipt_ref`; v3 deliberately adds +plural `eval_receipt_refs` because an orchestrator may weigh independent or +conflicting observations across attempts. A malformed advisory eval by itself +is only an eval diagnostic. A `control.json` that affirmatively cites a stale, +foreign, or malformed receipt makes a false protocol claim and is rejected as +control; the orchestrator can repair the references or submit a new disposition +without the bad citation. + +The v3 `unresolvable_error` form keeps D5's attempted-routes and evidence +requirements and does not require a handoff or eval. There is still no +`paused`/`waiting_for_human` state. + +### Workflow contract shape + +Completion ownership moves out of the eval sub-contract: + +```yaml +session_protocol_version: 3 +orchestration: + completion_role: outer + plan_owner: outer + handoff_owner: outer + task_acceptance_owner: outer +terminal_blocker_reporting_roles: [outer, inner, eval_reviewer, eval_runner] +evaluation: + advisory: true + check_author_roles: [eval_reviewer, outer] + check_runner_roles: [eval_runner, outer] +``` + +Allowing `outer` in the eval roles is accountability for an outer attempt that +directly coordinates or delegates evaluation; it does not assert that the same +underlying model both implemented and independently judged the work. D4 forbids +invented deterministic gates and the prompts strongly prefer cross-family +check analysis/review and a judge with different failure modes. Independence is +visible in the delegate/judge provenance and weighed by outer—it is not enforced +by preventing the durable orchestrator from obtaining evidence when a scheduled +eval role is unavailable. + +The PM contract names `planner` in the orchestration fields and omits scheduled +eval workflows, while declaring `planner` as a permitted check runner (and +author when it creates target-specific checks) for directly coordinated, +delegated, or prepared program-level evals: + +```yaml +orchestration: + completion_role: planner + plan_owner: planner + handoff_owner: planner + child_acceptance_owner: planner +terminal_blocker_reporting_roles: [planner, dispatcher] +evaluation: + advisory: true + check_author_roles: [planner] + check_runner_roles: [planner] +``` + +The exact schema may normalize repeated owners internally, but the top-level +semantic distinction is required: evaluation produces evidence; orchestration +decides. Declaring planner as a runner authorizes provenance; it does not add a +scheduled PM eval role. + +## State versus traces + +The compact session state contains facts needed to resume, schedule, or explain +the result: + +- goal, topology, plan, task/accepted-work ledger, and decisions; +- workflow roster, scheduler view, and harness capability roster; +- current-state and semantic handoff; +- child outcomes and parent dispositions; +- completion rationale; +- eval headline and receipt links when present; and +- git, delivery, recovery, and trace-seal receipts. + +The gitignored trace tree contains full prompts, visible turns, tool/spawn I/O, +independent candidate analyses, reviewer transcripts, raw eval reports, +process/provider identity, verbose git evidence, timing, and usage. A +coordinator must promote conclusions future attempts need into compact state. +It must never expect a later outer/planner attempt to read megabytes of raw +reports or reconstruct the plan from chat history. + +## Implemented protocol changes + +The 0.8.0/0.5.4 implementation delivers the following coordinated protocol +amendment. Its release validation covers the changed stock workflow defaults. + +1. **Models and workflow contracts** + (`src/loopy_loop/models.py`, workflow contract loader): add protocol v3, + top-level orchestration ownership, optional eval roles, v3 control, roster, + scheduler-view, handoff, and terminal-outcome schemas. Preserve frozen v1/v2 + readers. Audit every `session_protocol_version >= 2` branch and replace it + with explicit version dispatch where v2 and v3 semantics differ; v3 must + never enter `_validate_v2_control` or other v2 eval-gate paths. Preserve the + current compatibility rule that a contract file omitting + `session_protocol_version` defaults exactly to v2; do not let the new stock + default silently upgrade those custom contracts. +2. **Session creation and state** (`sessions.py`): create the canonical semantic + spine, session-frozen workflow roster, and session-tree capability roster; + use atomic writes and logical references. +3. **Assignments** (`assignments.py`, `coordinator_app.py`): add the named + absolute semantic paths—including child-local `layer_inputs` and stable + optional parent-context keys—roster paths, structured roster summaries, and + attempt-frozen scheduler view to `assignment.json` and rendered prompts. +4. **Scheduler context** (`coordinator_app.py`): expose mechanical history and a + conditional forecast without making it an eligibility promise or semantic + gate. +5. **Terminal control** (`coordinator_app.py`): authorize the top-level + completion role; remove the universal same-attempt passing receipt and + `goal_check.json` prerequisite for v3; validate any cited evidence + provenance; snapshot valid `handoff.json` into `session_outcome.json`; treat + absent/malformed handoff as completeness diagnostics; and resolve delivery + receipts across all session attempts rather than only the control attempt. +6. **Advisory eval handling** (`worker.py`, `coordinator_app.py`): keep valid + receipts, but record missing/malformed/non-passing output as diagnostics + rather than mechanical failure, `goal_check_broken`, or terminal starvation. + Validate receipt producers against the frozen `check_runner_roles`, not a + hard-coded `eval_runner` role. +7. **Stock one-layer prompts/contracts** + (`templates/inner_outer_eval/`): make outer the completion/handoff owner; + retain optional scheduled eval roles; make the set standalone-safe; render + the workflow/scheduler/capability context and cross-harness preferences. + Remove inner's “create a plan if absent” bootstrap; inner reports a missing + selection to outer. Replace flat state `accountable_roles` with per-artifact + owner/contributor metadata. Retire v3 `eval_readiness/` and project any + useful headline into `eval_state.md` plus scheduler context; preserve the + frozen v2 reader. +8. **Stock PM prompts/contracts** (`templates/pm_planner_dispatcher/`): ship + planner plus dispatcher only; dispatch phase/milestone outcomes; make planner + own completion/handoff and target-requested final evaluations. +9. **Capability config** (`config.py`): freeze the full matrix rather than only + resolved defaults/prose; use canonical tier semantics; render missing cells; + keep selection guidance/audit-only. Replace arbitrary stock tier examples + with complete `frontier`, `strong`, `standard`, and `economy` examples + without silently rewriting custom configs. +10. **Team-harness boundary**: carry the roster path/summary through nested + caller context and direct-spawn assignments; keep requested/effective + family/model/effort in traces. Change team-harness only where Loopy cannot + supply this through its existing caller/system-prompt contract. +11. **Version/capability handshake** (`coordinator_app.py`, `worker.py`): add + explicit protocol-v3 worker capabilities for the new assignment/control + contract and fail registration with HTTP 426 before dispatch when any are + missing. A v2-capable worker is not implicitly v3-capable. +12. **Docs and target setup**: update `README.md`, `docs/session-layout.md`, and + `docs/http-contract.md` after the code lands. Update Ultimate Memory's PM + plan to phase-sized outcomes, render the prepared final-eval path in its + planner goal, and use the standardized harness tiers. + +## Verification scenarios + +The amended release is not complete until tests prove: + +1. a fresh standalone root `inner_outer_eval` creates and advances its own plan, + handoff, and terminal outcome without parent-only assumptions, while inner + never creates/owns the plan when selection is missing; +2. the same set behaves identically as a child; +3. outer can complete with a valid eval, without any eval, and with a diagnosed + non-passing or malformed advisory eval; +4. optional receipts from any declared runner role are provenance-validated + across attempts, while stale/sibling/foreign or undeclared-role receipts are + rejected as evidence; +5. a PM session exposes only planner and dispatcher, sends a phase/milestone + goal, and lets the child outer decompose it into leaves; +6. parent acceptance consumes the child's handoff/outcome but never closes the + parent automatically; +7. workflow roster and conditional scheduler view are inspectable and frozen at + the documented boundaries; +8. every coordinator sees all enabled harness families and configured + `frontier`/`strong`/`standard`/`economy` cells, with no hard-coded + vendor/model IDs in stock role prompts; +9. eval-check prompts request parallel independent criteria analysis and + different-family review as a preference, while single-family/no-review paths + remain valid; +10. parallel delegates are assigned separate findings/trace outputs and one + coordinator owns integration; any contrary concurrent canonical edit is + observable for repair rather than claimed impossible by an ACL; +11. a valid terminal handoff is hash-bound into the same outcome shape for root + and nested sessions, while missing/malformed handoff is diagnosed without a + crash or control veto and earlier-attempt delivery receipts remain linked; +12. already-live v2 sessions resume under their frozen v2 authority and eval + requirements, v3 never enters a v2-only validation branch, and v3 + `eval_readiness/` is retired without breaking v2 readers; and +13. a worker missing any required v3 capability receives HTTP 426 before a v3 + assignment is mutated or dispatched. + +The coordinated release gate remains formatting, lint, type checking, and the +full relevant suites in loopy-loop and any changed support repositories. + +## Independent design review + +Claude Code and Antigravity independently reviewed the amendment, identified +concrete compatibility/state-path/provenance gaps, and re-reviewed the revised +documents. Both final passes returned **PASS** with no remaining design +blocker. The findings and maintainer dispositions are recorded in the +[Claude Code review](../analysis/claude-code-orchestrator-owned-completion-review.md) +and +[Antigravity review](../analysis/antigravity-orchestrator-owned-completion-review.md). +These verdicts cover the design only. The implemented 0.8.0/0.5.4 changes still +require their own code review and coordinated release evidence before the tags +are published. + +## Alternatives rejected + +**Keep eval-runner as completion owner.** This confuses an evidence producer +with the persistent role that understands the plan and makes one generic eval +shape mandatory. + +**Remove evaluation.** Independent outcome judgment remains valuable. The +amendment changes authority and cadence, not the availability or provenance of +evaluation. + +**Let both outer and eval-runner write success.** Dual terminal authority is +ambiguous and race-prone. One declared orchestrator decides after considering +all available evidence. + +**Hard-code an author/reviewer vendor chain.** It becomes stale, fails when one +provider is unavailable, and prevents the coordinator from adapting. Use the +enabled capability roster and semantic preferences. + +**Require two reviewers or all harness families.** Review value is +task-dependent; a quota wastes cost and becomes another gate. Preserve the +preference and audit trail, not a count. + +**Allow parallel agents to edit the canonical artifact together.** Independent +analysis parallelizes well; racing writes do not. One coordinator/integrator +owns the durable artifact. + +**Make the scheduler forecast a promise.** Terminal control, child dispatch, +failures, stops, and recovery can legitimately change the next workflow. A +conditional view provides useful context without lying. + +**Dispatch exact leaves from the PM layer.** This hides the real plan from the +child outer role, bloats parent context, and makes the one-layer orchestrator a +mere executor. Dispatch coherent outcomes and let each layer plan its own goal. + +**Put semantic plans and handoffs only in traces.** Traces are large, +gitignored, and independently retained. Continuity needs compact state that can +be read without reconstructing conversations. diff --git a/design/designs/recursive-loop-layer-contract.md b/design/designs/recursive-loop-layer-contract.md index 13b071f..aa869f7 100644 --- a/design/designs/recursive-loop-layer-contract.md +++ b/design/designs/recursive-loop-layer-contract.md @@ -1,9 +1,10 @@ # Design: Recursive Loop Layers, Dynamic Agent Delegation, and Execution Records -**Status:** Implemented and released in loopy-loop 0.7.0, team-harness 0.5.0, -and eval-banana 0.3.2. The canonical eval-definition digest interoperability -and complete per-check trace fix ships in loopy-loop 0.7.1 with eval-banana -0.3.5. +**Status:** The recursive/provenance/trace baseline was implemented in +loopy-loop 0.7.0–0.7.1, team-harness 0.5.0, and eval-banana 0.3.2–0.3.5. +The protocol-v3 orchestrator/state/schedule/capability amendment accepted on +2026-07-17 is implemented in loopy-loop 0.8.0 and team-harness 0.5.4. V2 +sessions retain their frozen historical contract. **Date accepted:** 2026-07-15 @@ -16,35 +17,48 @@ This is the binding companion design for D10–D12 in its boundaries exist. Exact HTTP bodies live in [`docs/http-contract.md`](../../docs/http-contract.md); exact paths and artifact purposes live in [`docs/session-layout.md`](../../docs/session-layout.md). +The accepted v3 completion, handoff, planning-granularity, and cross-harness +amendment is specified in +[`orchestrator-owned-completion-and-cross-harness-review.md`](./orchestrator-owned-completion-and-cross-harness-review.md). +Where this document describes v2's mandatory same-attempt eval gate for +compatibility, that newer design governs fresh protocol-v3 sessions. ## Summary The design keeps the existing principles: files and git are durable truth, one loopy worker advances one assignment at a time, team-harness coordinators may -delegate dynamically, and agents plus evaluation judge semantic quality. +delegate dynamically, and one named durable orchestrator judges semantic +completion from the available evidence. -The implemented contract adds five boundaries: +The recursive contract is organized around six boundaries. Protocol v2 shipped +the recursive identity, portable-path, delegation, provenance, and trace +baseline; the accepted v3 amendment strengthens the semantic-state and context +parts called out below. 1. A durable loop layer is one recursive **session node**. One-loop, planner/dispatcher double-loop, and future triple-loop systems are different depths of the same state machine, not separate schedulers. -2. Each session has a scoped goal, state, decisions, evals, and optional child. - A child result informs its parent but never completes the parent's broader - goal automatically. +2. Each session has a scoped goal, canonical plan/state/decision/handoff spine, + optional eval evidence, and optional child. A child result informs its + parent but never completes the parent's broader goal automatically. 3. A team-harness coordinator owns one workflow assignment and may dynamically spawn researchers, implementers, reviewers, or nested harness coordinators. Those delegates remain inside its session layer. 4. Durable records use portable logical references. Each running coordinator and direct spawn also receives explicit worker-local absolute paths, so no agent must infer which layer or directory it owns. -5. Compact correctness/recovery evidence stays with the session. Detailed +5. Every coordinator sees the complete scheduled-workflow roster, conditional + next-workflow forecast, and enabled harness/model capability roster. These + are context for judgment, not eligibility or team-shape gates. +6. Compact correctness/recovery evidence stays with the session. Detailed attempt I/O stays in a separately gitignored raw trace tree with explicit completeness and crash-safe sealing. This is structure at system boundaries, not programmable micromanagement. The engine validates identity, hashes, schemas, provenance, and state-machine shape. It does not prescribe a fixed agent graph, model choice, semantic plan, -branch policy, filesystem ACL, or deterministic quality gate. +evaluation requirement, branch policy, filesystem ACL, or deterministic +quality gate. ## Three kinds of nesting @@ -54,7 +68,7 @@ relationships. The contract distinguishes them: | Concept | Lifetime | Responsibility | | --- | --- | --- | | Loopy coordinator service | process | Scheduling, recovery, and engine state transitions | -| Session layer | durable | One scoped goal, semantic state, decisions, evals, attempts, and optional child | +| Session layer | durable | One scoped goal, semantic plan/state/handoff, decisions, attempts, optional eval evidence, and optional child | | Workflow role | one or more attempts | A set-defined responsibility such as planner, outer, inner, eval reviewer, or eval runner | | Harness coordinator | one attempt | Owns the workflow assignment and dynamically orchestrates agents | | Spawned agent | part of one attempt | Performs a delegated research, implementation, review, or test task | @@ -65,6 +79,10 @@ owner merely because it is called a subagent. A new loop layer exists only when the active session publishes a child request and loopy creates a child session. +The stock `inner_outer_eval` layer has the same semantics as a root or a child: +its `outer` owns that session's plan, handoff, and completion decision. Parent +input is optional origin context, not a hidden requirement. + This distinction preserves both D2 and dynamic orchestration: only one deepest loopy assignment advances at a time, while team-harness may run several agents inside that assignment when their work is independent. @@ -77,9 +95,12 @@ Every session owns: - immutable identity, topology, scoped goal, and workflow contract; - scheduler/recovery state and one revisioned state ledger; -- semantic progress, meaningful decisions, and append-only user inputs; +- `project_state/plan.md`, `tasks/`, `current_state.md`, `decisions/`, + `finished.md`, optional `eval_state.md`, and a rolling `handoff.json`; +- semantic progress, accepted-work evidence, and append-only user inputs; - workflow attempts and compact result/evidence receipts; -- session-scoped eval definitions, eval receipts, and terminal control; and +- optional session-scoped eval definitions/receipts and orchestrator-owned + terminal control; and - child requests, factual child outcomes, and parent acceptance records. The same parent/child edge composes to any depth: @@ -128,14 +149,16 @@ must inspect and integrate the result. There is no path-level write fence. An implementation delegate may edit the target repository, and a coordinator may deliberately delegate a state artifact. The assignment records intended ownership and observable effects; -evaluation detects bad results and the loop repairs them, as required by D8. +tests, reviews, optional evals, and direct inspection surface problems for the +orchestrator to repair or disposition, as required by D8. ## State/evidence and traces Users reason about two worlds: - **State and evidence** answer what the system believes, why, and what happens - next. This includes goals, progress, decisions, child handoffs, eval results, + next. This includes goals, layer plans, accepted work, decisions, semantic + handoffs, schedule/capability context, optional eval observations, git/delivery receipts, task ownership, and recovery records. - **Traces and logs** answer how one attempt unfolded. This includes generated prompts, visible turns, tool and spawn I/O, commands, process/provider @@ -155,12 +178,17 @@ The important layout is: ├── repository.json ├── sessions// │ ├── session.json, state.json, goal.md, goal_contract.json -│ ├── workflow_contract.json, events.jsonl, control.json -│ ├── project_state/, inputs/, eval_checks/, eval_receipts/ +│ ├── workflow_contract.json, workflow_roster.json, events.jsonl, control.json +│ ├── control_rejected/, protocol_failures/ +│ ├── harness_capability_roster.json, session_outcome.json +│ ├── project_state/ +│ │ ├── plan.md, tasks/, current_state.md, decisions/, finished.md +│ │ └── eval_state.md, handoff.json +│ ├── inputs/, eval_checks/, eval_receipts/ │ ├── child_requests/, child_outcomes/, parent_acceptance/ │ ├── git_receipts/, delivery_receipts/, trace_seals/ │ ├── iterations/_/ -│ │ ├── workflow_snapshot//assignment.json +│ │ ├── workflow_snapshot//assignment.json, scheduler_view.json │ │ ├── prompt.txt, result.json, pending_finished_request.json │ │ └── goal_check.json, trace_ref.json, salvage.json │ └── children//... @@ -190,13 +218,23 @@ the selected repository/session/trace root. These references survive a moved checkout or a different worker mount. Before execution, loopy resolves the relevant references into a frozen -`assignment.json`. That assignment gives the harness coordinator explicit -absolute paths to its repository, own/parent/root sessions, scoped goal, -project state, eval, child handoff, control, git/delivery evidence, attempt, and -trace locations. Its own absolute assignment path appears near the start of -the effective prompt. Agents do not derive paths from cwd or ambiguous names -such as `project_state/current_state.md`; cwd remains the target repository for -normal development tools. +`assignment.json`. The path map explicitly names absolute `layer_goal`, +`layer_goal_contract`, child-local `layer_inputs`, `layer_plan`, `layer_tasks`, +`layer_current_state`, +`layer_decisions`, `layer_finished_ledger`, `layer_eval_state`, +`layer_handoff`, `session_state`, `session_outcome`, `workflow_roster`, +`workflow_contract`, `scheduler_view`, `harness_capability_roster`, +`user_inputs`, `child_requests`, `children_index`, `child_outcomes`, +`parent_acceptance`, `git_receipts`, `delivery_receipts`, `session_control`, +attempt, and trace locations. Parent/request paths are additional origin +context, not substitutes for the session's own state; stable optional keys +include `parent_goal`, `parent_goal_contract`, `parent_handoff`, and +`accepted_child_request`. Topology-inapplicable paths are explicitly null. + +The assignment's own absolute path appears near the start of the effective +prompt. Agents do not derive paths from cwd or ambiguous names such as +`project_state/current_state.md`; cwd remains the target repository for normal +development tools. ### Immutable session and goal identity @@ -211,21 +249,36 @@ the parent's differently scoped completion criteria merely because it shares tree-wide execution settings such as provider, coordinator model, model tiers, and recovery policy. +The same rule works in reverse: a root `inner_outer_eval` session has no parent +request and needs none. Its own goal contract and canonical state spine are +sufficient for outer to plan, implement, optionally evaluate, hand off, and +complete the layer. + ### Frozen workflow and attempt contract -Each workflow set declares its layer kind, workflow-role responsibilities, -state accountability, eval author/runner/control owners, task-acceptance owner, -terminal-blocker reporters, and child interface. This describes responsibility -for prompts and audits; it is not a filesystem permission list. +Each amended workflow set declares its layer kind, workflow-role +responsibilities, state accountability, a top-level orchestration/completion +owner, optional eval authors/runners, terminal-blocker reporters, and child +interface. This describes responsibility for prompts and audits; it is not a +filesystem permission list. Protocol-v2 contracts retain their historical eval +control owner only for frozen-session compatibility. -For a v2 session, the complete selected workflow contract is also stored in +For a v2 or v3 session, the complete selected workflow contract is also stored in coordinator-owned `state.json`. The adjacent `workflow_contract.json` and its hash in `session.json` are agent-visible projections: the coordinator restores them from state before freezing a later attempt if both were rewritten. This keeps the protocol and role owners stable across attempts while leaving the -files inspectable. An explicit `contract.yaml` that omits -`session_protocol_version` selects v2; only a workflow set with no contract at -all uses the documented derived-v1 compatibility path. +files inspectable. New stock contracts explicitly select v3. Existing explicit +v2 contracts and derived-v1 compatibility sets keep their documented behavior; +the loader must never silently reinterpret an old contract as v3. + +The coordinator also freezes two readable context projections. The +session-wide `workflow_roster.json` lists every scheduled role, responsibility, +cadence, dependency, expected output, and authority. The attempt-local +`scheduler_view.json` summarizes recent mechanical history and states which +workflow would run next **if** the current attempt completes without control, +child dispatch, stop, failure, or recovery. That forecast is context, not a +promise or eligibility gate. Before dispatch, the coordinator freezes the selected workflow config, prompt body, workflow contract, and root execution config beneath that attempt's @@ -234,8 +287,8 @@ and freezes its SHA-256 in the task. The worker verifies the repository, snapshot identity, hashes, reconstructed assignment, and absolute location before calling a model. Scheduler and worker therefore cannot silently execute different live files after an attempt was selected. Runtime semantic context, -such as the newest eval-readiness record, remains deliberately late-bound and -is captured in the rendered attempt input. +such as the newest handoff/current-state revision and optional eval headline, +remains deliberately late-bound and is captured in the rendered attempt input. The rendered loopy prompt and team-harness coordinator input are persisted before their respective provider calls. A pre-first-turn crash still leaves a @@ -245,14 +298,22 @@ legible record of the attempted input. ### The coordinator chooses the team -The harness coordinator receives the session/attempt contract and owns its -workflow result. It decides whether to delegate, what roles and tasks exist, -which model tier and effort suit each task, what may run concurrently, whether -to retry or reroute, and when enough evidence exists to synthesize a result. - -Workflow prompts may suggest useful roles but must not encode a mandatory -researcher/implementer/reviewer graph. The point of the coordinator is to -adapt its team to the live situation. +The harness coordinator receives the session/attempt contract, full scheduled +workflow roster, conditional scheduler view, and session-tree-frozen harness +capability roster. The capability roster enumerates all enabled harness +families and their configured `frontier`, `strong`, `standard`, and `economy` +model/effort bundles, including unavailable cells. The coordinator owns its +workflow result and decides whether to delegate, what roles and tasks exist, +which family/tier suits each task, what may run concurrently, whether to retry +or reroute, and when enough evidence exists to synthesize a result. + +Workflow prompts should prefer parallel independent analysis and review by a +different enabled harness family for consequential work when useful. Eval-check +creation gets the strongest form of that guidance: cross-family criteria +analysis, one integrator, then different-family review of a stable draft. These +are judgment defaults, not a mandatory researcher/implementer/reviewer graph, +spawn count, or model gate. The point of the coordinator is to adapt its team +to the live situation. ### Every direct spawn knows its place @@ -265,6 +326,8 @@ that should not rely on remembered prompt boilerplate: - dynamic delegated role, task ID, objective, and expected outputs; - absolute parent assignment, direct-agent assignment, output, and relevant state paths; +- the delegate's selected harness/model/effort plus the absolute capability + roster path (a nested harness coordinator receives the full roster summary); - the delegate's state responsibility; and - both the authored and effective prompts. @@ -291,11 +354,15 @@ is created only through the loopy child-request protocol. ### Request and input freezing -The active workflow atomically publishes a v2 child request under +The active dispatcher atomically publishes a versioned child request under `child_requests/pending/`. It identifies the request and originating parent -attempt/work item, child workflow set, scoped goal, completion/stop criteria, -constraints, deliverables, required evidence, and hashed logical input -references. +attempt/milestone, child workflow set, scoped outcome, completion/stop criteria, +constraints, deliverables, relevant evidence, and hashed logical input +references. The planner normally selects a coherent phase, milestone, or +integrated feature outcome. The dispatcher preserves it rather than reducing +it to a prescribed leaf; the child outer owns work-package/task/PR +decomposition. The planner may split or combine outcomes when useful—the +engine does not enforce semantic size. When the source planning record is mutable, the workflow first writes an immutable per-request selection snapshot and hashes that snapshot in the @@ -323,100 +390,105 @@ different content. When the child becomes terminal, the engine writes a factual outcome containing request/child/goal identity, lifecycle, measured usage, evidence references, -trace reference, and artifact-presence/completeness facts. It reports what -happened; it does not judge that the parent should accept it. +trace reference, and artifact-presence/completeness facts. The same +topology-neutral `session_outcome.json` shape is used for every v3 terminal +lifecycle, both root and child, including engine stops that have no terminal +`control.json`. The engine freezes terminal identity and accepted +control/handoff bytes in state, so later edits or restart-time regeneration +cannot rewrite the linked outcome basis. The outcome reports what happened; it +does not judge that the parent should accept it. Missing handoff remains +visible incompleteness rather than a hidden engine veto. Invalid handoff JSON +is diagnosed rather than crashing stack recovery. Delivery evidence is +resolved across the whole session—not only the terminal attempt—because +implementation, PR creation, merge, and completion synthesis commonly occur +in different attempts. The parent workflow separately records `accepted`, `rework`, `reroute`, or a terminal-blocker disposition, together with evidence reviewed, rationale, deliverable/commit references, and any superseding request. Child `goal_met` is evidence for this decision, never the decision itself. -## Evaluation and terminal control +## Optional evaluation evidence and orchestrator-owned terminal control -### Every layer evaluates its own goal +### Every layer owns its completion decision -The delivery layer evaluates its task. A feature layer evaluates the integrated -feature using child evidence. The root evaluates the program or release. A -green leaf cannot close a broader ancestor goal. +Each durable session decides only its own goal. A delivery child can finish its +milestone without proving its parent's feature or release complete. The +workflow contract names one persistent orchestration/completion role: -Each workflow contract names the check author, check runner, task-acceptance -owner, terminal `goal_met` owner, and roles allowed to report a terminal -blocker. In the stock `inner_outer_eval` set: +- `outer` in the stock `inner_outer_eval` set; and +- `planner` in the stock `pm_planner_dispatcher` set. -- `outer` accepts task evidence and records readiness for layer evaluation; -- `eval_reviewer` authors outcome-oriented `harness_judge` checks; and -- `eval_runner` executes them, writes the receipt, and alone may request - session `goal_met`. +Eval author/runner roles are optional evidence producers. The orchestrator may +use their observations, invoke an eval directly, wait for an imminent scheduled +eval shown in its scheduler view, rerun/supersede weak evidence, or decide from +other facts. The stock PM set does not duplicate child eval roles; a target's +goal may still instruct planner to run prepared program-level evals near the +end. -Readiness is immutable semantic context rendered into later prompts. It is not -a scheduler condition: the stock scheduler retains mechanical cadence so the -engine does not interpret work quality. This preserves D3, D4, and D8. +The stock scheduler retains mechanical cadence so the engine does not interpret +semantic readiness. A scheduled eval is an opportunity for independent +evidence, not a condition for another role to be eligible or for the session to +close. -### Receipt provenance +### Eval provenance remains strict when evidence exists -The canonical eval receipt binds the verdict to: +A canonical eval receipt still binds: - root/session/goal identity and evaluated git state; - producing workflow, iteration, attempt, and harness run; -- every recursively discovered regular check file and its eval-banana - canonical definition hash copied from the raw report; +- every check definition and canonical definition hash; - judge provider, model, and reasoning effort; -- individual results and the final verdict; -- one canonical report plus its hash; and -- exactly one raw report from the producing attempt's trace plus its hash. - -For a passing verdict, the engine also verifies the raw eval-banana report's -absolute project/output paths, all-pass threshold/status, zero judge exits, -check identities, and effective judge settings. It recaptures live HEAD and the -versioned dirty-tree digest before terminal acceptance. These are provenance -and transport checks; the coordinator does not reinterpret the LLM judge's -semantic reasons or add deterministic stock checks. - -New sessions use iteration-local `goal_check.json` only as a projection of the -receipt. Verdict, reason, and receipt reference must agree. A projection or -receipt failure is recorded with field-qualified repair evidence and cannot -close the session. +- individual results and final verdict; and +- canonical/raw report paths and hashes. + +The engine validates these facts whenever it accepts or resolves the receipt, +and validates cited evidence against the exact subject. The producing workflow +must be in the frozen contract's declared runner roles rather than matching a +hard-coded `eval_runner` name. This proves what was observed. It does not make +the verdict terminal authority. + +Raw report bytes/hashes are validated when a canonical receipt is accepted and +sealed into compact session state. Later cross-attempt citations validate that +accepted receipt plus subject, producer role, evaluated-git identity, and seal; +they do not require independently retained gitignored trace bytes. Trace loss +therefore cannot become a delayed completion gate under D12. + +`goal_check.json` remains readable as a legacy or optional iteration +projection. An absent, non-passing, stale, or malformed advisory eval becomes a +field-qualified diagnostic. It does not flip a normally returned +`IterationResult.success`, increment a generic harness-failure counter, produce +`goal_check_broken`, consume terminal-control protocol-failure capacity, or +prevent another orchestrator turn. ### Identity-bound terminal control -Both v2 terminal signals must come from the exact session, workflow, and -attempt currently being completed. An older attempt, sibling session, or -spawned delegate cannot leave a signal for later work to apply. - -`goal_met` must be produced by the workflow contract's terminal-control owner -and cite the matching passing same-session eval receipt. The D5 last-resort -`unresolvable_error` form instead requires an allowed current role, a specific -blocker, and routes already tried. It does not require an eval receipt: - -```json -{ - "schema_version": 2, - "control_id": "control-9", - "state": "stopped", - "reason": "The required deployment identity is unavailable after checking documented local and CI routes.", - "stop_reason": "unresolvable_error", - "producer": { - "session_id": "session-delivery", - "workflow_id": "inner", - "attempt_id": "attempt-inner" - }, - "attempted_routes": ["documented local identity", "existing CI identity"], - "evidence_refs": ["session:/protocol_failures/deployment-blocker.json"], - "created_at": "2026-07-15T12:00:00Z" -} -``` - -A spawned agent reports a blocker to its harness coordinator; the accountable -workflow role publishes the layer-owned signal. No paused or -waiting-for-human state exists. - -Malformed v2 control is atomically archived under `control_rejected/` and -described under `protocol_failures/`; the session remains repairable until the -configured consecutive-protocol-failure cap is reached. The protocol version -comes from the frozen workflow contract, so rewriting mutable session files -cannot downgrade a live v2 attempt. - -Exact receipt, projection, `goal_met`, and rejection schemas are defined in the +Every terminal signal must come from the exact session, workflow, and attempt +currently being completed. An older attempt, sibling session, or spawned +delegate cannot leave a signal for later work to apply. + +For protocol v3, `goal_met` comes from the frozen top-level completion role and +contains a non-empty rationale. Evidence references are optional and may be +empty; any asserted eval receipt may come from another attempt in the same +session and is provenance-validated when cited. The engine does not require a +passing verdict or reinterpret the orchestrator's semantic disposition. + +The D5 last-resort `unresolvable_error` form still requires an allowed current +role, a specific blocker, autonomous routes already tried, and evidence when +available. It requires no eval or handoff. A spawned agent reports upward to +its harness coordinator; the accountable workflow role publishes any +layer-owned signal. No paused or waiting-for-human state exists. + +Malformed control is atomically archived under `control_rejected/` and +described under `protocol_failures/`; the session remains repairable until its +configured protocol-failure cap is reached. Advisory eval diagnostics are not +control failures; a control record that affirmatively cites invalid evidence is +itself a false protocol claim and is rejected until repaired. V2 sessions, +including ones explicitly created by custom v2 contracts, retain their frozen +historical same-attempt eval/control semantics; fresh stock sessions explicitly +select v3 in the loopy-loop 0.8.0 implementation. + +Exact v1/v2 compatibility and v3 control/receipt schemas are documented in the HTTP contract. ## Git, branches, PRs, and delivery @@ -436,8 +508,9 @@ state machine. The engine records facts but does not reject an assignment merely because the tree is dirty, no PR exists, or a branch differs. Research, repair, and local -work can legitimately have those shapes. Parent and eval roles assess the -evidence and repair problems under D8. +work can legitimately have those shapes. The accountable orchestrator assesses +the evidence, using parent review and optional eval observations, and repairs +or dispositions problems under D8. ## Observable agent I/O and trace lifecycle @@ -522,32 +595,39 @@ recursive scheduler. No parent and active child are dispatched together. ## Compatibility and coordinated rollout -Fresh sessions use protocol/state v2. Existing v1 trees resume in place: +The released 0.7 baseline creates protocol/state v2 sessions. The 0.8.0 +orchestrator-owned implementation introduces explicit protocol v3 for fresh +amended workflow contracts. Existing trees resume in place: - missing state/session schema means v1, and topology may be reconstructed from physical nesting and parent pointers; - v1 child requests, goal checks, terminal control, `harness_outputs/`, raw eval directories, and markdown updates remain readable under a protocol-v1 workflow contract; -- both legacy flat and v2 pending child-request inboxes are observed; and +- v2 sessions retain their frozen eval-owned same-attempt completion contract; +- legacy flat and versioned pending child-request inboxes remain readable; and - active legacy sessions are not relocated or retroactively assigned stronger - provenance. + provenance or different completion authority. -Packaged workflow contracts declare protocol v2 and treat v1 output as a -repairable protocol failure. Custom sets without a contract receive a -conservative derived v1 role contract, but a fresh tree still requires the v2 -worker handshake. +In the 0.8.0 implementation, packaged workflow contracts explicitly declare +protocol v3 and treat v1/v2-shaped output from a v3 attempt as a repairable +protocol failure. Custom sets retain the version they declare; sets without a +contract receive the conservative derived-v1 role contract. A contract file +that omits `session_protocol_version` remains pinned to the historical +explicit-contract default of v2. There is no implicit version upgrade. The cross-repository contract is fail-fast: - team-harness 0.5.0 supplies caller-owned run paths, input durability, direct-spawn envelopes/streams, nested caller context, and its capability - API; + API; team-harness 0.5.4 adds protocol-v3 capability-roster context transport; - eval-banana 0.3.5 supplies hermetic config selection, exact flat output, a public canonical check-definition digest, effective judge metadata, exact judge inputs, and collision-safe per-check artifacts; and -- loopy-loop 0.7.1 requires the advertised capabilities and returns HTTP 426 - before mutation when a fresh v2 tree meets an older worker. +- loopy-loop 0.7.1 requires the released v2 capabilities and returns HTTP 426 + before mutation when a fresh v2 tree meets an older worker. Loopy-loop 0.8.0 + adds and advertises roster/scheduler/handoff/control capabilities in the same + fail-fast manner; advertising v2 support does not imply v3 support. Support packages are published before the loopy-loop version that consumes them, so the loopy dependency lock resolves against public artifacts. Editable @@ -559,10 +639,14 @@ The three repositories test the boundary they own: - loopy covers recursive three-depth dispatch/unwind, stop/usage projection, assignment and reference confinement, child request/input freezing, - receipt/control provenance and repair, recovery, git evidence, and trace - finalization/integrity; + receipt/control provenance and repair, recovery, git evidence, trace + finalization/integrity, standalone root/nested parity, completion with and + without eval, advisory-eval diagnostics, canonical plan/handoff/outcome, + workflow/scheduler views, phase-sized PM dispatch, and capability-roster + rendering; - team-harness covers caller-owned runs, pre-call coordinator input, - direct-spawn assignments and canonical streams, nested caller identity, and + direct-spawn assignments and canonical streams, nested caller identity, + capability-roster propagation, requested/effective delegate settings, and structured process failures; and - eval-banana covers hermetic execution, check hashes, flat raw output, and reported judge settings. @@ -572,22 +656,25 @@ test suite in all three repositories. ## Independent implementation review -Claude Code and Antigravity independently reviewed the settled implementation and the -final bounded-shutdown follow-up. Both returned PASS with no remaining blocker. Their -evidence and adjudicated suggestions are recorded in the +Claude Code and Antigravity independently reviewed the released v2 baseline and the +final bounded-shutdown follow-up. Both returned PASS for that scope. Their evidence and +adjudicated suggestions are recorded in the [Claude Code review](../analysis/claude-code-recursive-loop-implementation-review.md) and [Antigravity review](../analysis/antigravity-recursive-loop-implementation-review.md). -This establishes code readiness; it does not bypass the support-package publication, -lock refresh, and install-path CI sequence above. +This does not establish implementation readiness for the later v3 amendment. The v3 +implementation is present for 0.8.0/0.5.4 but still requires its own implementation +review and coordinated release sequence before publication. ## Alternatives rejected **Fixed subagent graphs.** The coordinator sees the live problem and should decide whether it needs research, implementation, review, debugging, or no -delegation. +delegation. Cross-harness review is a preference informed by the enabled roster, +not a mandatory graph or quota. **Treat every spawn as a loop layer.** Ephemeral tasks do not need durable goal, -recovery, and evaluation state. Only an explicit child request creates a layer. +recovery, and optional-evidence state. Only an explicit child request creates a +layer. **Separate one-, double-, and triple-loop schedulers.** They would duplicate transition and recovery logic. One recursive node and edge express all depths. @@ -597,10 +684,14 @@ durable records must survive a moved checkout. Store logical identity and render absolute paths per attempt. **Enforce ownership with ACLs or scheduler vetoes.** D8 requires accountable -effects, evaluation, and repair rather than preventive fences. +effects, visible evidence, and repair/disposition rather than preventive fences. + +**Let a child outcome close its parent.** The child and parent have different +goals; integration or release work may remain after a child completes. -**Let a child verdict close its parent.** The child and parent have different -goals; integration or release work may remain after a leaf passes. +**Require a passing eval before control.** Evaluation is valuable evidence but +does not own the layer plan or integrate all other evidence. The declared +orchestrator decides. **Put correctness facts only in traces.** Detailed-trace retention is independent, so compact recovery and acceptance evidence stays in the session. @@ -624,7 +715,8 @@ role clarity, or evidence quality and would weaken D1's inspectable model. - `src/loopy_loop/recovery.py` and `sessions.py`: process/session recovery and atomic artifact helpers. - `src/loopy_loop/templates/inner_outer_eval/` and - `pm_planner_dispatcher/`: layer-scoped eval and recursive planning prompts. + `pm_planner_dispatcher/`: standalone layer planning, optional eval evidence, + milestone dispatch, handoff, and cross-harness prompt guidance. Cross-project anchors are team-harness's `caller_contract.py`, `harness.py`, and `tools/agent_tools.py`, plus eval-banana's `cli.py` and `config.py`. diff --git a/design/designs/success-semantics-and-evaluation.md b/design/designs/success-semantics-and-evaluation.md index 997e68b..7227699 100644 --- a/design/designs/success-semantics-and-evaluation.md +++ b/design/designs/success-semantics-and-evaluation.md @@ -1,206 +1,227 @@ -# Design: Success Semantics and Evaluation Strategy +# Design: Mechanical Success, Orchestrator Completion, and Evaluation Evidence + +**Status:** Accepted and implemented. D3's mechanical-success behavior is +released; the protocol-v3 orchestrator-owned completion amendment is +implemented in loopy-loop 0.8.0 and team-harness 0.5.4. V2 sessions retain +their frozen historical contract. -**Status:** Accepted (documenting decisions already in the codebase) **Date recorded:** 2026-07-12 -**Applies to:** `loopy-loop` coordinator/worker loop and the packaged -`inner_outer_eval` workflow set. -This document records two design decisions that are **deliberate and load-bearing**, -but were until now implicit in the code rather than written down. A reader skimming -the code can easily mistake each for a defect. They are not defects. This document -exists so the next reader — human or agent — does not "fix" them by accident. +**Date amended:** 2026-07-17 + +**Applies to:** the loopy-loop coordinator/worker boundary, terminal control, +and optional evaluation in packaged workflow sets. -Both decisions share one principle: +This document is the binding companion for D3 and D4 in +[`design/decisions.md`](../decisions.md). The complete recursive role, state, +handoff, schedule, and cross-harness contract lives in +[`orchestrator-owned-completion-and-cross-harness-review.md`](./orchestrator-owned-completion-and-cross-harness-review.md). -> **Do not infer semantic success from noisy mechanical signals. Push the -> success/acceptance decision to an explicit, purpose-built evaluation layer.** +The shared principle is: -D11 and its companion -[`recursive loop-layer contract`](./recursive-loop-layer-contract.md) refine -how that evaluation layer composes across session depths: each session evaluates -its own goal and names one terminal goal-control owner. This document remains -authoritative for D3/D4's mechanical-success and LLM-as-judge boundaries; the -new design adds subject provenance and ownership without changing either. +> Do not infer semantic success from noisy mechanical signals. Give one named +> durable orchestrator the relevant evidence and responsibility to decide. ---- +Evaluation can be excellent evidence. It is not the universal owner of that +decision. -## Decision 1 — Iteration success means "the assignment ran," not "the work is good" +## Decision 1 — Iteration success means “the assignment ran,” not “the work is good” ### Decision -`IterationResult.success` is `True` whenever a `team-harness` run returns normally, -and `False` only when the harness itself raises (`ConfigError`, `TeamHarnessError`, -or an unexpected exception). It is **not** a judgment about whether the requested -work was actually accomplished. `TeamHarnessResult.agents` (per-worker statuses and -exit codes) is intentionally **not** consulted to decide iteration success. +`IterationResult.success` is `True` whenever a `team-harness` invocation +returns normally and `False` only when the harness itself raises. It is not a +judgment about whether the requested work was accomplished. Per-worker status +and exit codes in `TeamHarnessResult.agents` are intentionally not converted +into that boolean. -Reference: `src/loopy_loop/harness_runner.py` — `_normalize_harness_result()` -returns `success=True`; the `success=False` paths live only in `run_harness_iteration()`'s -exception handlers. +Reference: `src/loopy_loop/harness_runner.py` — +`_normalize_harness_result()` returns `success=True`; the `success=False` paths +are exception handlers in `run_harness_iteration()`. -**Semantic success is decided elsewhere**, by artifacts the workflow writes: +Semantic completion is a separate, explicit act. Each workflow contract names +one persistent orchestration role. That role writes identity-bound +`control.json` when it judges its session goal complete: -- `control.json` — the session stop switch (`running` → `stopped` with a - `stop_reason`). In a fresh v2 session it must identify the exact current - session/workflow/attempt; successful control comes from the declared - goal-control role and cites the same-session eval receipt. This, and only - this, stops the loop. -- `goal_check.json` — a per-iteration projection of the canonical eval receipt. - Evidence only; a valid `goal_check.json` does **not** by itself stop the loop. +- `outer` for `inner_outer_eval`; and +- `planner` for `pm_planner_dispatcher`. -### Context / why +The orchestrator may consider its plan and accepted-work ledger, implementation +evidence, repo-owned tests, direct reviews, child outcomes, decisions, git and +delivery receipts, and optional eval observations. No one evidence type is a +generic protocol prerequisite. + +### Why mechanical success stays narrow `team-harness`'s coordinator is an orchestrator, not a build system. It can -legitimately return a normal result after a worker has failed — it may synthesize a -final answer, decide it has enough information, or route around a dead worker. Worker -exit codes are therefore a **noisy** proxy for "did the assignment succeed": a -non-zero worker can accompany a perfectly good outcome, and an all-green set of -workers can accompany a useless one. Mapping those signals to a boolean would -manufacture false precision. - -So `loopy-loop` draws the line at the only thing it can observe reliably — *did the -assignment run to completion without the harness itself erroring* — and delegates the -"was it any good" question to an explicit evaluation step that produces -an eval receipt and matching `goal_check.json`, with workflow-owned -`control.json` as the actual gate. D11 defines the exact role and provenance -contract; D5 keeps human involvement out of normal operation. - -This has been the behavior since the first commit of `harness_runner.py` -(`a4cca5e`, 2026-04-19); it is original design intent, not drift. +legitimately return a useful synthesis after one delegate fails, route around a +dead worker, or decide that another delegate supplied enough evidence. +Conversely, every delegate can exit zero while producing useless work. Mapping +worker exits to “the task is good” would manufacture false precision. + +Loopy can reliably observe whether the harness invocation itself completed. +It records that mechanical fact and leaves the semantic decision to the role +that has durable knowledge of the goal and plan. This behavior dates to the +first `harness_runner.py` implementation (`a4cca5e`, 2026-04-19); the amendment +changes who owns semantic completion, not mechanical success. ### Consequences -- **The evaluation layer is the real arbiter of completion**, not the harness return - value. Everything downstream depends on that layer being run and being honest - (see Decision 2, and the "known limitation" below). -- **The scheduler keys cadence off mechanical success.** `run_every`, `must_follow`, - and `run_after_successes` all read `HistoryEntry.success`. A run where a worker - actually failed but the harness returned normally still advances these counters. - This is an **accepted, bounded inaccuracy**: `control.json`/`goal_check.json` remain - the true gates, so the worst case is a slightly-off cadence, not a false "goal met." -- **Crash recovery treats a locally-written result as authoritative.** The - `pending_finished_request.json` / `result.json` recovery path trusts the recorded - iteration result; it does not re-derive success from worker artifacts. - -### Known limitation (documented, not a call to revert) - -Because acceptance for an entire iteration ultimately rests on the evaluation layer, -and that layer is LLM-as-judge by design (Decision 2), a whole iteration's -"success" can rest on a single model judgment with no deterministic backstop. For -low-stakes goals this is an acceptable, conscious trade. For high-stakes work it -should be **backstopped**, not reverted — see the note in Decision 2 about -repo-owned deterministic checks and the active -[`P1.2` proposal](../proposals/improvement-proposals.md#p12--target-owned-deterministic-evaluation-backstop). - -### Alternatives considered and rejected - -- *Derive iteration success from worker exit codes / `result.agents`.* Rejected: - unreliable for the reasons above; produces false negatives (good outcome, failed - worker) and false positives (all-green, useless outcome). -- *Make `goal_check.json` directly stop the loop.* Rejected: conflates evidence with - control. Keeping `control.json` as the sole stop switch means the accountable - current workflow must make an explicit, auditable stop decision. A human gate - is not part of this path (D5). - -### When to revisit - -If cadence inaccuracy causes a concrete problem, tune the workflow set's -mechanical eval frequency and the evidence rendered into eval prompts. Do not -make semantic readiness or an accepted eval determine workflow eligibility: -D8 forbids semantic scheduling gates, and D11 keeps readiness as prompt context. -Accepted eval affects terminal control, not which assignment may run next. - ---- - -## Decision 2 — Evaluation is LLM-as-judge; agents do not author deterministic checks +- Scheduler cadence (`run_every`, `must_follow`, and + `run_after_successes`) continues to use mechanical history. +- A delegate's non-zero exit can coexist with a mechanically successful + attempt, because the coordinator may still have integrated useful work. +- A missing, non-passing, stale, or malformed advisory eval is recorded as + evidence/diagnostics. It does not retroactively flip `HistoryEntry.success`, + consume the generic harness-failure budget, or starve the orchestrator. +- Crash recovery continues to trust a matching locally persisted result; it + does not reconstruct success from raw worker streams. +- `control.json`, written by the exact current completion owner, is the sole + semantic stop request. Evidence artifacts do not stop the loop by + themselves. + +### Protocol integrity is different from semantic judgment + +The engine still rejects a stale or sibling control record, a producer that is +not the frozen completion owner, malformed paths, invalid topology, or false +evidence provenance. Those are claims about the durable protocol, not opinions +about work quality. + +For protocol v3, successful control contains exact current +session/workflow/attempt identity and a non-empty rationale. Evidence references +are optional and may be empty; asserted references are validated. Eval receipt +references are optional and may come from an earlier +attempt in the same session. If cited, their subject identity and hashes must +validate and their producer must be a runner role declared by the frozen +contract. The engine does not require a passing verdict or reinterpret the +orchestrator's weighting of conflicting evidence. + +Protocol-v2 sessions retain their frozen same-attempt eval requirements, +including a session explicitly created later from a custom v2 contract. Their +authority is never silently reinterpreted as v3. + +### Alternatives rejected + +**Derive success from delegate exit codes.** This creates both false negatives +(one delegate failed but the coordinator recovered) and false positives (all +delegates exited cleanly but the outcome is wrong). + +**Make an evidence artifact stop the loop automatically.** Evidence and +decision are different responsibilities. The persistent orchestrator must +integrate the evidence and leave a reasoned terminal disposition. + +**Let both orchestrator and evaluator write success.** Dual authority creates +races and ambiguity. Evaluation informs one owner; it does not become another +owner. + +## Decision 2 — When evaluation is used, stock checks are outcome-oriented LLM judgments ### Decision -In the packaged `inner_outer_eval` workflow set, the eval workflows create **only** -`harness_judge` (LLM-as-judge) checks that describe *desired outcomes*. Authoring -deterministic checks is explicitly forbidden in the stock template. - -Reference: -`src/loopy_loop/templates/inner_outer_eval/.loopy_loop/workflow_sets/inner_outer_eval/workflows/eval_reviewer/prompt.txt` -— "Only create harness_judge checks"; "Do not create deterministic checks. -Deterministic checks are forbidden." - -### Context / why - -This rule comes from direct experience, not theory. When agents were allowed to -**author** deterministic checks, they produced bad ones: brittle string-matching, -checks that tested the wrong thing, checks that passed for the wrong reason, and -checks an agent could trivially satisfy without doing the real work. In practice, -letting the implementer invent its own pass/fail criteria let it game itself. - -`harness_judge` on a described *outcome* removes that failure mode: the check states -what good looks like in natural language, and a judge evaluates against it. The -implementer cannot quietly redefine "done" into something it already produced. - -### Scope and boundary (important — read before applying to other repos) - -The thing that failed was **agent-authored** checks, not deterministic checks as a -category. Two very different things get conflated under "deterministic check": - -- **Agent invents a check** → the failure mode above. Correctly forbidden. -- **Run a check the repo already owns** → e.g. `uv run pytest`, `import-linter`, - `alembic upgrade`, `make test`, evaluated on exit code. The agent did not invent - these; they are the project's own contract. Running them is deterministic but is - **not** the failure mode this rule targets. - -Therefore the "deterministic forbidden" rule is correct **for generic target repos -where the only deterministic checks would be agent-invented**. For a target repo that -already owns a trustworthy contract-test suite, the right configuration is *both*: -LLM-as-judge for the qualitative "did this achieve the outcome," **and** a -deterministic gate that shells out to the repo's own suite as a backstop under the -judge. That backstop does not reintroduce the agent-authoring problem, and it removes -the single-judgment point of failure noted in Decision 1. A workflow set targeting -such a repo should override the stock rule accordingly, in a dedicated child workflow -set rather than by loosening the stock template. - -### Consequences - -- **Evaluation is outcome-focused and resistant to self-gaming**, at the cost of the - usual LLM-as-judge properties: non-determinism, per-check inference cost, and the - judge as a point of trust. -- **The judge should not share failure modes with the implementer.** Prefer judging - with a different model family than the one that implemented the change. -- **A single judge pass is evidence, not a hard gate for high-stakes stops.** Keep - `control.json` as the stop switch (Decision 1); for high-stakes goals, require - repeated/independent judgments or a deterministic backstop before a terminal - `goal_met`. - -### Alternatives considered and rejected - -- *Let agents author deterministic checks (the prior state).* Rejected on evidence: - produced nonsensical, gameable checks. -- *Deterministic-first everywhere, judge as residual (the "obvious" best practice).* - Rejected **as a blanket rule** because in generic repos the only deterministic - checks available are the agent-authored ones that failed. It is the *right* rule - only where the deterministic checks are repo-owned (see boundary above). - -### When to revisit - -Revisit per target repo, not globally: when a target owns a trustworthy contract-test -suite, add the deterministic backstop (do not remove the judge). When judge cost or -flakiness becomes material, add repetition/consensus and cross-family judging rather -than abandoning the approach. - ---- - -## Summary - -| | Decision 1 | Decision 2 | -|---|---|---| -| **What** | Iteration success = harness completed, not work-is-good | Eval = LLM-as-judge on outcomes; agents don't author deterministic checks | -| **Why** | Worker exit codes are a noisy proxy for real success | Agent-authored deterministic checks were gameable nonsense | -| **True gate** | identity-bound `control.json` (stop) + eval receipt/`goal_check.json` (evidence) | The judge's verdict, recorded as evidence | -| **Shared principle** | Delegate the success decision to an explicit eval layer | Same | -| **Backstop for high-stakes** | Run eval on a suitably frequent mechanical cadence; keep semantic acceptance in control | Add repo-owned deterministic check under the judge | - -Both decisions are sound. Neither should be reverted. The one thing worth adding — -for high-stakes targets only — is a deterministic backstop built from the target -repo's **own** contract tests, which strengthens both decisions without undoing -either. That conditional follow-up is tracked as -[`P1.2`](../proposals/improvement-proposals.md#p12--target-owned-deterministic-evaluation-backstop), -not as part of this implemented design. +The generic packaged eval workflow authors only outcome-oriented +`harness_judge` checks. It does not invent deterministic checks. This boundary +applies when an orchestrator chooses to use evaluation; it does not require an +eval run and does not grant an evaluator terminal authority. + +Reference: the `eval_reviewer` prompt under +`src/loopy_loop/templates/inner_outer_eval/.loopy_loop/workflow_sets/inner_outer_eval/workflows/`. + +### Why agents do not invent deterministic checks + +This rule comes from observed failure. Agent-authored deterministic checks +became brittle string matches, targeted the wrong behavior, passed for the +wrong reason, or were easy for the implementer to game. Allowing the current +implementer to invent its own machine-enforced pass criteria let it redefine +“done” around what it had already produced. + +An outcome-oriented judge instead states what good behavior looks like in +natural language and evaluates the evidence against that description. It has +non-determinism and model-trust costs, but avoids pretending that a weak +agent-invented script is an objective contract. + +### Important boundary: repository-owned checks are legitimate evidence + +The rejected category is **agent-invented pass/fail logic**, not deterministic +testing in general. Running an existing repository-owned command such as +`pytest`, `import-linter`, `alembic upgrade`, or a prepared implementation-eval +suite does not have the same self-grading failure mode. The project established +those criteria independently of the current implementation attempt. + +A target-specific workflow may therefore combine: + +- qualitative `harness_judge` observations; +- repository-owned deterministic tests; and +- prepared project-level evals. + +All are evidence for the orchestrator. None becomes a universal Loopy engine +gate merely because it is deterministic or expensive. + +### Eval-check authoring needs stronger independent review + +Eval definitions are high-leverage artifacts: missing, overlapping, ambiguous, +or gameable checks distort every later observation. For a non-trivial check set, +the eval-reviewer coordinator should normally: + +1. run independent goal-coverage and failure-mode analyses in parallel across + different enabled harness families when available; +2. assign one accountable author/integrator to draft the canonical checks; +3. ask a different family to review the stable draft, with parallel reviewers + where useful; +4. explicitly test coverage gaps, false positives/negatives, implementation + coupling, gameability, wording ambiguity, and evidence discoverability; and +5. synthesize disagreements into one coherent outcome-oriented check set. + +Use the session's `frontier` tier for subtle or high-stakes eval-policy/check +work when the capability roster offers it and the confidence gain justifies the +cost. Concrete model IDs and enabled families come from the frozen roster, not +from hard-coded stock prompt text. + +This is strong prompt guidance, not a required number of agents, a fixed vendor +graph, an all-family quorum, or a completion receipt. If only one family is +usable or the check is trivial, the coordinator proceeds autonomously with an +appropriate smaller review shape. + +### Running and interpreting evals + +- Prefer a judge whose harness/model family differs from the primary + implementer and check author where practical. +- The eval runner publishes a provenance-rich observation. It never writes the + session's successful terminal control in the amended contract. +- A scheduled eval may run before, during, or near the end of work. The + orchestrator can also invoke one directly or avoid duplicate work when the + scheduler view shows an eval role is about to run. +- A passing judge result is evidence, not proof. A failing result is also + evidence, not an unconditional veto. The orchestrator repairs, reruns, + supersedes, explains, or weighs it against other facts. +- When an eval receipt is produced or cited, exact subject and provenance + validation remains mandatory even though semantic deference is not. + +### Alternatives rejected + +**Let agents invent deterministic checks.** Rejected from experience: the +checks were brittle, wrong-target, and gameable. + +**Run deterministic-first everywhere.** This is appropriate when the target +already owns a trustworthy suite, but not as a blanket rule for generic repos +where the current agent would have to invent the gate. + +**Require an eval before every completion.** The persistent orchestrator can +have stronger direct evidence, a target may already own better checks, and a +scheduled evaluator can fail for reasons unrelated to work quality. + +**Restore a hard-coded author/reviewer model chain.** Cross-family review is +valuable, but concrete providers and models change. The enabled harness/tier +roster supplies current choices and the coordinator adapts. + +## Compact summary + +| Question | Binding answer | +| --- | --- | +| What does `IterationResult.success` mean? | The harness invocation returned normally | +| Who decides the session goal is complete? | The workflow contract's persistent orchestrator (`outer` or `planner` in stock sets) | +| Is evaluation required? | No; it is optional evidence unless a target's own goal asks for a particular eval | +| What stops the session? | Exact-current-attempt, identity-bound `control.json` from the completion owner | +| What happens to bad eval output? | It becomes visible diagnostic evidence, not mechanical failure or a universal gate | +| What kind of stock checks may agents author? | Outcome-oriented `harness_judge` checks, not invented deterministic gates | +| How should non-trivial eval checks be designed? | Prefer parallel independent cross-family analysis, one integrator, and different-family review | +| Are review diversity and model tiers enforced? | No; the roster informs prompt-guided, audited coordinator judgment | diff --git a/design/proposals/state-driven-eval-fast-paths.md b/design/proposals/state-driven-eval-fast-paths.md new file mode 100644 index 0000000..4ebd5c6 --- /dev/null +++ b/design/proposals/state-driven-eval-fast-paths.md @@ -0,0 +1,458 @@ +# Proposal: State-Driven Evaluation Fast Paths + +**Status:** Superseded before implementation — historical, non-binding + +**Date:** 2026-07-17 + +**Applies to:** workflow sets with separate task-acceptance, eval-author, and +eval-runner roles, including `inner_outer_eval` and `pm_planner_dispatcher` + +> **Do not implement this proposal as written.** It assumes the earlier D11 +> contract in which `eval_runner` alone owned completion and a same-attempt +> passing eval was mandatory. D11 was replaced on 2026-07-17: the durable +> orchestrator now owns completion and evals are optional evidence. The useful +> efficiency need is handled by the accepted workflow roster and conditional +> scheduler view in +> [`orchestrator-owned-completion-and-cross-harness-review.md`](../designs/orchestrator-owned-completion-and-cross-harness-review.md). + +The remainder records the proposed efficiency improvement against the former +protocol-v2/D11 baseline and is intentionally preserved as historical analysis. +Its present-tense descriptions of eval-owned completion are no longer current. +The binding decisions are +[`D3`, `D4`, `D8`, and `D11`](../decisions.md), and the replacement design above +defines the implementation direction. + +## Summary + +The stock delivery loop evaluates after every three mechanically successful +`inner` assignments. That cadence is simple during active implementation, but +it becomes wasteful when the delivery is already accepted and only the +evaluation-owned receipt or terminal decision remains. + +The proposed fast path lets the workflow contract's task-acceptance role ask +for evaluation now. The request is not proof that the work is good. The engine +checks only the request's identity and provenance, then runs the same +eval-author and eval-runner roles earlier than the fixed cadence would. A +mistaken request therefore causes an early LLM evaluation, whose failing +verdict routes the loop back to repair; it can never manufacture acceptance. + +Normal cadence remains the fallback when no valid request exists. No workflow +is prevented from running until a semantic condition is proven. + +## The concrete problem + +In a July 2026 double-loop run, a child delivery had already: + +1. landed and merged its one scoped pull request; +2. been accepted by the `outer` role; +3. published a compact eval-readiness record; and +4. passed all five LLM-as-judge checks in the raw eval report. + +The eval runner then failed at the harness/protocol layer while reading that +large report, before it could publish the same-session eval receipt and +`goal_check.json`. The repository work was not lost, but the fixed cadence +required more `outer → inner` cycles before evaluation became eligible again. +Those assignments repeatedly verified that the accepted work had not changed +and that no implementation leaf remained. + +This is expensive and confusing. Global iteration 19 looked like nineteen +implementation attempts to an operator, while it was actually a mixture of +outer, inner, and eval workflows; the only scoped pull request had already +merged. + +## Current behavior + +`WorkflowConfig` in `src/loopy_loop/config.py` declares `run_every`, +`must_follow`, `priority`, and `run_after_successes`. +`choose_next_workflow()` in `src/loopy_loop/scheduler.py` receives only those +definitions, `HistoryEntry` records, and the completed iteration count: + +- `_run_after_successes_satisfied()` counts mechanically successful target + workflows since the candidate last succeeded; +- `_last_successful_workflow_id()` supplies the `must_follow` relationship; + and +- `_failed_workflow_retry()` retries the latest mechanically failed workflow + only when no normally eligible workflow exists. + +The packaged `inner_outer_eval` reviewer has: + +```yaml +run_after_successes: + workflow_id: inner + every: 3 +``` + +The eval runner then follows the reviewer through +`must_follow: eval_reviewer`. This produces the mechanical sequence +`(outer → inner) × 3 → eval_reviewer → eval_runner`, as asserted in +`src/tests/test_template_contracts.py`. + +Eval readiness is deliberately not scheduler input today. +`_semantic_prompt_context()` in `src/loopy_loop/worker.py` exposes the latest +parseable readiness JSON to agents as prompt context. The binding +[`recursive-loop-layer-contract.md`](../designs/recursive-loop-layer-contract.md) +and [success/evaluation design](../designs/success-semantics-and-evaluation.md) +explicitly say that readiness does not affect workflow eligibility. + +## Goals + +- Reach layer-owned evaluation promptly when the accountable acceptance role + believes implementation is ready. +- Recover an eval chain without paying for implementation roles that have no + remaining implementation work. +- Keep the engine ignorant of whether the work is semantically ready or good. +- Preserve the same eval author, LLM-as-judge checks, eval runner, receipt, and + terminal-control ownership. +- Make the optimization durable and exactly recoverable across process or + laptop restarts. +- Preserve the normal fixed cadence when no valid fast-path request exists. + +## Non-goals + +- Do not infer readiness from git state, prose, pull-request status, test output, + or the presence of an accepted-work ledger. +- Do not let `outer`, `inner`, a planner, or a spawned subagent publish + `goal_met`. +- Do not calculate or cache a semantic verdict in the scheduler. +- Do not skip the LLM judge merely because the repository subject appears + unchanged. +- Do not merge the reviewer and runner roles in the first version. +- Do not add another loopy worker, parallel child session, human gate, approval + state, or preventive agent fence. + +## Binding invariants + +Any implementation must preserve these decisions: + +- **D2:** one loopy worker and one deepest active durable assignment. +- **D3:** `IterationResult.success` remains mechanical harness success. An eval + wake request does not change it. +- **D4:** the stock semantic verdict remains LLM-as-judge. Request validation is + provenance validation, not a deterministic quality check. +- **D5:** no normal human checkpoint is introduced. +- **D8:** the request may accelerate detection but never prevent a workflow + until readiness is proven. A false-positive request is handled by evaluation + and repair. +- **D9/D10:** coordinator model policy and dynamic harness delegation remain + unchanged. +- **D11:** only the declared eval runner may close its own layer, using a valid + same-attempt receipt and control signal. +- **D12:** compact scheduling facts belong in durable correctness state; verbose + validation and execution detail belongs in traces. + +## Proposed contract + +### 1. Readiness v2 may carry an eval wake request + +The workflow contract already names `task_acceptance_role`, +`eval.author_role`, and `eval.runner_role`. A fresh v2 readiness record may +ask the engine to start that layer's declared eval chain: + +```json +{ + "schema_version": 2, + "readiness_id": "readiness-...", + "session_id": "current-session-id", + "goal_hash": "sha256:...", + "producer": { + "workflow_id": "outer", + "iteration": 12, + "attempt_id": "current-attempt-id", + "harness_run_id": "current-harness-run-id" + }, + "request_eval_now": true, + "accepted_evidence_refs": ["session:/..."], + "rationale": "The scoped delivery is accepted; layer evaluation owns the remaining decision.", + "created_at": "2026-07-17T12:00:00Z" +} +``` + +The assignment envelope supplies the exact absolute readiness directory plus +session/workflow/iteration/attempt identity. Team Harness's embedded caller +context supplies the current `harness_run_id`. The acceptance role chooses +whether to write the request. The engine does not parse `rationale` or inspect +the evidence references to decide whether the request is wise. + +This is layer-local. A child acceptance role can wake only its child eval chain; +a child request cannot wake or close its parent. A parent planner can make a +separate request only for the parent layer. + +### 2. Capture provenance while the producer attempt is current + +`CoordinatorService._record_finished_task()` should examine a readiness request +only while its claimed producer is still the exact `CurrentTask`. It accepts +the optimization request only when: + +- session id and goal hash match the current durable session; +- producer workflow, iteration, attempt, and harness run match the finishing + task; +- the producer role equals the frozen workflow contract's + `task_acceptance_role`; +- the record is valid JSON with the supported schema; and +- the producer attempt reaches final mechanical/protocol success. + +The engine must not rescan readiness later and trust a record merely because it +names an old successful attempt. Under D8, a later agent can physically write +any session file; history proves that the old attempt existed, not that the +record existed when that attempt finished. + +Malformed, mismatched, or multiply-claimed requests disable only the +optimization. They produce an explicit diagnostic and remain available for +inspection, but they do not change iteration success or stop the loop. Repair +means preserving that rejected artifact and publishing a fresh request from a +later accountable attempt; an immutable rejected request is never rewritten or +retroactively accepted. + +### 3. Persist an engine-owned pending eval wake + +After validation, the coordinator stores a compact `pending_eval_wake` in +`LoopState`, containing: + +- readiness id, logical reference, and content hash; +- producer workflow, iteration, attempt, and harness run; +- target eval-author and eval-runner roles from the frozen contract; +- current chain step; and +- retry/dispatch facts needed for crash recovery. + +The agent-authored readiness record remains compact session evidence. The +engine-owned state is the scheduling source of truth. Validation details, +duplicate candidates, and raw execution output stay in the attempt trace. + +### 4. A pending wake accelerates; it does not gate + +`CoordinatorService._advance()` keeps its existing precedence: + +1. recover or serve a suspended child; +2. apply stop and terminal control; +3. dispatch a requested child after a successful parent assignment; +4. handle a pending eval wake; and +5. otherwise call normal `choose_next_workflow()`. + +For the first chain step, the coordinator selects the declared eval-author role +without waiting for its `run_after_successes` bucket. It still honors: + +- the frozen workflow set and contract; +- `enabled` and `not_before_iteration`; +- the single-worker/current-task invariant; and +- all session stop and child-suspension precedence. + +No request means step 5 runs exactly as it does today. An invalid request also +falls back to normal scheduling. Therefore readiness is never a prerequisite +for evaluation or implementation. + +### 5. Keep the wake through the complete eval chain + +After the eval author succeeds, the pending wake advances to the declared +eval-runner step. The existing `must_follow` relationship remains the normal +ordering proof. + +The wake is consumed only when the eval runner completes one of these +same-attempt protocol outcomes: + +- a passing receipt and `goal_check.json`, together with applied same-attempt + `goal_met` control, close the session through the existing D11 path; +- a valid non-passing verdict consumes the wake and normal scheduling resumes + with that evidence available for repair; and +- a mechanical or protocol failure retains the wake so the eval-owned role can + retry rather than forcing no-op implementation cycles. + +Current control handling also permits a runner to publish a valid passing +receipt/projection while leaving `control.json` in its non-terminal +`state: running` placeholder. That does **not** consume the wake: the runner +step remains pending because the layer is neither terminal nor validly +non-passing. Its next attempt must publish a new same-attempt receipt, +projection, and control; D11 forbids reusing the prior attempt's receipt as if a +later attempt had produced it. + +An immediate retry may bypass only the failed workflow's ordinary cadence +delay. Existing `goal_check_consecutive_failures_cap` handling and the +per-workflow consecutive-failure limit remain the bounded escapes from a +permanently broken eval workflow. This retry rule must be tested against the +case where a restart occurs after reviewer success but before runner dispatch. + +The engine validates receipt/control structure and provenance exactly as it +does today. It still does not reinterpret the judge's semantic reasons. + +## Why this is not a semantic scheduler gate + +A gate says, “workflow X may not run until semantic fact Y is proven.” This +proposal says, “the accountable agent requested the already-configured +evaluation workflow now.” + +The difference matters: + +- absence of readiness does not block any existing path; +- invalid readiness falls back to mechanical cadence; +- false-positive readiness only spends an early eval attempt; +- a non-passing eval returns the system to autonomous repair; and +- only existing eval receipt/control validation can terminate the session. + +The engine is transporting an agent decision across attempts, not making that +decision itself. + +## Required decision amendment before implementation + +D11 and its companion designs currently state that readiness is prompt context +and never scheduler input. Implementing this proposal without changing those +texts would silently contradict a binding decision. + +If this proposal is accepted, amend D11 narrowly: + +> Readiness may carry an identity-bound, additive request that accelerates the +> current layer's declared eval-author → eval-runner chain. It never proves +> semantic readiness, gates another workflow, or authorizes terminal control. +> Fixed mechanical cadence remains the fallback. + +D8 need not change if the implementation remains additive as specified above. +If an implementation instead uses readiness to exclude, delay, or approve other +workflows, that would be a semantic scheduling veto and would require a more +fundamental D8 redesign. + +The binding recursive-loop and success/evaluation designs, README, session +layout documentation, stock contracts, and prompts must change in the same +implementation PR as the decision amendment. + +## State, traces, and observability + +The fast path should emit compact events such as: + +- `eval_wake_accepted`; +- `eval_wake_rejected` with a repairable reason; +- `eval_wake_dispatched`; +- `eval_wake_step_completed`; and +- `eval_wake_consumed`. + +`loopy status` should show the active wake and current chain step. As a related +observability improvement, status should distinguish global iteration from +per-role run count, for example: + +```text +current_task: inner (global iteration 19, inner run 6) +pending_eval_wake: none +``` + +This does not change scheduling, but it prevents operators from mistaking +iteration count for the number of implementation attempts. + +### Related but separate: parent-plan discoverability + +The parent session intentionally keeps the complete backlog in the versioned +roadmap and phase files rather than copying every work package into session +state. That avoids two competing plan authorities. However, agent-maintained +`project_state/current_state.md` and `child_sessions.md` can remain stale while +the parent is suspended behind a live child; only engine-owned `children.json` +is current during that interval. A separate state/observability proposal should +define one obvious engine-maintained overview that links to the canonical +roadmap and reports the selected item, active child, next-candidate/blocked +summary, and accepted/finished references without duplicating full work-package +descriptions. This proposal does not make that projection part of eval +scheduling. + +## Failure and recovery cases + +- **Crash after readiness capture:** `pending_eval_wake` survives in + `LoopState`; restart dispatches the same logical next step. +- **Crash with an eval step live:** normal `CurrentTask` identity/liveness + recovery applies. The wake is not consumed by abandonment alone. +- **Duplicate finish request:** readiness id/hash and attempt identity make + capture idempotent. +- **Stale or forged readiness:** provenance validation rejects the optimization; + normal cadence continues. +- **False semantic readiness:** the independent LLM judge fails and normal + repair continues. +- **Eval author failure:** retain the author step and use existing failure + accounting. +- **Eval runner transport/protocol failure:** retain the runner step; retry + without manufacturing a receipt. +- **Passing receipt/projection without applied terminal control:** retain the + runner step and require a fresh receipt/projection/control from its next + attempt. +- **Valid non-passing eval:** consume the wake and expose the receipt/reasons to + subsequent roles. +- **New child dispatch:** child-suspension precedence remains higher than the + parent's pending wake; each layer keeps its own state. + +## Rollout and tests + +1. Decide whether to accept the narrow D11 amendment. +2. Add a typed readiness-v2 model and `pending_eval_wake` state model. +3. Capture requests in `_record_finished_task()` and add the wake transition + before normal scheduling in `_advance()`. +4. Update both packaged workflow sets, assignment paths, prompts, status, events, + README, and binding designs. +5. Test: + - exact producer/session/goal matching; + - wrong-role, stale, late-written, duplicate, and malformed requests; + - false-positive requests leading to non-passing eval and repair; + - unchanged fallback cadence without a request; + - author and runner failure/retry; + - passing receipt/projection with running control retaining the runner step; + - crash recovery before and during both eval steps; + - child/parent isolation at two and three durable depths; + - one live worker/current task across the whole tree; and + - no `goal_met` without the existing valid same-attempt eval receipt and + control. + +All new or touched functions must use named arguments at call sites where the +repository convention requires them and must have useful docstrings. + +## Alternatives + +### Run evaluation after every inner assignment + +Changing `run_after_successes.every` from three to one is fully mechanical and +fits current D11. It removes most waiting but can multiply expensive judge runs +during normal implementation. A target repository can use it today when +latency matters more than eval cost. + +### Retry a mechanically failed eval before normal workflows + +A configurable scheduler preference for the latest mechanically failed eval +would solve the observed report-transport incident without semantic state. It +does not solve the broader case where accepted work should be evaluated before +the next fixed cadence bucket. It can complement the wake mechanism. + +### Skip full evaluation for an unchanged subject + +A layer-local subject fingerprint could avoid repeat judge calls when the goal, +checks, judge binding, git state, accepted evidence, and user-input cursor are +unchanged. This requires a new honest non-terminal “not evaluated/unchanged” +disposition; it must never reuse an old passing receipt or authorize +`goal_met`. It is a promising separate proposal, not part of the first wake +implementation. + +### Let the engine infer readiness + +Rejected. Parsing plans, PR status, tests, or accepted ledgers would make the +coordinator a semantic judge and create exactly the eligibility coupling D3, +D8, and D11 avoid. + +### Let outer invoke eval-banana and close the layer + +Rejected. It collapses task acceptance, check authorship, execution, and +terminal ownership into one role, weakening D4 and contradicting D11. + +## Open questions + +- Should runner retry remain pending until the existing failure cap, or should + the workflow set declare a smaller immediate-retry count before returning to + normal cadence? +- Should a newer valid readiness request supersede an older pending wake, or be + recorded as a duplicate while the single current chain completes? +- Should readiness v1 remain prompt-only forever, with only v2 capable of + requesting a wake? The safer migration answer is yes. +- Should per-role run counts be added in the same implementation or a small + separate observability change? + +## Primary implementation anchors + +- `src/loopy_loop/config.py` — workflow and contract configuration +- `src/loopy_loop/models.py` — typed readiness and pending wake state +- `src/loopy_loop/coordinator_app.py` — + `_record_finished_task()`, `_advance()`, goal-check and receipt validation +- `src/loopy_loop/scheduler.py` — bounded cadence bypass for a validated wake +- `src/loopy_loop/assignments.py` — absolute layer-local paths in assignments +- `src/loopy_loop/worker.py` — semantic prompt context +- `src/tests/test_scheduler.py` and + `src/tests/test_template_contracts.py` — cadence behavior +- recursive session/recovery tests — durable layer isolation and crash windows diff --git a/docs/http-contract.md b/docs/http-contract.md index c12f198..d7254e7 100644 --- a/docs/http-contract.md +++ b/docs/http-contract.md @@ -1,38 +1,34 @@ # HTTP Contract -loopy-loop exposes two coordinator endpoints: `POST /register` assigns the one -available worker to the deepest runnable session, and `POST /finished` records -one mechanical harness result before returning the next assignment. Recursive -parent/child handoffs are files, not additional HTTP endpoints. - -The wire models live in `src/loopy_loop/models.py`; endpoint validation and -state transitions live in `create_coordinator_app()` and `CoordinatorService` -in `src/loopy_loop/coordinator_app.py`. - -## Protocol generations - -Every fresh 0.7 coordinator run creates a session tree with state schema v2. -An existing state schema v1 tree can still be resumed with its historical wire -behavior. Protocol generation is selected from durable session state, not from -a client preference: - -- `/register` always requires a `worker` process identity. This requirement - predates v2; a missing identity returns HTTP 400. -- A v1 session accepts a registration with just that identity. The new - registration fields are optional in the Pydantic model so old persisted - sessions remain usable. -- A v2 session requires `worker_protocol_version >= 2` and every capability - named below. Missing support returns HTTP 426; the coordinator does not - silently downgrade a fresh session. -- The bundled v2 worker also sends the absolute checkout path and the stable - checkout identity from `.loopy_loop/repository.json`. The coordinator checks - either value when supplied. This catches a worker connected from the wrong - checkout, but is not a general filesystem sandbox. - -The v2 capability set is deliberately feature-named rather than inferred from -package versions: +loopy-loop has two coordinator endpoints: + +- `POST /register` gives the single worker the deepest runnable assignment; +- `POST /finished` records one mechanical harness result and returns the next + assignment or stop response. + +Recursive parent/child handoffs, semantic progress, evaluation evidence, and +terminal decisions are files, not additional HTTP endpoints. Wire models live +in `src/loopy_loop/models.py`; endpoint checks and transitions live in +`create_coordinator_app()` and `CoordinatorService` in +`src/loopy_loop/coordinator_app.py`. + +## Version and capability negotiation + +The agent-facing protocol is frozen by each session's +`workflow_contract.json` and engine-owned contract copy. It is not selected by +the worker. Fresh stock `inner_outer_eval` and `pm_planner_dispatcher` sessions +declare protocol v3. The persisted coordinator-state schema remains v2; those +are separate version axes. + +`POST /register` always requires a process identity. For a protocol-v3 tree it +also requires: + +- `worker_protocol_version >= 3`; +- the absolute checkout path and stable checkout-local repository ID; and +- every feature capability below. ```text +# carried forward from protocol v2 assignment_v1 frozen_workflow_v1 trace_manifest_v1 @@ -40,39 +36,57 @@ caller_run_record_v1 coordinator_input_v1 spawn_assignment_v1 nested_caller_context_v1 + +# added for protocol v3 +assignment_v2 +harness_capability_roster_v1 +orchestrator_control_v3 +scheduler_view_v1 +semantic_handoff_v1 +capability_roster_context_v1 ``` -The first three are loopy worker capabilities. The last four are advertised by -team-harness through `get_capabilities()` and forwarded by -`worker._worker_capabilities()`. This lets the coordinator reject an -installation that can run agents but cannot satisfy the selected provenance -and trace contract. +The first group combines loopy worker features with team-harness caller/trace +features. The v3 additions mean that the worker understands the semantic path +map, tree-wide harness roster, scheduler view, orchestrator-owned control, and +rolling handoff. `capability_roster_context_v1` is advertised by team-harness: +it means the worker can forward the root roster's absolute path, SHA-256, and +compact summary through caller context so a built-in nested `type=harness` +coordinator inherits the same catalog. + +`nested_caller_context_v1` is narrower: it preserves root/current session, +depth, workflow role, parent loopy attempt, direct-agent assignment, nested +trace root, and parent harness-run identity. A nested harness remains a dynamic +delegate in the current durable assignment; it is not a child Loopy session. -`nested_caller_context_v1` means a built-in `type=harness` spawn inherits the -same root/current session, depth, workflow role, and loopy attempt; changes to -the direct agent's absolute assignment and nested trace root; and records the -parent harness-run ID. It is another harness coordinator inside the same -workflow assignment, not another durable loop layer. Generic subprocesses are -not inferred to have this lineage. +Missing version or capability support returns HTTP 426 before a v3 assignment +is dispatched. The coordinator never silently downgrades the session. A wrong +checkout path or repository identity returns HTTP 409. -## POST /register +## `POST /register` -The bundled 0.7 worker sends: +The bundled v3 worker sends: ```json { "worker": { "hostname": "buildbox", "pid": 4242, - "starttime": "lstart:Sun Jul 12 00:00:00 2026" + "starttime": "lstart:Fri Jul 17 10:00:00 2026" }, - "worker_protocol_version": 2, + "worker_protocol_version": 3, "capabilities": [ "assignment_v1", + "assignment_v2", "caller_run_record_v1", + "capability_roster_context_v1", "coordinator_input_v1", "frozen_workflow_v1", + "harness_capability_roster_v1", "nested_caller_context_v1", + "orchestrator_control_v3", + "scheduler_view_v1", + "semantic_handoff_v1", "spawn_assignment_v1", "trace_manifest_v1" ], @@ -81,68 +95,69 @@ The bundled 0.7 worker sends: } ``` -`worker.starttime` is team-harness's pid-reuse-resistant process token. It can -be null for an older identity record, in which case same-host liveness may be -unknown rather than verified. `repository_id` is a random, checkout-local -identity created once by `assignments.ensure_repository_identity()`; it is not -a git remote or commit identifier. +`worker.starttime` is team-harness's pid-reuse-resistant process token. It may +be null for an older identity record, in which case same-host liveness can be +unknown. `repository_id` is a random identity created once in +`.loopy_loop/repository.json`; it is not a Git remote or commit. ### Run response -A v2 run response contains the legacy scheduling fields plus protocol, -repository, immutable-snapshot, and assignment fields: +A v3 run response includes scheduling identity, the tree-frozen execution +snapshot, required capabilities, repository binding, and exact attempt +snapshot/assignment coordinates: ```json { "action": "run", "workflow_set": "inner_outer_eval", - "workflow_id": "inner", - "session_id": "20260715_143022_71393ee22450_ab12cd34", - "iteration": 3, + "workflow_id": "outer", + "session_id": "20260717_100000_71393ee22450_ab12cd34", + "iteration": 4, "attempt_id": "a1b2c3d4e5f6", "config_snapshot": { - "goal": "Ship a minimal working landing page", + "goal": "Deliver the scoped outcome", "goal_hash": "71393ee22450", "workflow_set": "inner_outer_eval", - "completion_criteria": ["Homepage renders without errors"], - "stop_criteria": ["A workflow publishes valid terminal control"], - "max_turns": 20, + "completion_criteria": ["The declared outcome is evidenced"], + "stop_criteria": ["A valid terminal control is accepted"], + "max_turns": 40, "goal_check_consecutive_failures_cap": 3, - "team_harness_provider": "openai_compat", - "team_harness_model": "gpt-5.5", - "team_harness_agents": ["codex"], - "team_harness_agent_models": {"codex": "gpt-5.5"}, - "team_harness_agent_reasoning_efforts": {"codex": "high"}, - "team_harness_max_retries": null, - "team_harness_retry_base_delay_s": null, - "team_harness_retry_max_delay_s": null, - "team_harness_api_base": "https://openrouter.ai/api/v1", - "team_harness_api_key_env": "OPENROUTER_API_KEY", - "team_harness_system_prompt_extension": "" + "team_harness_provider": "", + "team_harness_model": "", + "team_harness_agents": [""], + "team_harness_agent_models": {}, + "team_harness_agent_reasoning_efforts": {}, + "team_harness_system_prompt_extension": "" }, "stop_reason": null, - "coordinator_protocol_version": 2, + "coordinator_protocol_version": 3, "required_capabilities": [ "assignment_v1", + "assignment_v2", "caller_run_record_v1", + "capability_roster_context_v1", "coordinator_input_v1", "frozen_workflow_v1", + "harness_capability_roster_v1", "nested_caller_context_v1", + "orchestrator_control_v3", + "scheduler_view_v1", + "semantic_handoff_v1", "spawn_assignment_v1", "trace_manifest_v1" ], "repo_root": "/absolute/path/to/target-repo", "repository_id": "repo-c50d0d9a46c843ecaa493243baed524f", - "assignment_path": "/absolute/path/to/target-repo/.loopy_loop/sessions/session-id/iterations/0003_inner/workflow_snapshot/a1b2c3d4e5f6/assignment.json", + "assignment_path": "/absolute/path/to/target-repo/.loopy_loop/sessions//iterations/0004_outer/workflow_snapshot/a1b2c3d4e5f6/assignment.json", "assignment_sha256": "sha256:<64 hex characters>", "workflow_snapshot": { "schema_version": 1, - "session_id": "20260715_143022_71393ee22450_ab12cd34", + "session_id": "20260717_100000_71393ee22450_ab12cd34", "workflow_set": "inner_outer_eval", - "workflow_id": "inner", - "iteration": 3, + "workflow_id": "outer", + "iteration": 4, "attempt_id": "a1b2c3d4e5f6", - "snapshot_root": "/absolute/path/to/target-repo/.loopy_loop/sessions/session-id/iterations/0003_inner/workflow_snapshot/a1b2c3d4e5f6", + "snapshot_root": "/absolute/path/to/workflow_snapshot/a1b2c3d4e5f6", "workflow_config_path": "/absolute/path/to/workflow_snapshot/config.yaml", "workflow_prompt_path": "/absolute/path/to/workflow_snapshot/prompt.txt", "workflow_contract_path": "/absolute/path/to/workflow_snapshot/workflow_contract.yaml", @@ -155,32 +170,40 @@ repository, immutable-snapshot, and assignment fields: } ``` -`config_snapshot.goal` is resolved text, not the configured goal-file path. -The snapshot freezes execution settings for the whole recursive tree; a child -changes its scoped goal, criteria, and workflow set but does not silently pick -up later model/config edits. - -`workflow_snapshot` identifies the scheduler-selected workflow config, prompt, -workflow-set contract, and root execution config. The coordinator materializes -these files and hashes before dispatch, creates `assignment.json`, and freezes -its SHA-256 in the task response. The worker verifies their identity, absolute -location, manifest, hashes, and independently reconstructed assignment at the -returned absolute `assignment_path`. The assignment binds one checkout, -session layer, workflow role, iteration, and attempt and gives the harness -coordinator absolute paths for that layer. See -`assignments.materialize_workflow_snapshot()`, -`assignments.verify_workflow_snapshot()`, and -`assignments.build_attempt_assignment()`. +The actual `config_snapshot` also carries the remaining frozen retry/provider +fields defined by `RootConfigSnapshot`; they are omitted from this example for +readability. The snapshot freezes execution settings for the entire recursive +tree. A child changes its scoped goal, criteria, and selected workflow set but +does not silently pick up later configuration edits. + +Before returning the response, the coordinator: -A v1 run response uses the same additive response model but has -`coordinator_protocol_version: 1`, an empty `required_capabilities`, no -workflow snapshot or assignment path, and follows the legacy workflow/output -path. Fields that do not apply are null. +1. freezes the selected workflow config, prompt, workflow contract, and root + execution snapshot; +2. writes a conditional `scheduler_view.json` for this attempt; +3. builds schema-v2 `assignment.json` with actor/objective identity, the full + absolute semantic path map, compact workflow/scheduler/capability context, + and provenance hashes; and +4. freezes the assignment SHA-256 in the current task and response. + +The worker verifies the immutable files, locations, manifest, hashes, and an +independently reconstructed assignment before invoking team-harness. See +`assignments.materialize_workflow_snapshot()`, +`assignments.build_attempt_assignment()`, and +`assignments.verify_workflow_snapshot()`. + +The assignment paths include the canonical layer goal/contract/inputs, +plan/tasks/current state/decisions/finished ledger/eval state/handoff, engine +state, terminal outcome, workflow contract/roster, scheduler view, root +capability roster, user inputs, recursive request/outcome/acceptance records, +Git and delivery receipts, session control, attempt root, and trace root. +Parent-only origin paths are explicit nulls at the root. See +[`session-layout.md`](./session-layout.md#attempt-snapshot-and-absolute-assignment-paths) +for the stable key list. ### Stop response -When the active tree is terminal, `/register` returns the same model with no -assignment: +When the active tree is terminal, the same response model has no assignment: ```json { @@ -197,53 +220,50 @@ assignment: "repo_root": null, "repository_id": null, "assignment_path": null, + "assignment_sha256": null, "workflow_snapshot": null } ``` ### Registration, liveness, and recovery -If a prior `current_task` is still durable, registration follows these rules: - -1. If its recorded same-host worker is verifiably alive with the same pid and - starttime, the coordinator returns HTTP 409 and does not duplicate work. -2. Otherwise it first looks for a matching - `pending_finished_request.json` or `result.json` in the iteration directory. - Recoverable completion is recorded before scheduling anything new. -3. With no recoverable result, it applies the configured drain/reap policy to - team-harness processes discoverable for that attempt. Replacement is - refused with HTTP 409 while ownership or a remaining process cannot be - resolved safely. A processed recovery is described by `salvage.json`. -4. Once the interrupted task is settled, it is recorded as abandoned and the - normal stop/child/workflow scheduler advances. - -Recovery runs its potentially long process handling outside the state lock, -then revalidates and commits under the lock. The bundled worker therefore uses -an unbounded read timeout only for `/register`. Process liveness/reaping is -same-host; a remote identity cannot be assumed dead. - -On `--resume`, the coordinator follows durable `active_child_session_id` -pointers to the deepest session. Terminal descendants are finalized and the -tree is unwound iteratively. The scheduler never dispatches a parent and its -active child at the same time. - -Relevant error statuses are: - -- HTTP 400: missing worker identity. -- HTTP 409: a live owner, stale different owner, wrong checkout/identity, or - unresolved recovery makes dispatch unsafe. -- HTTP 426: a v2 tree requires a newer worker protocol or capability. +If `state.json` still contains a current task, registration proceeds as +follows: + +1. A verifiably live same-host owner with matching PID/starttime receives HTTP + 409; work is not duplicated. +2. Otherwise, the coordinator looks for that attempt's matching + `pending_finished_request.json` or `result.json` and records a recoverable + completion before scheduling new work. +3. Without a result, it applies the configured drain/reap policy to tracked + team-harness processes. Replacement remains HTTP 409 while ownership or a + possibly live process is unresolved. A completed recovery is described by + `salvage.json`. +4. After safe abandonment, normal terminal/child/workflow scheduling resumes. + +Potentially long process handling occurs outside the state lock, then the +coordinator revalidates under the lock. On resume it follows durable +`active_child_session_id` pointers to the deepest session and iteratively +unwinds terminal descendants. A parent and its active child are never +dispatched concurrently. + +Relevant statuses are: + +- HTTP 400: required worker identity is absent; +- HTTP 409: live/stale ownership, wrong checkout, or unresolved recovery makes + dispatch unsafe; +- HTTP 426: protocol version or a required capability is missing; and - HTTP 503: the coordinator state lock could not be acquired in time. -## POST /finished +## `POST /finished` -The bundled v2 worker posts: +The bundled worker posts one exact completion envelope: ```json { - "workflow_id": "inner", - "session_id": "20260715_143022_71393ee22450_ab12cd34", - "iteration": 3, + "workflow_id": "outer", + "session_id": "20260717_100000_71393ee22450_ab12cd34", + "iteration": 4, "attempt_id": "a1b2c3d4e5f6", "success": true, "text": "done", @@ -259,148 +279,111 @@ The bundled v2 worker posts: "worker": { "hostname": "buildbox", "pid": 4242, - "starttime": "lstart:Sun Jul 12 00:00:00 2026" + "starttime": "lstart:Fri Jul 17 10:00:00 2026" }, "repository_id": "repo-c50d0d9a46c843ecaa493243baed524f", "assignment_sha256": "sha256:<64 hex characters>", "harness_run_id": "run-123", - "trace_manifest_path": "/absolute/path/to/trace_manifest.json" + "trace_manifest_path": "/absolute/path/to/trace_manifest.json", + "trace_incomplete": false, + "trace_error": null } ``` -The response is another `TaskResponse`: it either dispatches the next task or -stops. Posting `/finished` is therefore both acknowledgement and the next -scheduling poll. - -For a task dispatched under the complete v2 worker handshake, completion must -echo the exact worker owner, repository identity, and SHA-256 of the immutable -`assignment.json`. Every modern task also has a unique `attempt_id`; a late -completion from a superseded attempt cannot complete a new retry with the same -session/workflow/iteration coordinates. - -For a matching active task, the coordinator records: - -- `success`, result/error, and `failure_kind` in history; -- optional coordinator-model usage and worker-measured duration in the durable - ledger; -- the team-harness run ID, logical `trace_manifest_ref`, accepted assignment - hash, and SHA-256 hashes of the exact `/finished` request and - exact returned response. Only legacy history retains the worker's absolute - `trace_manifest_path`; a crash-abandonment entry records the assignment hash - but has no finished request/response hashes; -- any structurally valid eval/control transition produced by the workflow. - -The coordinator created the canonical active trace when it dispatched this -task; the worker records the exact run `TaskResponse` there as -`protocol/task_response.json`. A supplied `trace_manifest_path` must be that -absolute session/attempt-derived manifest. Missing or different paths mark -capture incomplete and cannot redirect finalization. - -`success` has the deliberately narrow D3 meaning: team-harness completed -without raising an execution error. It does not mean the work is correct and -does not consult spawned-agent exit codes for semantic acceptance. Eval and -control artifacts decide whether a session goal is met. - -`usage` covers team-harness coordinator turns found in the returned -`run_json_path`; agent CLI subprocess usage is not measurable. Missing usage -means unknown, not zero. `turns_without_usage > 0` also makes the total only a -measured subtotal. `failure_kind` is `transient`, `deterministic`, or `unknown` -for worker-reported failures; coordinator crash recovery records `crash`. - -Stale and replay rules are: - -- A mismatch in session, workflow, iteration, or modern attempt ID does not - mutate the live task. -- The live task can be replayed only to its recorded owner. A stale completion - from another identified worker returns HTTP 409. -- If no task is active, `/finished` advances exactly as `/register` would. -- A stale call may receive the current scheduler response, but it never appends - history, creates or updates a finalization intent, records a finished - exchange, or seals the stale attempt. -- A task persisted before attempt IDs or owner binding retains only its legacy - comparison behavior; tolerance belongs to that old task, never to a new one. +The response is another `TaskResponse`, so `/finished` is both completion +acknowledgement and the next scheduling poll. + +For protocol v3, completion must echo the exact worker owner, repository ID, +assignment SHA-256, session, workflow, iteration, and attempt ID. A late result +from a superseded attempt cannot complete a retry with otherwise identical +coordinates. + +The coordinator records the mechanical result, optional usage/duration, +harness run, relocatable trace reference, assignment hash, and hashes of the +exact request and returned response. The worker-written absolute +`trace_manifest_path` must equal the canonical session/attempt-derived trace; +it cannot redirect finalization. + +`success: true` has the narrow D3 meaning: team-harness returned without an +execution error. It does not assert that implementation work is correct, +accepted, merged, or complete, and it does not infer semantic failure from a +spawned process exit code. In protocol v3, the contract's durable orchestrator +decides `goal_met` through `control.json` after weighing optional eval and other +evidence. + +For protocol v3, `_record_finished_task()` also: + +- scans current-attempt receipts when the workflow is a declared check runner, + accepting valid provenance into compact seals and emitting diagnostics for + invalid observations without changing mechanical success; and +- observes the rolling handoff's identity/revision/hash, again as diagnostics + rather than a semantic gate. + +Stale/replay rules are strict: a coordinate or owner mismatch does not mutate +the live task, and a stale call cannot append history, create a trace +finalization intent, or seal the stale attempt. If no task is active, +`/finished` advances as `/register` would. The worker writes `result.json` and `pending_finished_request.json` before the -HTTP call. It removes the pending handoff only after acknowledgement. This is -why a worker crash between local completion and `/finished` can be recovered -without rerunning the harness. - -For an exact matching completion, the coordinator writes a trace-finalization -intent before committing state, then records the exact observed response and -seals. Startup acts only when history proves that completion or abandonment -committed; an interrupted response is marked unavailable rather than invented. -Trace failure is logged and does not roll back semantic state. The full -artifact ordering and repair rules are in -[`session-layout.md`](./session-layout.md#caller-owned-attempt-traces). +HTTP call, removing the latter only after acknowledgement. This closes the +worker-crash window between local completion and coordinator acceptance. ## File protocols consumed during `/finished` -The following are not HTTP bodies. They are durable workflow outputs that the -coordinator reads while recording a matching completion. The assignment -contains their absolute paths; portable cross-file links use logical -`session:/`, `parent:/`, `root:/`, or `trace:` references. +These records are not HTTP bodies. Workflows publish them at assignment-provided +absolute paths. Durable links use confined logical references such as +`session:/`, `root:/`, `parent:/`, `session::/`, and `trace:`. -### Recursive child request v2 +### Recursive child request -A workflow requests a depth-first child by atomically publishing a unique JSON -file under its assigned `child_requests/pending/` directory: +Protocol-v3 recursive sessions continue to use child-request schema v2: ```json { "schema_version": 2, - "request_id": "feature-auth-1", + "request_id": "foundation-phase", "workflow_set": "inner_outer_eval", "origin": { "parent_attempt_id": "a1b2c3d4e5f6", - "parent_work_item_id": "FEATURE-4", + "parent_work_item_id": "PHASE-0", "supersedes_request_id": null }, "assignment": { - "goal": "Implement the selected authentication slice.", - "completion_criteria": ["The child-scoped outcome passes evaluation"], + "goal": "Make the development foundations ready and evidenced.", + "completion_criteria": ["The phase outcome is complete and reviewable"], "stop_criteria": ["A genuinely terminal blocker is established"], "constraints": [], - "deliverables": ["code and verification evidence"], - "required_evidence": ["eval, git, and delivery receipts"] + "deliverables": ["implementation and verification evidence"], + "required_evidence": ["appropriate Git, review, test, and delivery refs"] }, "inputs": [ { - "ref": "session:/project_state/dispatch_inputs/feature-auth-1.json", + "ref": "session:/project_state/dispatch_inputs/foundation-phase.json", "sha256": "sha256:<64 hex characters>" } ] } ``` -The packaged PM dispatcher creates that per-request snapshot before hashing -and publishing the request. It never declares mutable `work_items.md` as an -input: the dispatcher must update that ledger after publication, while the -coordinator validates the declared input bytes after the attempt returns. - -The coordinator checks the schema, requested workflow set, input reference -confinement, and input hashes. A dispatchable body is copied unchanged to -`child_requests/accepted/.json`, hashed, indexed in -`children.json`, and copied again into the child's immutable -`inputs/accepted_request.json`. Every declared parent `inputs[]` reference is -resolved and hash-checked from the parent's scope, then its exact bytes are -copied to the child's `inputs/artifacts/`. The child goal contract and all of -its attempts use only those child-local references, hashes, and worker-local -absolute paths; `session.json.origin` retains the parent source-to-copy mapping -for provenance. Later parent edits therefore cannot change or wedge an accepted -child assignment. The pending file is then removed. `request_id` supplies -idempotency across retries; it cannot be reused with a different accepted body. - -An invalid or undispatchable v2 pending request is moved to -`child_requests/rejected/` and gets a separate rejection receipt containing -the reason and original hash. This is a terminal disposition of that request, -not a human approval gate. A later workflow may autonomously publish a repaired -request with a new identity. - -### Eval receipt and goal-check projection - -The canonical evaluation artifact is a compact receipt under -`eval_receipts/`. Its own schema is currently version 1 even when the enclosing -session protocol is v2: +The coordinator validates the schema, current origin attempt, requested +workflow set, reference confinement, and hashes. It copies a valid body into +`child_requests/accepted/`, binds it into `children.json`, and freezes exact +request/input bytes in the child's own `inputs/` and `goal_contract.json`. +`request_id` makes dispatch idempotent and cannot be reused with contradictory +bytes. + +Invalid requests move to `child_requests/rejected/` with their original hash +and reason. This is a repairable autonomous disposition, not a human approval +step. The stock PM dispatcher publishes one planner-selected high-level +milestone outcome; it does not pre-decompose the child's leaf plan. + +### Advisory eval receipt and acceptance seal + +The canonical receipt schema remains v1 inside a protocol-v3 session. It binds +the evaluated root/session/goal/Git subject, exact producing +workflow/iteration/attempt/harness run, check definitions, judge settings, +per-check results, verdict, canonical report, raw report references/hashes, and +creation time. A compact example is: ```json { @@ -415,7 +398,7 @@ session protocol is v2: }, "producer": { "workflow_id": "eval_runner", - "iteration": 4, + "iteration": 3, "attempt_id": "fedcba654321", "harness_run_id": "run-456" }, @@ -426,129 +409,199 @@ session protocol is v2: "kind": "harness_judge" } ], - "judge": {"provider": "codex", "model": "gpt-5.5", "reasoning_effort": "high"}, + "judge": { + "provider": "", + "model": "", + "reasoning_effort": "" + }, "check_results": [ - {"check_id": "goal-outcome", "passed": true, "reason": "Evidence passes."} + {"check_id": "goal-outcome", "passed": false, "reason": "Gap found."} ], - "verdict": {"goal_met": true, "reason": "All declared checks passed."}, + "verdict": {"goal_met": false, "reason": "The gap remains."}, "canonical_report_ref": "session:/eval_receipts/eval-a1b2c3d4e5f6.report.md", "canonical_report_sha256": "sha256:<64 hex characters>", - "raw_report_refs": [ - "trace:trace-fedcba654321:/eval/report.json" - ], + "raw_report_refs": ["trace:trace-fedcba654321:/eval/report.json"], "raw_report_sha256s": { "trace:trace-fedcba654321:/eval/report.json": "sha256:<64 hex characters>" }, - "created_at": "2026-07-15T12:00:00Z" + "created_at": "2026-07-17T10:00:00Z" } ``` -An eval-emitting workflow also writes a small iteration-local -`goal_check.json` whose verdict and reason exactly project that receipt: +When the exact producing attempt completes, the coordinator accepts a receipt +only if its producer role is in the frozen `evaluation.check_runner_roles` and +all subject, harness, check-definition, report, judge, hash, and evaluated-Git +provenance validates. It then writes an engine sidecar: ```json { - "schema_version": 2, - "goal_met": true, - "reason": "All declared checks passed.", - "eval_receipt_ref": "session:/eval_receipts/eval-a1b2c3d4e5f6.json" + "schema_version": 1, + "receipt_ref": "session:/eval_receipts/eval-a1b2c3d4e5f6.json", + "receipt_sha256": "sha256:<64 hex characters>", + "subject": {"root_session_id": "root-session-id", "session_id": "current-session-id", "goal_hash": "sha256:<64 hex characters>", "git_commit": "0123456789abcdef", "dirty_tree_digest": "sha256:<64 hex characters>"}, + "producer": {"workflow_id": "eval_runner", "iteration": 3, "attempt_id": "fedcba654321", "harness_run_id": "run-456"}, + "evaluated_git": {"git_commit": "0123456789abcdef", "dirty_tree_digest": "sha256:<64 hex characters>"}, + "accepted_at": "2026-07-17T10:01:00Z" } ``` -Every receipt, passing or failing, must have exactly one `raw_report_refs` -entry: the producing attempt's canonical `eval/report.json`. -`raw_report_sha256s` has that same single key, and both the canonical and raw -artifacts must match their full SHA-256 values. Each check's -`definition_sha256` is eval-banana's canonical, versioned definition digest -copied from that report, not a raw hash of the YAML file. The coordinator -independently recomputes the value with eval-banana before accepting the -receipt. -`CoordinatorService._read_goal_check_signal()` and -`_validate_eval_receipt_artifacts()` also check the exact current -producer/attempt/iteration, root/session/goal identity, every regular -`*.yaml`/`*.yml` check discovered recursively below `eval_checks/`, -definition-byte hashes, and agreement between receipt and projection. Symlinks -and non-files are rejected. Receipt JSON/schema failures retain up to eight -field-qualified validation messages in failed history and terminal-control -rejection evidence so the responsible workflow can repair the exact field. A -passing report must additionally record the exact absolute target -`project_root`, the producing attempt's absolute canonical `output_dir`, -`run_passed: true`, `pass_threshold: 1.0`, exactly the receipt's check IDs with -every status `passed` and exit code zero, and per-check -`details.agent_type`/`details.model`/`details.reasoning_effort` matching the -receipt's judge. At terminal acceptance the coordinator recaptures live Git -state and requires its HEAD and `loopy-git-status-diff-v1-sha256` digest to -match the receipt. The digest binds filtered porcelain records plus Git's -staged and unstaged binary diffs. It intentionally relies on Git's observable -boundary rather than independently parsing and re-hashing the complete index. - -These checks establish provenance and all-pass mechanics. The coordinator does -not re-evaluate the LLM judge's semantic reasons or require stock deterministic -checks (D3/D4/D8). - -A valid `goal_check.json` is not a stop switch. - -### Terminal control v2 - -A packaged v2 workflow set accepts `goal_met` only from its declared -`goal_control_role` and only when it cites the matching passing eval receipt: +The sidecar lives at `eval_receipts/accepted/.json`; its trust-root +copy lives in `state.json`. A later orchestrator attempt can cite the receipt +without requiring gitignored raw trace bytes to remain present. The receipt's +verdict may pass or fail: acceptance establishes provenance, not semantic +agreement. + +Missing, malformed, or non-passing advisory eval output is recorded in an +`eval_observation` event. It does not flip `IterationResult.success`, consume a +generic workflow failure budget, create `goal_check_broken`, or block +orchestrator completion. `goal_check.json` is optional/legacy in v3. + +### Protocol-v3 terminal control + +Successful control comes from the frozen orchestration owner (`outer` in +`inner_outer_eval`, `planner` in `pm_planner_dispatcher`): ```json { - "schema_version": 2, - "control_id": "control-fedcba654321", + "schema_version": 3, + "control_id": "control-a1b2c3d4e5f6", "state": "stopped", - "reason": "The session-scoped eval passed.", + "reason": "Why this layer's own goal is complete.", "stop_reason": "goal_met", "producer": { "session_id": "current-session-id", - "workflow_id": "eval_runner", - "attempt_id": "fedcba654321" + "workflow_id": "outer", + "attempt_id": "a1b2c3d4e5f6" }, - "eval_receipt_ref": "session:/eval_receipts/eval-a1b2c3d4e5f6.json", - "attempted_routes": [], + "evidence_refs": ["session:/delivery_receipts/pr-42.json"], + "eval_receipt_refs": [ + "session:/eval_receipts/eval-a1b2c3d4e5f6.json" + ], + "handoff_ref": "session:/project_state/handoff.json", + "created_at": "2026-07-17T10:02:00Z" +} +``` + +`evidence_refs`, `eval_receipt_refs`, and `handoff_ref` are optional; lists may +be empty. The engine requires exact current producer identity and completion +authority. Any cited evidence must resolve to a file. Any cited eval receipt +must match an engine acceptance seal, this session/root/goal, and a declared +runner role. A cited handoff must be the canonical layer handoff with matching +session/goal and declared owner when it has a producer. + +No eval receipt, passing verdict, same-attempt eval, or `goal_check.json` is +required. The orchestrator owns the semantic disposition, including how it +weighs conflicting observations. + +The D5 last-resort blocker keeps exact identity but does not cite eval or +handoff records: + +```json +{ + "schema_version": 3, + "control_id": "control-terminal-blocker-a1b2c3d4e5f6", + "state": "stopped", + "reason": "The specific unavoidable blocker and why a human is required.", + "stop_reason": "unresolvable_error", + "producer": { + "session_id": "current-session-id", + "workflow_id": "inner", + "attempt_id": "a1b2c3d4e5f6" + }, + "attempted_routes": ["The autonomous recovery route already tried"], + "evidence_refs": ["session:/project_state/decisions/blocker.md"], + "created_at": "2026-07-17T10:02:00Z" +} +``` + +The producer must be in `terminal_blocker_reporting_roles`; attempted routes +must be non-empty. There is no `paused` or `waiting_for_human` state. + +Invalid v3 terminal control is not accepted as a stop. The original bytes move +to `control_rejected/`, `protocol_failures/` records its hash, producer, and +reasons, and `control.json` becomes a running repair placeholder. Consecutive +invalid control records are bounded by the configured control-failure counter. +An invalid cited receipt is a false control claim; an uncited malformed +advisory observation is only an eval diagnostic. + +### `session_outcome.json` and child link + +After any v3 terminal transition, the coordinator writes the same engine-owned +result shape for root and child sessions. Control-owned outcomes include the +accepted control hash; engine-owned lifecycle stops use `control: null`: + +```json +{ + "schema_version": 1, + "session_id": "current-session-id", + "root_session_id": "root-session-id", + "goal_sha256": "sha256:<64 hex characters>", + "lifecycle": "terminal", + "terminal_status": "goal_met", + "stop_reason": "goal_met", + "terminal_state_revision": 13, + "control": { + "ref": "session:/control.json", + "sha256": "sha256:<64 hex characters>" + }, + "handoff": { + "status": "valid", + "ref": "session:/project_state/handoff.json", + "sha256": "sha256:<64 hex characters>", + "revision": 7 + }, + "fallback_summary": null, "evidence_refs": [], - "created_at": "2026-07-15T12:00:00Z" + "delivery_refs": [], + "eval_refs": [], + "trace_seal_refs": [], + "created_at": "2026-07-17T10:02:01Z" +} +``` + +Handoff status may also be `missing`, `invalid`, or `non_monotonic`. In those +cases `fallback_summary` copies the authenticated control reason, or the +factual engine stop reason when `control` is null; it does not invent a +semantic summary. A weak handoff is completeness information, not a control +veto. Delivery evidence is projected across the whole session history, not +only the terminal attempt. + +The terminal status, revision, timestamp, accepted control bytes, and accepted +handoff bytes are frozen in engine-owned `state.json`. Regeneration restores +those exact bytes rather than trusting later file edits. Only the list of +trace-seal references may expand as already-accepted attempts finish trace +finalization. + +A v3 parent's `child_outcomes/.json` is a small link: + +```json +{ + "schema_version": 2, + "request_id": "foundation-phase", + "child_session_id": "child-session-id", + "session_outcome_ref": "session:child-session-id:/session_outcome.json", + "session_outcome_sha256": "sha256:<64 hex characters>" } ``` -The D5 last-resort `unresolvable_error` form instead requires a producer role -listed by the workflow contract, at least one attempted autonomous route, and -no eval receipt. It may include logical evidence references. - -Both terminal forms must identify the exact current session/workflow/attempt -whose matching `/finished` is being recorded. Control written by an earlier -attempt, another layer, or a spawned agent is rejected. A spawned agent reports -to its harness coordinator; the current accountable workflow publishes the -session-owned record. - -Invalid terminal v2 control is not accepted as a stop. The coordinator moves -the original to `control_rejected/`, writes a reasoned record in -`protocol_failures/`, restores a running control record, and lets later work -repair the protocol. Repeated invalid control is bounded by -`goal_check_consecutive_failures_cap`, which is currently shared by the eval -projection and terminal-control protocol. There is no -paused/waiting-for-human control state. - -## Legacy compatibility boundary - -Resume keeps old work usable without weakening new work: - -- Missing `LoopState.schema_version` means v1. Fresh state explicitly writes - v2 and a revision. -- A v1 session may use legacy child requests shaped as - `{"schema_version":1,"workflow_set":"...","goal":"..."}` in the flat - `child_requests/` directory, terminal control v1, and goal-check v1. -- The child-request reader observes both the legacy flat inbox and v2 - `pending/`. Flat rejected requests retain the historical `.rejected` naming; - new pending requests use the `rejected/` archive. -- A v1 task has no immutable workflow snapshot/assignment requirement and may - use `harness_outputs/` rather than the caller-owned trace path. -- A packaged v2 workflow contract does not accept v1 terminal control, - goal-check output, or child requests. Such artifacts are repairable protocol - failures, not implicit downgrades. - -Compatibility code is concentrated in -`CoordinatorService._workflow_contract_for_state()`, the dual child-request -reader in `_dispatch_child_session_if_requested()`, and the optional/additive -fields in `models.py`. +The parent independently decides acceptance, rework, or reroute. A child +outcome never sets ancestor `goal_met`. + +## Frozen v1/v2 compatibility + +The current implementation still resumes historical sessions under their +frozen behavior: + +- v1 may register with process identity alone and may use flat child requests, + terminal control v1, goal-check v1, and legacy output paths; +- v2 requires worker protocol 2 plus the seven carried-forward capabilities, + repository binding, immutable assignment/snapshot, identity-bound terminal + control, and its historical same-attempt passing receipt plus matching + `goal_check.json` for `goal_met`; and +- v2 control uses singular `eval_receipt_ref`, while v3 uses plural + `eval_receipt_refs`. + +An explicit custom contract omitting `session_protocol_version` remains pinned +to v2; a workflow set with no contract derives conservative v1. Compatibility +is selected from durable session state, never by accepting a lower worker +version for a v3 tree. diff --git a/docs/session-layout.md b/docs/session-layout.md index 831fd1b..4458812 100644 --- a/docs/session-layout.md +++ b/docs/session-layout.md @@ -1,29 +1,28 @@ # Session Layout -loopy-loop separates compact durable state from detailed execution traces: +loopy-loop keeps two deliberately different runtime trees: -- `.loopy_loop/sessions/` is the state machine and compact evidence needed to - schedule, recover, evaluate, and understand a recursive run. -- `.loopy_loop/traces/` is detailed, attempt-scoped observability: model - envelopes, team-harness records, direct-agent assignments and logs, raw eval - output, and verbose git evidence. Traces are independently sealable and - gitignored. +- `.loopy_loop/sessions/` is compact durable state. It contains the facts + needed to schedule, resume, explain, and hand off a run. +- `.loopy_loop/traces/` is detailed attempt observability: prompts, harness + records, spawned-agent assignments and streams, raw eval output, and verbose + Git evidence. -Both trees are runtime output and are ignored by the generated `.gitignore`. -Ignoring session state does not make it disposable while a run is active: -continuity lives in these files. Trace retention is independent because the -compact receipts needed for correctness remain in the session. +Both trees are gitignored by the generated `.gitignore`. Session state is still +required while a run is active; gitignored does not mean disposable. Traces may +be retained, deleted, or exported independently because any evidence used by +the state machine is represented by compact, hash-bound session receipts. -Path construction is centralized in `src/loopy_loop/sessions.py`. Fresh 0.7 -runs use the v2 layout below. Existing v1 sessions remain readable and are -described at the end. +Path construction is centralized in `src/loopy_loop/sessions.py`. The stock +`inner_outer_eval` and `pm_planner_dispatcher` workflow sets declare semantic +protocol v3. The persisted coordinator-state schema remains v2: state schema +and agent-facing session protocol are separate version axes. -## Recursive tree +## Recursive session tree -A root session is a direct child of `.loopy_loop/sessions/`. Every requested -child is physically nested under its parent. The same shape can recurse to any -depth; there is no separate hard-coded state machine for a planner/dispatcher -or a three-loop deployment. +A root session is a direct child of `.loopy_loop/sessions/`. A requested child +is physically nested under its parent, and the same shape can recurse again. +There is no separate hard-coded tree format for one-, two-, or three-loop use. ```text .loopy_loop/ @@ -34,18 +33,27 @@ or a three-loop deployment. │ ├── goal_contract.json │ ├── session.json │ ├── workflow_contract.json +│ ├── workflow_roster.json +│ ├── harness_capability_roster.json # root only; shared by the tree │ ├── state.json │ ├── events.jsonl │ ├── control.json +│ ├── session_outcome.json # after any v3 terminal transition │ ├── inputs/ │ │ ├── user_updates.jsonl -│ │ ├── accepted_request.json # child sessions only -│ │ └── artifacts/ # frozen parent inputs +│ │ ├── accepted_request.json # child only +│ │ └── artifacts/ # frozen child inputs │ ├── project_state/ -│ │ └── finished.md +│ │ ├── plan.md +│ │ ├── tasks/ +│ │ ├── current_state.md +│ │ ├── decisions/ +│ │ ├── finished.md +│ │ ├── eval_state.md +│ │ └── handoff.json │ ├── eval_checks/ -│ ├── eval_readiness/ │ ├── eval_receipts/ +│ │ └── accepted/ # engine acceptance seals │ ├── child_requests/ │ │ ├── pending/ │ │ ├── accepted/ @@ -59,400 +67,354 @@ or a three-loop deployment. │ ├── control_rejected/ │ ├── protocol_failures/ │ ├── iterations/ -│ │ └── 0001_/ -│ ├── harness_outputs/ +│ ├── harness_outputs/ # legacy-compatible projection │ └── children/ │ └── / │ ├── parent.json -│ ├── goal.md -│ ├── goal_contract.json -│ ├── session.json -│ ├── workflow_contract.json -│ ├── state.json -│ ├── ...same session directories... -│ └── children/ -│ └── /... +│ ├── workflow_roster.json +│ ├── ...same per-session files... +│ └── children//... ├── traces/ │ └── /sessions//attempts// └── trace_finalization_outbox/ ``` -Only the deepest live session receives work. While a child runs, its parent has -`active_child_session_id` set and is suspended. When a child becomes terminal, -the coordinator writes a factual outcome, clears the pointer, and resumes the -parent; multiple terminal ancestors can be unwound in one iterative transition. -The durable pointer and recovery logic are in -`CoordinatorService._suspended_parent_response()` and -`_resume_parent_if_active_child_completed()`. +Only the deepest live session receives work. A parent with +`active_child_session_id` is suspended until that child becomes terminal. The +coordinator then records the child result, clears the pointer, and resumes the +parent. `CoordinatorService._suspended_parent_response()` and +`_resume_parent_if_active_child_completed()` implement this depth-first stack. -`.loopy_loop/repository.json` is not session state. It contains the stable -checkout identity created by `assignments.ensure_repository_identity()` and is -used to bind coordinator and worker to the same checkout. +`.loopy_loop/repository.json` is checkout identity, not session state. +`assignments.ensure_repository_identity()` creates it so coordinator and +worker can prove that they are operating in the same checkout. -## Immutable identity and contracts +## Frozen identity, scope, and role contracts -These files explain what one loop layer is and what it owns. They are written -when the session is created and are not a substitute for mutable progress -state. +### `goal.md` and `goal_contract.json` -### `goal.md` +`goal.md` is the exact resolved goal for this layer. A child receives its +child-scoped goal, never the parent's broader goal. `goal_contract.json` binds +that text and full hash to completion criteria, stop criteria, constraints, +deliverables, required evidence, and origin metadata. -The exact resolved goal text for this layer. A child receives its -child-assignment goal, not the parent's broader goal. The configured root goal -file may later change; active work continues to use this session-local copy. +A child's contract also binds the accepted request and every declared parent +input by logical reference and SHA-256. The exact bytes are copied into the +child's own `inputs/` tree. Later parent edits therefore cannot silently change +an already accepted child assignment. -### `goal_contract.json` +### `session.json` and `parent.json` -The typed scope for this layer: goal and full hash, completion/stop criteria, -constraints, deliverables, required evidence, and terminal-blocker policy. A -child contract additionally freezes: +`session.json` is the immutable manifest: session/root/parent identity, depth, +workflow set, layer kind, goal and contract hashes, origin provenance, and +creation time. A child also has `parent.json`. Logical-reference resolution +checks these declarations and physical nesting before allowing a cross-layer +reference. -- the originating `request_id`; -- a child-local logical reference and hash of the accepted request body; and -- child-local logical references and hashes for exact copies of every declared - parent input. +### `workflow_contract.json` -`assignments.build_attempt_assignment()` verifies these hashes again before -the harness runs and exposes worker-local absolute paths for accepted inputs. -`session.json.origin` keeps the original parent references/hashes and their -mapping to these frozen copies, so provenance does not depend on mutable parent -state. +This is the session-frozen role and authority contract. In protocol v3 it +separates orchestration from evaluation: -### `session.json` +- `orchestration` names one role that owns the layer plan, rolling handoff, and + successful completion decision; +- `evaluation` names optional check authors and runners; their results are + advisory evidence; and +- `terminal_blocker_reporting_roles` names roles allowed to report the D5 + last-resort `unresolvable_error`. -The immutable session manifest. V2 records include `session_id`, -`root_session_id`, `parent_session_id`, `depth`, `workflow_set`, `layer_kind`, -goal and goal-contract hashes, origin provenance, and creation time. A child -also has `parent.json`, which records the parent identity and physical relative -path. `references.LogicalReferenceResolver` validates physical nesting against -these declarations before resolving cross-layer references. +The stock `inner_outer_eval` contract has four scheduled roles. `outer` owns +the plan, handoff, leaf acceptance, and `goal_met`; `inner` executes one +outer-selected leaf; `eval_reviewer` optionally authors checks; and +`eval_runner` optionally publishes observations. The same workflow set works +as either a root or a child. -Session IDs use `___`. The short -fragment makes names readable; the contracts use full SHA-256 where provenance -requires it. +The stock `pm_planner_dispatcher` contract has only `planner` and `dispatcher`. +`planner` owns the high-level milestone plan, child acceptance/rerouting, +handoff, and `goal_met`. `dispatcher` transports one planner-selected outcome +into a typed child request. PM-level scheduled eval roles are intentionally not +duplicated; the planner may obtain or run optional final evidence itself. -### `workflow_contract.json` +The complete parsed contract is also held in engine-owned `state.json`. Before +dispatch, the coordinator restores rewritten on-disk contract projections from +that trust root and freezes the exact bytes into the attempt snapshot. -The selected workflow set's agent-visible role-contract projection: session protocol version, -layer kind, named role responsibilities, state declarations, eval author/ -runner/control roles, task-acceptance role, terminal-blocker reporting roles, -and child interface. The complete v2 contract is also stored in coordinator-owned -`state.json`; before a later attempt, the coordinator restores this file and -its `session.json` hash from that trust root if both projections were rewritten. -The attempt snapshot then freezes and hashes the restored bytes. +### `workflow_roster.json` -## Coordinator state and observations +Protocol v3 creates this once per session from the frozen workflow definitions +and contract. It lists every scheduled role with its responsibility, cadence +(`priority`, `run_every`, `run_on_start`, `must_follow`, +`run_after_successes`, and related fields), expected outputs, and authorities. +It is the inspectable answer to “which durable roles exist in this layer and +what are they responsible for?” It is not a catalog of agents dynamically +spawned inside one team-harness run. -### `state.json` +### `harness_capability_roster.json` + +Protocol v3 creates one roster at the root and every descendant references the +same absolute file. It freezes: + +- the configured harness-coordinator provider/model; +- every enabled delegate harness family; and +- the `frontier`, `strong`, `standard`, and `economy` model/effort bundle for + each family, with missing cells explicitly marked unavailable. -The coordinator-owned source of truth. V2 state is revisioned on mutations and -contains: +The roster contains no credentials. It is guidance and audit context, not a +model-policy gate. Prompts use tier names, while concrete model IDs remain in +configuration and this generated roster. The worker also forwards the path, +hash, and compact roster object through team-harness caller context so nested +harness coordinators see the same tree-wide catalog. -- this layer's status, goal/workflow identity, root/parent/depth, turn limits, - and frozen execution snapshot; -- `current_task`, including attempt, worker owner, repository identity, and - frozen workflow-snapshot descriptor; -- iteration history and per-workflow failure counters; -- the active-child pointer and child-request/work-item provenance; -- token/duration totals and stop/control-failure state. +## Coordinator state and event projection + +### `state.json` + +`state.json` is the engine-owned scheduling source of truth. Its v2 persisted +schema contains the current task, iteration history, failure and usage ledgers, +active-child pointer, frozen execution and workflow contracts, and terminal +state. For protocol v3 it additionally keeps the root capability roster trust +root, the latest observed handoff revision/hash, and accepted eval-receipt +seals. The convenient phase (`ready`, `executing`, `suspended`, or `terminal`) is -derived from status, current task, and active child; it is not a second durable -state machine. `StateStore` owns state writes. Agents receive `state.json` as -read-only engine state through `assignment.json`; they communicate desired -transitions through typed workflow-owned files. +derived from status, current task, and child pointer. Agents receive the +absolute `state.json` path as read-only engine state and request transitions by +publishing typed workflow-owned files. ### `events.jsonl` -A best-effort observability projection appended after state mutations commit. -Each line is a schema-v1 envelope with `event_id`, UTC timestamp, session ID, -type, and payload. Current event types include `session_started`, -`task_dispatched`, `task_finished`, `iteration_abandoned`, `goal_check`, -`child_started`, `child_finished`, and `session_stopped`. +This is a best-effort post-commit observability stream. Consumers must tolerate +duplicates, gaps, and a truncated last line and deduplicate by `event_id`. +Scheduling and recovery use `state.json`, not this projection. Protocol-v3 +observations include `eval_observation` and `handoff_observed`; neither is a +semantic acceptance gate. -Consumers must tolerate gaps, duplicates, and a truncated final line and -deduplicate by `event_id`. Scheduling and recovery use `state.json`, not the -event stream. `loopy events --follow` tails it. +## Protocol-v3 semantic state spine -### `control.json` +`sessions._create_v3_semantic_spine()` scaffolds the same compact state in +every root and child: -The workflow-owned stop request. A fresh v2 session begins with a neutral, -v1-compatible running record: +| Path | Accountable owner | Purpose | +| --- | --- | --- | +| `project_state/plan.md` | layer orchestrator | Current outcomes, dependencies, revision, active selection, and replanning triggers for this layer | +| `project_state/tasks/` | orchestrator; leaf/dispatcher may contribute evidence | Stable per-task objective, status, dependencies, and accepted evidence | +| `project_state/current_state.md` | orchestrator | Short resumption view: active outcome, blockers, risks, and next decision | +| `project_state/decisions/` | orchestrator | Durable choices and rationale later attempts should not rediscover | +| `project_state/finished.md` | orchestrator | Append-only accepted-work ledger with commit, PR, test, review, and delivery references | +| `project_state/eval_state.md` | orchestrator; eval roles contribute observations | Optional evaluation intent, observations, disagreement, provenance, and possible next action | +| `project_state/handoff.json` | orchestrator | Rolling semantic summary for a parent or operator | -```json -{ - "state": "running", - "reason": "session active", - "stop_reason": null, - "schema_version": 1 -} -``` +The coordinator scaffolds headings and a revision-zero handoff but does not +parse plan prose, choose tasks from it, or judge semantic sufficiency. `outer` +or `planner` keeps the spine coherent. `inner` executes the selected task and +reports evidence; it does not create the layer plan when selection is absent. -For a workflow contract with `session_protocol_version: 2`, terminal control -must use the identity-bound v2 form. Successful `goal_met` must be authored by -the declared goal-control role and cite a valid passing receipt in this -session. Last-resort `unresolvable_error` must come from an allowed role and -record autonomous routes already exhausted. See -`CoordinatorService._apply_session_control()` and `_validate_v2_control()`. - -Both terminal forms must match the session, workflow, and attempt of the exact -current task whose `/finished` transition is being processed. Control from a -prior attempt or another layer is rejected even when its role name is allowed. -A spawned agent reports a conclusion to its harness coordinator; that current -workflow role publishes the session-owned control record. - -Invalid terminal v2 control is moved unchanged to `control_rejected/`. A -`protocol_failures/` record preserves its hash, producer, and rejection reasons, -and `control.json` is restored to running so autonomous repair can continue. -Consecutive invalid controls use `goal_check_consecutive_failures_cap`, shared -with invalid goal-check projections. This is detection and repair, not -preventive write fencing or a human gate. - -### `protocol_failures/` - -Compact, durable protocol diagnoses. In addition to invalid terminal control, -the coordinator uses this directory to preserve an unreadable v2 -`children.json` and record bounded reconstruction. Reconstruction derives only -from accepted requests, child manifests, and child state; it does not silently -treat a damaged ledger as empty. - -## Append-only inputs - -### `inputs/user_updates.jsonl` - -The v2 input channel used by `loopy update`. Each operator update appends a -`user_input` record with a unique ID, target scope/session, routed destination, -timestamp, exact text, and pending acknowledgement state. Without -`--session`, the CLI routes it to the deepest active layer; an explicit session -targets that layer. - -Before an attempt is rendered, `worker._semantic_prompt_context()` reads the -journal, appends one `input_delivery` record per pending input/attempt, and -includes pending records in the prompt. The workflow is instructed to append a -`user_input_acknowledgement` with disposition after acting. Earlier lines are -never edited. This distinguishes input creation, delivery, and semantic -acknowledgement without pretending the coordinator understood the text. - -`updates_from_user.md` is still scaffolded/readable for legacy compatibility, -but new operator updates use the JSONL journal. - -## Workflow-owned durable state and evidence - -### `project_state/` - -Markdown/JSON state maintained by workflow roles. The coordinator does not -interpret arbitrary files here. Typical packaged workflows keep current -planning context, decisions, accepted work, and handoff summaries. `goal.md` -and `goal_contract.json` remain authoritative for the layer's target; agents -should not create a competing goal in project state. - -`project_state/finished.md` is the outer/task-acceptance role's accepted-work -ledger. Mechanical harness completion does not append acceptance by itself. -Implementation entries should link compact eval, git, and delivery evidence. - -The packaged PM set also uses -`project_state/dispatch_inputs/.json` for immutable, per-request -selection snapshots. The dispatcher hashes this snapshot into a child request -before updating its mutable `work_items.md` ledger. A published snapshot is -never edited; a rework or reroute request receives a new request id and file. - -### Evaluation directories - -- `eval_checks/` contains session-scoped eval-banana YAML definitions. -- `eval_readiness/` contains task-acceptance/readiness context for later eval - work. The worker includes the latest receipt in semantic prompt context; it - does not force scheduler eligibility. -- `eval_receipts/` contains compact, identity-bound verdict receipts and - canonical report copies retained independently from detailed traces. - -Raw eval-banana output belongs under the attempt trace's `eval/` directory. -The attempt assignment exposes that exact absolute path as `raw_eval_output`. -Packaged eval workflows use hermetic `--no-project-config`, an explicit judge, -and pass threshold 1.0. Eval-banana persists each harness judge's exact input -as `checks/.prompt.txt`; the matching result, stdout, stderr, -and deterministic evidence directory use the same bounded stem. The stem -includes the full digest of the exact check ID, so case-only, normalized-label, -and long IDs remain distinct on the local filesystem. - -An eval receipt's own schema is v1 inside the v2 session protocol. It records: - -- root/session/goal and optional git subject identity; -- producer workflow, iteration, attempt, and harness run; -- a unique inventory of harness-judge check IDs and eval-banana canonical - definition SHA-256 values copied from the raw report; -- effective judge provider/model/reasoning effort; -- per-check results and the all-pass verdict; -- a canonical report reference/hash and the producing attempt's exact - `eval/report.json` reference, - with `raw_report_sha256s` keyed exactly by `raw_report_refs`; -- creation time. - -`CoordinatorService._validate_eval_receipt_artifacts()` resolves and verifies -the definition and report hashes. A definition digest is not the raw YAML file -hash: eval-banana frames the exact definition bytes with its versioned digest -protocol, emits that value in `report.json`, and exposes the same computation -for loopy-loop to verify independently. Every receipt, passing or failing, must have -exactly one `raw_report_refs` entry: the producing attempt's canonical -`eval/report.json`; the trace and harness identities must match the producer. -Every regular `*.yaml`/`*.yml` check found recursively beneath `eval_checks/` -must be valid, uniquely identified, included, and bound to the exact bytes -eval-banana ran; symlinks and non-files are rejected. JSON/schema failures -preserve up to eight field-qualified diagnostics in failed history and -terminal-control rejection evidence. For a passing receipt it additionally -requires the report's absolute `project_root` to match the target repository, -its absolute `output_dir` to match the producing attempt's canonical `eval/` -directory, `run_passed: true`, `pass_threshold: 1.0`, the exact receipt check -inventory with every status `passed` and exit code zero, and each check's -effective agent/model/reasoning effort to match the receipt judge. It also -recaptures live Git state and requires it to match the evaluated HEAD and -dirty-tree digest. -These are transport, provenance, and all-pass mechanics. The coordinator does -not reinterpret the LLM judge's semantic reasons (D3/D4/D8). - -An eval-emitting iteration also writes `goal_check.json` beside its result. V2 -contains `goal_met`, the exact receipt verdict reason, and an -`eval_receipt_ref`. It is an iteration-local projection, not a stop switch. - -### Parent/child handoff directories - -- `child_requests/pending/` is the v2 inbox. A request supplies a unique - `request_id`, origin attempt/work-item context, child-scoped assignment, and - optional hashed input references. -- `child_requests/accepted/` preserves the exact request body selected for - dispatch. Its hash is frozen into the child contract and ledger. -- `child_requests/rejected/` preserves invalid/undispatchable pending requests - and separate receipts with reasons and original hashes. -- `children.json` is the revisioned parent index of dispatch intent and child - lifecycle. It records request/child identity, accepted request evidence, - status, outcome, usage, and any failed dispatch projections. -- `child_outcomes/.json` is written by the coordinator when the - child becomes terminal. It reports lifecycle, usage, compact handoff/eval/ - git/delivery references when present, latest trace reference, and - completeness flags. It is factual evidence, not parent acceptance. -- `parent_acceptance/` is separately owned by the accountable parent workflow. - A child's `goal_met` result never automatically closes its parent. - -The reader observes both v2 `pending/*.json` and the legacy flat -`child_requests/*.json` inbox. Request identity prevents v2 redispatch; -recorded dispatch state also closes the crash window between creating a child -and removing the pending file. - -### `git_receipts/` and `delivery_receipts/` - -The worker writes compact git boundary receipts before and after each v2 -attempt. Verbose status/diff material goes into the trace, while hashes and -summary facts remain under `git_receipts/`. Workflow roles may write delivery -receipts for branches, PRs, merges, or other declared deliverables. These -compact records do not depend on detailed trace retention. - -Schema-v2 receipts use `loopy-git-status-diff-v1-sha256`. The digest binds -byte-stable, non-runtime porcelain status records plus Git's staged and -unstaged binary diffs. Engine runtime session/trace/outbox/state paths are -excluded, but versioned `.loopy_loop/workflow_sets/` definitions are product -input and remain part of the subject. Untracked and nested repositories remain -visible through porcelain paths; the compact digest is not a second archive of -their complete contents, which belong in the verbose trace when observable. - -## Iteration directory - -Each loopy task has a recovery-critical directory: +`handoff.json` binds the session and goal, a monotonically increasing revision, +the producing workflow/attempt, a summary, and flexible accepted/open/risk/ +decision/evidence/delivery/eval lists. The engine validates its structure and +records continuity diagnostics. A missing, malformed, or non-monotonic handoff +does not override an authentic orchestrator completion decision. + +Protocol v3 retires `eval_readiness/`; readiness and evaluation headlines live +in `project_state/eval_state.md`, and scheduled-role timing is visible in the +attempt's scheduler view. Frozen v2 sessions retain `eval_readiness/`. + +## Append-only user input + +`inputs/user_updates.jsonl` is the channel used by `loopy update`. An update +records its ID, target, routed session, timestamp, exact text, and pending +acknowledgement state. Before an attempt, `worker._semantic_prompt_context()` +records delivery and renders pending inputs. After acting, the workflow appends +an acknowledgement rather than editing earlier lines. + +## Attempt snapshot and absolute assignment paths + +Every modern task has a recovery-critical iteration directory: ```text -.loopy_loop/sessions//iterations/_/ -├── workflow_snapshot/ -│ └── / -│ ├── assignment.json -│ ├── config.yaml -│ ├── prompt.txt -│ ├── workflow_contract.yaml -│ ├── root_config_snapshot.json -│ └── manifest.json +iterations/_/ +├── workflow_snapshot// +│ ├── assignment.json +│ ├── scheduler_view.json # protocol v3 +│ ├── config.yaml +│ ├── prompt.txt +│ ├── workflow_contract.yaml +│ ├── root_config_snapshot.json +│ └── manifest.json ├── prompt.txt ├── result.json ├── result_text.txt ├── harness_run_id.txt ├── pending_finished_request.json ├── trace_ref.json -├── goal_check.json # only when the workflow emits it -└── salvage.json # only after applicable crash recovery +├── goal_check.json # optional/legacy projection +└── salvage.json # only after applicable recovery ``` -### `workflow_snapshot//` - -Created by `assignments.materialize_workflow_snapshot()` before dispatch. It -freezes the exact selected workflow config, prompt, workflow-set contract, root -execution snapshot, identity, repository identity, and hashes for one attempt. -The worker verifies all members before executing. This prevents a mid-run -workflow edit from silently changing an already-dispatched task. - -### `workflow_snapshot//assignment.json` +`materialize_workflow_snapshot()` freezes the scheduler-selected workflow +configuration, prompt, contract, root execution snapshot, repository identity, +and hashes. `assignment.json` then binds one actor, objective, checkout, +session, workflow, iteration, and attempt. The worker verifies the snapshot, +reconstructs the assignment independently, and checks the coordinator's frozen +assignment SHA-256 before calling the harness. -Created by the coordinator before dispatch, hashed into `CurrentTask`, and -verified independently by the v2 worker before the model call. It contains: +Protocol-v3 `assignment.json` has schema version 2 and carries absolute paths +under stable keys. The semantic and orchestration keys are: -- root/current/parent session, depth, request/work-item, workflow, iteration, - and attempt identity; -- actor kind `harness_coordinator`, workflow role, layer kind, and role - responsibility; -- the scoped objective, expected outputs/evidence, and frozen child inputs; -- absolute paths to this checkout's relevant state, evidence, iteration, and - trace locations; -- plain-language ownership: integrate within this session, use typed receipts - across parent boundaries, and treat engine state as read-only; -- repository/config/workflow/goal/git provenance hashes. - -Absolute paths are execution-time capabilities for agents in this checkout. -Durable records use logical references so the session tree can be inspected -after relocation. The coordinator does not enforce assignment ownership as a -filesystem ACL; violations are detected through eval/evidence (D8). - -### `prompt.txt` - -The exact rendered task passed to `TeamHarness.run(task=...)`, persisted before -the first provider call by `harness_runner.write_iteration_inputs()`. It -includes assignment/role context, absolute paths, pending user inputs, latest -eval-readiness context, and the frozen workflow prompt. - -### `result.json`, `result_text.txt`, and `harness_run_id.txt` - -`result.json` is the worker-normalized `IterationResult`: mechanical success, -text/error/failure detail, optional usage and duration, harness run ID, explicit -harness output/run-record paths, trace manifest path, and attempt ID. -`result_text.txt` and `harness_run_id.txt` are convenient projections. - -`success: true` means the harness call returned normally. It is not semantic -acceptance and is not inferred from a spawned agent's exit status (D3). - -### `pending_finished_request.json` - -The exact HTTP completion body, written atomically after result artifacts and -before `/finished`. It is removed only after acknowledgement. If the worker -exits in that window, the next `/register` validates it against the active -attempt and records completion exactly once. With no pending body, a matching -`result.json` can be used to reconstruct the handoff. - -### `trace_ref.json` - -A compact link from the durable iteration to `trace::/...` for the -attempt. v2 session history records the same relocatable logical reference; -only legacy history retains the absolute manifest path supplied by an older -worker. Accepted v2 history also records the frozen assignment hash and hashes -of the exact `/finished` request and returned response. -Crash-abandonment history records the assignment hash without a finished -exchange. - -### `salvage.json` +```text +layer_goal layer_goal_contract +layer_inputs layer_plan +layer_tasks layer_current_state +layer_decisions layer_finished_ledger +layer_eval_state layer_handoff +session_state session_outcome +workflow_contract workflow_roster +scheduler_view harness_capability_roster +user_inputs child_requests +children_index child_outcomes +parent_acceptance git_receipts +delivery_receipts session_control +attempt_root trace_root +``` -Written only when crash recovery processes at least one tracked harness run. -It records policy and per-process drain/reap/skip reports. It does not fabricate -a result. If any process remains potentially live, the coordinator refuses -replacement work until the ambiguity is resolved. +The full map also retains operational aliases such as `repo_root`, +`session_root`, `eval_checks`, and `eval_receipts`. Stable origin keys +`parent_goal`, `parent_goal_contract`, `parent_handoff`, and +`accepted_child_request` are absolute paths for a child and explicit nulls when +inapplicable. `layer_inputs` always means this layer's own immutable input +directory. + +The assignment's `context` embeds compact copies of the workflow roster, +scheduler view, capability roster, and current layer identity; `provenance` +hashes their canonical files. The rendered prompt prints the authoritative +assignment path and every named absolute path. Agents must use those values +rather than infer `.loopy_loop/sessions/...` from cwd or directory names. + +Absolute paths are execution-time coordinates for this checkout. Durable +cross-file evidence uses logical references so a completed tree remains +inspectable after relocation. Ownership guidance is not implemented as a +filesystem ACL (D8). + +### `scheduler_view.json` + +This attempt-frozen artifact shows recent mechanical history and a conditional +next-role forecast. The coordinator simulates the current attempt returning as +a mechanical success and runs the unchanged scheduler against that projected +history. The view records the possible next workflow, reasons, and assumptions +that no terminal control, child request, failure, update, stop, or recovery +changes state. + +It is not a reservation. Its purpose is to let an orchestrator reason about an +imminent scheduled evaluator or reviewer without falsely promising what will +happen after the current attempt. + +## Optional evaluation evidence + +`eval_checks/` contains session-scoped check definitions. The stock +`inner_outer_eval` check authors use outcome-oriented `harness_judge` checks; +repo-owned test suites and prepared evaluations may provide additional +evidence. `eval_receipts/` contains compact identity- and hash-bound verdicts. +Raw reports remain under the producing attempt's trace `eval/` directory. + +In protocol v3 evaluation is advisory: + +- a missing, malformed, or non-passing observation does not turn a normally + returned harness result into failure, consume the workflow failure budget, + or prevent the orchestrator from receiving another turn; +- any declared `evaluation.check_runner_roles` workflow may produce a receipt; + this is not hard-coded to a role named `eval_runner`; and +- `goal_check.json`, when present, is an optional/legacy iteration projection, + not a session stop switch. + +When an authorized runner finishes, `_accept_current_eval_receipts()` validates +the current-attempt receipt and its raw/canonical bytes, check definitions, +subject, producer, harness run, judge settings, and evaluated Git identity. +Each valid receipt gets an engine-owned sidecar in +`eval_receipts/accepted/.json`, and the same acceptance seal is stored +in `state.json`. The sidecar binds receipt reference/hash, subject, producer, +evaluated Git identity, and acceptance time. + +Later `control.json` may cite accepted receipts from earlier attempts. Control +validation checks the compact receipt against its seal and current +session/root/goal/authorized-runner identity; it does not need to reopen raw +trace bytes. Citing a stale, foreign, modified, or unaccepted receipt is a +false provenance claim and invalidates that control record. Merely having a +bad advisory receipt does not. + +## Terminal control and topology-neutral outcome + +`control.json` begins as a neutral running record. In protocol v3 a terminal +record must identify the exact current session/workflow/attempt. `goal_met` +must come from the contract's `orchestration.completion_role`; it may cite +logical evidence, zero or more accepted eval receipts, and the canonical +handoff. No eval receipt, passing verdict, or same-attempt eval is required. + +`unresolvable_error` is the D5 last resort. It must come from an authorized +role and list autonomous routes already tried. There is no paused or +waiting-for-human state. The exact schemas and rejection behavior are in +[`http-contract.md`](./http-contract.md#protocol-v3-terminal-control). + +After any v3 terminal transition, `_ensure_session_outcome()` writes +`session_outcome.json`. This includes orchestrator control (`goal_met` or +`unresolvable_error`) and engine lifecycle stops such as `max_turns`, +`workflow_failure_cap`, or `stop_requested`. The projection has the same shape +for a root or child and binds: + +- session/root/goal identity, terminal status, and the frozen transition + revision/timestamp; +- stop reason and, for control-owned stops, the exact accepted control hash; +- handoff status (`valid`, `missing`, `invalid`, or `non_monotonic`) plus its + reference/hash/revision when available; +- the exact control reason or factual engine stop reason as fallback when no + valid handoff exists; and +- evidence, session-wide delivery, accepted eval, and trace-seal references. + +The coordinator stores accepted terminal-control and valid-handoff bytes in +`state.json`. Outcome regeneration restores those exact bytes if mutable files +were later changed, so restart or trace-finalization refreshes cannot rewrite +the terminal basis. Trace-seal references may grow as asynchronous trace +finalization completes; terminal identity, control, and handoff do not. + +A v3 parent's `child_outcomes/.json` does not synthesize a second +story. It contains the request/child identity and a logical reference plus hash +of that child's `session_outcome.json`. The parent orchestrator separately +records `accepted`, `rework`, or `reroute` in `parent_acceptance/`; a child's +`goal_met` never completes its parent. + +Invalid terminal control is moved unchanged into `control_rejected/`, with a +reasoned compact record in `protocol_failures/`. The engine restores a running +placeholder so a later attempt can repair the protocol. This enforces identity, +schema, containment, and truthful provenance, not semantic sufficiency. + +## Child request and acceptance records + +A recursive workflow atomically publishes a schema-v2 request under +`child_requests/pending/`. It supplies a unique request ID, exact origin +attempt, child workflow set, scoped outcome goal and criteria, and optional +hashed input references. A valid request is copied unchanged into +`accepted/`, indexed in revisioned `children.json`, and frozen into the child. +An invalid request is archived in `rejected/` with its original hash and +diagnosis. This is autonomous disposition, not a human approval gate. + +The stock PM planner selects a phase- or milestone-sized outcome; the +dispatcher faithfully transports it. The child `outer` owns decomposition into +leaves. `parent_acceptance/` remains a distinct parent-owned semantic decision +after the child result arrives. + +## Git, delivery, and trace receipts + +The worker writes compact Git boundary receipts before and after every modern +attempt. Verbose status and diff bytes stay in traces. Workflow roles may add +delivery receipts for branches, PRs, merges, or other declared deliverables. +`project_state/finished.md` should link accepted work to these receipts. + +Trace-seal receipts under `trace_seals/` bind the final attempt manifest and +inventory without copying raw traces into durable state. Trace finalization is +best effort with a durable outbox; failure to capture a trace does not roll back +an accepted semantic transition. ## Caller-owned attempt traces -For a v2 task, the coordinator creates the active trace during dispatch and -the worker reopens it: - ```text .loopy_loop/traces//sessions//attempts// ├── trace_manifest.json @@ -462,109 +424,33 @@ the worker reopens it: │ ├── rendered_prompt.txt │ ├── iteration_result.json │ ├── finished_request.json -│ └── finished_response.json # only when observed -├── harness/ -│ └── / -│ ├── run.json -│ ├── coordinator_input.json -│ ├── worker_sessions.json -│ ├── agents//agent_assignment.json -│ └── workers/