diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index fada226..5695248 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -20,7 +20,7 @@ "url": "https://github.com/choiyounggi/dev-loop.git" }, "homepage": "https://github.com/choiyounggi/dev-loop", - "version": "1.0.0", + "version": "1.1.0", "tags": [ "orchestrator", "verification-loop", diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index a18b720..3c3aecc 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "dev-loop", "description": "loop-orchestrator's verification loop (TDD / PDCA / Reflexion) with the plan step FIXED to a wiki-grounded planning methodology (wiki-plan): every design decision is routed to a bundled semantic-layer wiki before code is written. Adds a knowledge-capture loop — sessions emit verified insights, and knowledge-flush researches, dedups, routes, and opens a wiki PR for owner review.", - "version": "1.0.0", + "version": "1.1.0", "author": { "name": "choiyounggi", "url": "https://github.com/choiyounggi" diff --git a/skills/orchestrate/SKILL.md b/skills/orchestrate/SKILL.md index 8d027d1..026968d 100644 --- a/skills/orchestrate/SKILL.md +++ b/skills/orchestrate/SKILL.md @@ -13,7 +13,8 @@ implementation loop; two human gates bracket it (task-split, pre-merge). Scripts referenced below live in `${CLAUDE_PLUGIN_ROOT}/skills/orchestrate/scripts/`. Communication: session→orchestrator via `.orchestration/status/.json`; -orchestrator→session via `tmux send-keys` one-liners (templates/session-prompt.md). +orchestrator→session via `tmux send-keys` one-liners (templates/session-prompt.md +§1–§4) on tmux, or the Task `--spec` (same file, §O1–§O4) on Orca. ## Tool profile Resolve the pluggable tool profile once up front: @@ -120,9 +121,15 @@ to you instead of making you poll. Replace steps 1–3 below with O1–O5: `orca orchestration run-create --objective "" --json` → keep `run_id` for the whole run (on re-entry, `orca orchestration run-use --id --json` instead). - **O2 — one Task per task-*phase*.** - `orca orchestration task-create --spec "" --json` - → `task_id`. A `worker_done` settles a Task exactly once, so plan / implement / - rework are separate Tasks (chain with `--deps` when one must follow another). + `orca orchestration task-create --spec "" --task-title "" --json` + → `task_id`. Use `templates/session-prompt.md` §O1–§O4 as the `--spec` body; the + tmux §1–§4 one-liners are for `send-keys` and say "wait", which is wrong here. + A `worker_done` settles a Task exactly once, so plan / implement / rework are + separate Tasks. **You sequence them yourself** — dispatch phase N+1 only after + phase N reads `completed`. Do *not* chain with `--deps`: it takes a JSON array + (`--deps '["task_x"]'`; a bare id errors), and even then a task created that way + came back `task_not_startable` with its dependency already `completed` + (isolated: an identical task with no deps started fine on the same terminal). - **O3 — start the worker.** First make sure the integration branch exists in the main repo (`git branch ` if it does not) — the tmux path gets this from `setup-worktrees.sh` step 1, this path does not. Then create the Orca @@ -152,22 +159,49 @@ to you instead of making you poll. Replace steps 1–3 below with O1–O5: `orca terminal list --worktree id:<...> --json` that the extra handle is an idle shell — not a configured default tab — and close just that one with `orca terminal close --terminal --json`. + A `worker-start` that fails **spends the Task**: it goes to `status=failed` and + every later attempt on it returns `task_not_startable`. Do not retry the same + Task — create a fresh one with the same spec. (Seen with `runtime_unavailable`, + which is what you get when that terminal is still busy with another Dispatch.) - **O4 — wait on pushed mail, not on a timer.** - `GROUNDWORK_ESCALATION_DIR= scripts/orca-wait.sh + `GROUNDWORK_ESCALATION_DIR= scripts/orca-wait.sh [--until-all] []` → **0** completions arrived (acked — process them), **2** window elapsed *or* the ack did not land (checkpoint, just re-run), **3** a worker reported failure, **5** escalation pending (approve/deny, **clear `.orchestration/escalations/`**, then re-run — like watch-status, code 5 recurs while a record is still on disk, by design), **6** question pending (`orca orchestration reply --id --body "" --json`, re-run). - Pass this Wave's task ids to get a `completed=/` line scoped to this Wave; - without them, an earlier Wave's completed Tasks would inflate the count. Repeat - until every Task of this Wave is `completed`. Codes 3/5/6 leave the batch unread - on purpose, so an unhandled event is never silently dropped — which also means + **Always pass this Wave's task ids — that is correctness, not a progress + nicety.** Heartbeats reach this mailbox despite `--types`, and so do completions + from an earlier Wave or an unrelated Task; without the ids none of that can be + told apart from your own work. With them, exit 0 means a *successful + `worker_done` for one of your ids* and the `completed=/` line is scoped to + this Wave. Add `--until-all` to keep consuming batches until every listed id is + completed, so one Wave costs one coordinator turn instead of one per batch; it + still returns immediately on 3/5/6. Codes 3/5/6 leave the batch unread on + purpose, so an unhandled event is never silently dropped — which also means delivery is **at-least-once**: a replayed batch must be processed idempotently - (key off `taskId`, never off a local counter). -- **O5 — liveness.** `scripts/orca-worktree-alive.sh ` (0 alive / - 1 dead / 2 unknown — treat unknown as *not* dead) replaces watch's tmux check. + (key off `taskId`, never off a local counter). `ORCA_WAIT_RECHECK_MS` (default + 15000) slices the wait so a guardrails record written *while* you are blocked + surfaces within one interval instead of one full ``. +- **O5 — liveness, in two questions.** `scripts/orca-worktree-alive.sh ` + (0 alive / 1 dead / 2 unknown — treat unknown as *not* dead) replaces watch's + tmux check. It only asks whether a terminal is attached, which a wedged worker + passes: measured, three workers sat on an interactive CLI prompt for 75 minutes, + ALIVE the whole time, Tasks still `dispatched`, no escalation, diffs unchanged. + So also run `scripts/orca-worker-stalled.sh ` (0 progressing / 1 stalled / + 2 unknown — treat unknown as *not* stalled; `ORCA_STALL_MS`, default 600000). + A stall is not a failure to act on blindly: read the worker's screen + (`orca orchestration worker-read --dispatch --limit 40 --json`) before you + decide, because "wedged on a prompt" and "finished but never reported" look the + same from the outside and need opposite responses. + +**You cannot steer a running worker.** `orchestration send --to dispatch:` +lands in the worker's mailbox, which a Claude worker never polls, and a new +Dispatch cannot be delivered while the current one runs (that is O2's +`runtime_unavailable`). So a mid-flight correction does not arrive: the supported +path is the Phase-4 review/rework round. (If you do send one, omit `--type` — +`note`/`guidance`/`message`/`info` are all rejected as `invalid_argument`.) **Worker protocol on this substrate — put this in every `--spec` you dispatch.** The worker still calls `status-update.sh` at each phase (the status files remain the @@ -211,7 +245,8 @@ reasoning-effort flags) that `worker-start` cannot express. a UI-facing task fill `` with the `design` role's pulled spec (Phase 2) — then `scripts/launch-session.sh lo- bypassPermissions ""` - (plan prompt = templates/session-prompt.md §1, with the subagent protocol block). + (plan prompt = templates/session-prompt.md §1 — the tmux set — with the + subagent protocol block). 3. `scripts/watch-status.sh plan_ready ` in the background; when it exits, collect `plans/.md`. *(Orca substrate: `scripts/orca-wait.sh ` per O4 instead — same exit-code contract, @@ -254,8 +289,13 @@ artifacts exist. Resume from the earliest incomplete step (idempotently skip don steps). Check `tmux ls`; relaunch dead sessions and re-inject the right prompt. On the Orca substrate, rebind the Run first (`orca orchestration run-use --id --json`), then measure with `orca orchestration task-list --json` + -`scripts/orca-worktree-alive.sh `; restart a proven-dead worker with -`scripts/orca-worker-start.sh --task --worktree id:<...> --agent claude`. +`scripts/orca-worktree-alive.sh ` **and** `scripts/orca-worker-stalled.sh ` +— a Task reading `dispatched` proves only that it was handed out, never that the +worker is moving. Restart a proven-dead worker with a NEW Task (the old one is +spent, see O2) via `scripts/orca-worker-start.sh --task --worktree +id:<...> --agent claude`; that call now probes first and rebinds to a live agent +terminal on that worktree instead of creating a second one, so re-running it is +safe (it exits 6 rather than guess when it cannot tell). `setup-worktrees.sh` is idempotent (existing branches/worktrees are detected and kept), so re-running it is safe. Note the difference from **partial resume** (Phase 0): that handles work done *outside* this orchestration — children with no diff --git a/skills/orchestrate/scripts/orca-wait.sh b/skills/orchestrate/scripts/orca-wait.sh index f221265..a2ead1c 100755 --- a/skills/orchestrate/scripts/orca-wait.sh +++ b/skills/orchestrate/scripts/orca-wait.sh @@ -1,18 +1,25 @@ #!/bin/sh -# orca-wait.sh — block on the Orca coordinator mailbox for ONE Delivery and -# classify it into watch-status.sh's exit-code contract. This is the Orca- +# orca-wait.sh — block on the Orca coordinator mailbox for ONE Delivery (or, with +# --until-all, for as many as this Wave needs) and classify it into +# watch-status.sh's exit-code contract. This is the Orca- # substrate replacement for polling `.orchestration/status/*.json` every 15s: # workers push `worker_done` / `escalation` / `question`, so the coordinator # wakes on the event instead of on a timer (measured: question surfaced in 2.3s). # -# usage: orca-wait.sh [task_id,task_id,...] -# how long this window blocks (real tasks run 15-60 min) +# usage: orca-wait.sh [--until-all] [task_id,task_id,...] +# --until-all keep consuming successful worker_done batches until every task +# id listed below is completed, then exit 0 once. Requires the +# task id list. Exit 3/5/6 still return immediately, unacked. +# total budget for this call (real tasks run 15-60 min); with +# --until-all it covers ALL batches and is never reset per batch # [task ids] if given, also report how many of exactly THOSE tasks are # completed — scoping the progress line to this Wave instead of # to every task the Run has ever completed # # exit 0 worker_done(s) received, all succeeded, Delivery acked — process them -# exit 2 no message in this window, or the ack did not land — CHECKPOINT: re-run +# exit 2 no completion in this call, or the ack did not land — CHECKPOINT: re-run +# (--until-all: some tasks are still running; batches already acked in +# this call are consumed and are not replayed) # exit 3 a worker reported outcome=failed (or an unprovable worker_done) # exit 5 an escalation is pending — resolve it, then re-run # exit 6 a worker question is pending — `orchestration reply --id `, re-run @@ -22,13 +29,25 @@ # dropped" property watch-status.sh's exit 5 has. Orca delivery is at-least-once, # so the coordinator must treat a replayed batch idempotently. # -# GROUNDWORK_ESCALATION_DIR is checked BEFORE blocking: guardrails writes an -# escalation record from inside the worker regardless of whether the worker -# survives to send the matching Orca message, so this file check is the safety -# net that keeps a stuck worker from costing a whole timeout window. +# A Delivery carrying NO worker_done (heartbeats reach this mailbox even though +# --types excludes them) is acked but never reported as exit 0 — it is consumed +# and the wait continues. Acking it is deliberate: liveness needs no decision, so +# replaying it forever would livelock the wait, while exit 0 would send the +# coordinator after completions that never happened. +# +# GROUNDWORK_ESCALATION_DIR is checked before EVERY blocking window: guardrails +# writes an escalation record from inside the worker regardless of whether the +# worker survives to send the matching Orca message, so this file check is the +# safety net that keeps a stuck worker from costing a whole timeout window. The +# record can land at any moment, including while this script is already blocked, +# so re-checking between windows is what bounds the detection delay to one +# interval instead of one full . # # env (also test hooks): -# GROUNDWORK_ESCALATION_DIR guardrails escalation sink to pre-check (optional) +# GROUNDWORK_ESCALATION_DIR guardrails escalation sink to check (optional) +# ORCA_WAIT_RECHECK_MS escalation re-check interval in ms (default 15000, +# matching watch-status.sh's 15s poll); the total +# is split into windows of this size # ORCA_BIN orca executable (default: orca) # ORCA_WAIT_DRYRUN print the ack instead of sending it # ORCA_WAIT_CHECK_JSON canned `check --wait --json` Delivery (tests) @@ -39,17 +58,43 @@ set -u ORCA="${ORCA_BIN:-orca}" JQ=$(command -v jq) || { echo "orca-wait: jq not found" >&2; exit 127; } +usage() { echo "usage: orca-wait.sh [--until-all] [task_id,...]" >&2; exit 1; } + +until_all=0 +while [ $# -gt 0 ]; do + case "$1" in + --until-all) until_all=1; shift ;; + --) shift; break ;; + -*) usage ;; + *) break ;; + esac +done + tmo="${1:-}"; want_ids="${2:-}" -case "$tmo" in ''|*[!0-9]*) echo "usage: orca-wait.sh [task_id,...]" >&2; exit 1 ;; esac +case "$tmo" in ''|*[!0-9]*) usage ;; esac case "$want_ids" in '') : ;; *[!A-Za-z0-9_,-]*) echo "orca-wait: task id list must be comma-separated ids" >&2; exit 1 ;; esac +# "all of" needs a set to be all of; without ids there is no termination condition. +if [ "$until_all" = 1 ] && [ -z "$want_ids" ]; then + echo "orca-wait: --until-all needs the task id list to be 'all' of" >&2 + exit 1 +fi + +# The blocking window is sliced this small so the escalation dir is re-read often. +# Rejected rather than clamped when unusable: a 0 interval would spin forever, and +# `00` passes the digit test, so the arithmetic test below is not redundant. +recheck="${ORCA_WAIT_RECHECK_MS:-15000}" +bad_recheck() { echo "orca-wait: ORCA_WAIT_RECHECK_MS must be a positive integer (ms)" >&2; exit 1; } +case "$recheck" in ''|*[!0-9]*) bad_recheck ;; esac +[ "$recheck" -gt 0 ] || bad_recheck # 1) guardrails escalations first — a worker blocked on an `ask` must not cost a # full window, and this path does not depend on the worker staying alive. -escdir="${GROUNDWORK_ESCALATION_DIR:-}" -if [ -n "$escdir" ] && [ -d "$escdir" ]; then +esc_scan() { # 0 = pending (message printed), 1 = none + escdir="${GROUNDWORK_ESCALATION_DIR:-}" + [ -n "$escdir" ] && [ -d "$escdir" ] || return 1 pending="" for e in "$escdir"/*.json; do [ -f "$e" ] || continue @@ -57,98 +102,139 @@ if [ -n "$escdir" ] && [ -d "$escdir" ]; then erule=$("$JQ" -r '.rule // "?"' "$e" 2>/dev/null || echo "?") pending="$pending ${etk}:${erule}" done - if [ -n "$pending" ]; then - echo "[orca-wait] guardrails escalation pending —$pending — approve/deny and clear $escdir" - exit 5 - fi -fi - -# 2) block for one Delivery -if [ -n "${ORCA_WAIT_CHECK_JSON:-}" ]; then - out="$ORCA_WAIT_CHECK_JSON" -else - out=$("$ORCA" orchestration check --wait \ - --types worker_done,escalation,question --timeout-ms "$tmo" --json 2>/dev/null) || out="" -fi + [ -n "$pending" ] || return 1 + echo "[orca-wait] guardrails escalation pending —$pending — approve/deny and clear $escdir" + return 0 +} # A payload arrives as a JSON *string* on the wire, but tolerate an object (or # nothing) so a shape change can never read as success. PAYLOAD='(.payload | if type=="string" then (fromjson? // {}) elif type=="object" then . else {} end)' -count=$(printf '%s' "$out" | "$JQ" -r '.result.count // 0' 2>/dev/null) || count=0 -case "$count" in ''|*[!0-9]*) count=0 ;; esac -if [ "$count" -lt 1 ]; then - echo "[orca-wait] no message in ${tmo}ms — checkpoint, keep waiting" - exit 2 -fi +# 2) block for one Delivery, in windows of $recheck rather than one long wait, so +# an escalation written mid-wait is seen within one interval. The windows sum +# to exactly $tmo: the caller's deadline is neither extended nor truncated. +remaining="$tmo"; completed=0; total=0 +while :; do + if esc_scan; then exit 5; fi + if [ "$remaining" -le 0 ]; then + echo "[orca-wait] no message in ${tmo}ms — checkpoint, keep waiting" + exit 2 + fi + slice="$recheck" + [ "$slice" -gt "$remaining" ] && slice="$remaining" -delivery=$(printf '%s' "$out" | "$JQ" -r '.result.deliveryId // empty' 2>/dev/null) + if [ -n "${ORCA_WAIT_CHECK_JSON:-}" ]; then + out="$ORCA_WAIT_CHECK_JSON" + else + out=$("$ORCA" orchestration check --wait \ + --types worker_done,escalation,question --timeout-ms "$slice" --json 2>/dev/null) || out="" + fi + remaining=$((remaining - slice)) -printf '%s' "$out" | "$JQ" -r ' - .result.messages[]? | . as $m | '"$PAYLOAD"' as $p | - "msg type=\($m.type // "?") id=\($m.id // "-") from=\($m.from_handle // "-") task=\($p.taskId // "-") outcome=\($p.outcome // "-") files=\(($p.filesModified // []) | join(",")) :: \($m.subject // "")" -' 2>/dev/null + count=$(printf '%s' "$out" | "$JQ" -r '.result.count // 0' 2>/dev/null) || count=0 + case "$count" in ''|*[!0-9]*) count=0 ;; esac + [ "$count" -lt 1 ] && continue -n_of() { printf '%s' "$out" | "$JQ" -r "[.result.messages[]? | select(.type==\"$1\")] | length" 2>/dev/null || echo 0; } -esc=$(n_of escalation); q=$(n_of question) -bad=$(printf '%s' "$out" | "$JQ" -r ' - [.result.messages[]? | select(.type=="worker_done") | '"$PAYLOAD"' | select((.outcome // "") != "succeeded")] | length -' 2>/dev/null) || bad=0 -for v in esc q bad; do - eval "val=\$$v"; case "$val" in ''|*[!0-9]*) eval "$v=0" ;; esac -done + delivery=$(printf '%s' "$out" | "$JQ" -r '.result.deliveryId // empty' 2>/dev/null) -if [ "$esc" -gt 0 ]; then - echo "[orca-wait] escalation pending — approve/deny, then re-run (left unread: ${delivery:--})" - exit 5 -fi -if [ "$q" -gt 0 ]; then - printf '%s' "$out" | "$JQ" -r '.result.messages[]? | select(.type=="question") - | "[orca-wait] next: orca orchestration reply --id \(.id) --body \"\" --json"' 2>/dev/null - echo "[orca-wait] question pending — answer it, then re-run (left unread: ${delivery:--})" - exit 6 -fi -if [ "$bad" -gt 0 ]; then - echo "[orca-wait] a worker reported failure (or an unprovable completion) — left unread: ${delivery:--}" - exit 3 -fi + printf '%s' "$out" | "$JQ" -r ' + .result.messages[]? | . as $m | '"$PAYLOAD"' as $p | + "msg type=\($m.type // "?") id=\($m.id // "-") from=\($m.from_handle // "-") task=\($p.taskId // "-") outcome=\($p.outcome // "-") files=\(($p.filesModified // []) | join(",")) :: \($m.subject // "")" + ' 2>/dev/null -# Everything in this batch was a successful completion: consume it so the next -# window returns the following batch instead of replaying this one. If the ack -# does not land, this batch WILL replay, so report the checkpoint code rather -# than exit 0 — exit 0 must mean "consumed", or the coordinator double-counts. -if [ -n "$delivery" ]; then - acked=0 - if [ -n "${ORCA_WAIT_DRYRUN:-}" ]; then - [ -z "${ORCA_WAIT_ACK_FAIL:-}" ] && acked=1 - elif "$ORCA" orchestration check --ack "$delivery" --json >/dev/null 2>&1; then - acked=1 + n_of() { printf '%s' "$out" | "$JQ" -r "[.result.messages[]? | select(.type==\"$1\")] | length" 2>/dev/null || echo 0; } + esc=$(n_of escalation); q=$(n_of question); dones=$(n_of worker_done) + # How many of this batch's completions are ours. A worker_done belonging to an + # EARLIER Wave (or to an unrelated task on the same Run) is delivered here too, + # so "the batch had a worker_done" is not "this Wave progressed" — measured live: + # a batch of one previous-Wave plan completion + one unrelated probe completion + + # a heartbeat returned exit 0 while all three of this Wave's tasks still ran. + # Without ids we cannot judge relevance, so every completion counts (as before). + if [ -n "$want_ids" ]; then + dones=$(printf '%s' "$out" | "$JQ" -r --arg ids "$want_ids" ' + ($ids | split(",") | map(select(length > 0))) as $want | + [.result.messages[]? | select(.type=="worker_done") | '"$PAYLOAD"' + | select((.outcome // "") == "succeeded") + | select(.taskId as $t | $want | index($t))] | length' 2>/dev/null) || dones=0 fi - if [ "$acked" = 1 ]; then - echo "ack $delivery" - else - echo "[orca-wait] could not consume $delivery — it will replay; treat this window as a checkpoint" >&2 - exit 2 + bad=$(printf '%s' "$out" | "$JQ" -r ' + [.result.messages[]? | select(.type=="worker_done") | '"$PAYLOAD"' | select((.outcome // "") != "succeeded")] | length + ' 2>/dev/null) || bad=0 + for v in esc q bad dones; do + eval "val=\$$v"; case "$val" in ''|*[!0-9]*) eval "$v=0" ;; esac + done + + if [ "$esc" -gt 0 ]; then + echo "[orca-wait] escalation pending — approve/deny, then re-run (left unread: ${delivery:--})" + exit 5 + fi + if [ "$q" -gt 0 ]; then + printf '%s' "$out" | "$JQ" -r '.result.messages[]? | select(.type=="question") + | "[orca-wait] next: orca orchestration reply --id \(.id) --body \"\" --json"' 2>/dev/null + echo "[orca-wait] question pending — answer it, then re-run (left unread: ${delivery:--})" + exit 6 + fi + if [ "$bad" -gt 0 ]; then + echo "[orca-wait] a worker reported failure (or an unprovable completion) — left unread: ${delivery:--}" + exit 3 fi -fi -# Progress, scoped to the task ids this Wave actually dispatched. Counting every -# completed task in the Run would fold in earlier Waves and overstate progress. -if [ -n "$want_ids" ]; then - if [ -n "${ORCA_WAIT_TASKLIST_JSON:-}" ]; then - tl="$ORCA_WAIT_TASKLIST_JSON" - else - tl=$("$ORCA" orchestration task-list --status completed --json 2>/dev/null) || tl="" + # Nothing in this batch needs a decision: consume it so the next window returns + # the following batch instead of replaying this one. If the ack does not land, + # this batch WILL replay, so report the checkpoint code rather than exit 0 — + # exit 0 must mean "consumed", or the coordinator double-counts. + if [ -n "$delivery" ]; then + acked=0 + if [ -n "${ORCA_WAIT_DRYRUN:-}" ]; then + [ -z "${ORCA_WAIT_ACK_FAIL:-}" ] && acked=1 + elif "$ORCA" orchestration check --ack "$delivery" --json >/dev/null 2>&1; then + acked=1 + fi + if [ "$acked" = 1 ]; then + echo "ack $delivery" + else + echo "[orca-wait] could not consume $delivery — it will replay; treat this window as a checkpoint" >&2 + exit 2 + fi fi - # `index` is exact membership. `inside` would be substring containment - # (["task_1"] | inside(["task_12"]) is true), so a shorter id from another Wave - # would count here — the exact inflation this scoping exists to prevent. - completed=$(printf '%s' "$tl" | "$JQ" -r --arg ids "$want_ids" ' - ($ids | split(",") | map(select(length > 0))) as $want | - [.result.tasks[]? | select((.status // "completed") == "completed") - | select(.id as $i | $want | index($i))] | length' 2>/dev/null) || completed=0 - case "$completed" in ''|*[!0-9]*) completed=0 ;; esac - total=$(printf '%s' "$want_ids" | tr ',' '\n' | grep -c '[^[:space:]]') - echo "completed=$completed/$total" -fi -exit 0 + + # A Delivery can arrive carrying no worker_done at all — heartbeats reach this + # mailbox even though --types asks only for worker_done/escalation/question. Exit + # 0 means "completions arrived, process them", so returning it here sends the + # coordinator after artifacts nobody produced. The batch is already acked above: + # liveness needs no decision, and replaying it forever would livelock the wait. + # Treat it as an empty window that happened to carry noise and keep waiting. + if [ "$dones" -lt 1 ]; then + echo "[orca-wait] batch carried no completion for these task ids — kept waiting" + continue + fi + + # Progress, scoped to the task ids this Wave actually dispatched. Counting every + # completed task in the Run would fold in earlier Waves and overstate progress. + if [ -n "$want_ids" ]; then + if [ -n "${ORCA_WAIT_TASKLIST_JSON:-}" ]; then + tl="$ORCA_WAIT_TASKLIST_JSON" + else + tl=$("$ORCA" orchestration task-list --status completed --json 2>/dev/null) || tl="" + fi + # `index` is exact membership. `inside` would be substring containment + # (["task_1"] | inside(["task_12"]) is true), so a shorter id from another Wave + # would count here — the exact inflation this scoping exists to prevent. + completed=$(printf '%s' "$tl" | "$JQ" -r --arg ids "$want_ids" ' + ($ids | split(",") | map(select(length > 0))) as $want | + [.result.tasks[]? | select((.status // "completed") == "completed") + | select(.id as $i | $want | index($i))] | length' 2>/dev/null) || completed=0 + case "$completed" in ''|*[!0-9]*) completed=0 ;; esac + total=$(printf '%s' "$want_ids" | tr ',' '\n' | grep -c '[^[:space:]]') + echo "completed=$completed/$total" + fi + + # --until-all: this batch is consumed and acked, but the Wave is not finished + # until every requested task id is completed. Keep waiting on the SAME budget — + # termination keys off actual completions, never off "a batch arrived". + if [ "$until_all" = 1 ] && [ "$completed" -lt "$total" ]; then + continue + fi + exit 0 +done diff --git a/skills/orchestrate/scripts/orca-worker-stalled.sh b/skills/orchestrate/scripts/orca-worker-stalled.sh new file mode 100755 index 0000000..b7808b2 --- /dev/null +++ b/skills/orchestrate/scripts/orca-worker-stalled.sh @@ -0,0 +1,102 @@ +#!/bin/sh +# orca-worker-stalled.sh — is an ALIVE worker actually moving? +# +# orca-worktree-alive.sh answers "is a terminal attached", which is not the same +# question. A worker can hold a live PTY and produce nothing for hours — a Claude +# CLI sitting on an interactive prompt (a usage-limit chooser, a trust screen, an +# auth re-login) is alive, its Orca Task still reads `dispatched`, its +# `worker`/`terminal` status still read `ready`/`running`, and orca-wait.sh just +# keeps checkpointing. Measured: three workers held that state for 75 minutes +# with byte-identical diffs, and nothing in the alive/dead split could see it. +# This script supplies the missing third state. +# +# Signal: `orca worktree ps --json` reports, per worktree, an `agents[]` array +# carrying each agent's own `state` and `updatedAt` (epoch ms). Silence in the +# AGENT's state longer than the threshold, while a terminal is still live, is a +# stall. +# +# Not `lastOutputAt` — measured and rejected. That field tracks terminal writes, +# and a TUI agent repaints its spinner continuously, so a worker wedged on an +# interactive prompt still reports output "0 seconds ago" (in practice a +# fraction of a second in the future, because the field is millisecond-precision +# while `date +%s` is not). It cannot distinguish work from a redraw. +# +# exit 0 progressing — the agent moved within the threshold (or no live +# terminal at all — that is orca-worktree-alive.sh’s question) +# exit 1 STALLED — a live terminal whose agent has not moved for >= threshold +# exit 2 cannot tell (orca down / query failed / worktree untracked / no +# timestamp) — the caller MUST treat unknown as "not stalled", never +# kill a worker on it +# +# env (also test hooks): +# ORCA_STALL_MS silence threshold in ms (default 600000 = 10 min) +# ORCA_BIN orca executable (default: orca) +# ORCA_WORKTREE_PS_JSON canned `worktree ps` output (tests) +# ORCA_STALL_NOW_MS override "now" in epoch ms (tests) +set -u + +ORCA="${ORCA_BIN:-orca}" +JQ=$(command -v jq) || exit 2 +wt="${1:-}" +[ -n "$wt" ] || { echo "usage: orca-worker-stalled.sh " >&2; exit 2; } + +thr="${ORCA_STALL_MS:-600000}" +case "$thr" in ''|*[!0-9]*) exit 2 ;; esac +[ "$thr" -gt 0 ] || exit 2 + +now="${ORCA_STALL_NOW_MS:-}" +if [ -z "$now" ]; then + # seconds -> ms. `date +%s` is POSIX; multiplying keeps us off GNU-only %N. + now=$(date +%s 2>/dev/null) || exit 2 + case "$now" in ''|*[!0-9]*) exit 2 ;; esac + now=$((now * 1000)) +fi +case "$now" in ''|*[!0-9]*) exit 2 ;; esac + +if [ -n "${ORCA_WORKTREE_PS_JSON:-}" ]; then + ps="$ORCA_WORKTREE_PS_JSON" +else + ps=$("$ORCA" worktree ps --json /dev/null) || exit 2 +fi +[ -n "$ps" ] || exit 2 + +# Untracked worktree -> unknown, not "fine": it may simply be tmux-spawned, and +# claiming health for something we cannot see is the damaging direction. +present=$(printf '%s' "$ps" | "$JQ" -r --arg p "$wt" \ + '[.result.worktrees[]? | select(.path==$p)] | length' 2>/dev/null) || exit 2 +case "$present" in ''|*[!0-9]*) exit 2 ;; esac +[ "$present" -ge 1 ] || exit 2 + +live=$(printf '%s' "$ps" | "$JQ" -r --arg p "$wt" \ + '[.result.worktrees[]? | select(.path==$p) | (.liveTerminalCount>0 or .hasAttachedPty==true)] | any' 2>/dev/null) || exit 2 +# No live terminal is not a stall — that is the alive/dead question, and +# orca-worktree-alive.sh already answers it. Do not report both. +[ "$live" = "true" ] || exit 0 + +# Most recent agent activity on this worktree. No agents array at all means Orca +# is not tracking an agent here (a plain shell, or an older runtime) — unknown, +# not healthy. +last=$(printf '%s' "$ps" | "$JQ" -r --arg p "$wt" \ + '[.result.worktrees[]? | select(.path==$p) | .agents[]? | .updatedAt // empty] | max // empty' 2>/dev/null) +case "$last" in ''|*[!0-9]*) exit 2 ;; esac + +# `now` is floor-to-second while `updatedAt` is millisecond-precision, so a +# genuinely fresh timestamp can read as up to a second in the future. Treat a +# small overshoot as zero silence; a large one is real clock skew and unknown. +if [ "$last" -gt "$now" ]; then + [ $((last - now)) -le 5000 ] || exit 2 + silent=0 +else + silent=$((now - last)) +fi + +if [ "$silent" -ge "$thr" ]; then + # The agent's own state is advisory context, not the verdict — print it when + # Orca supplies it so the coordinator can tell "wedged on a prompt" from + # "finished but never reported". + st=$(printf '%s' "$ps" | "$JQ" -r --arg p "$wt" \ + '[.result.worktrees[]? | select(.path==$p) | (.agents[]? | .state) ] | join(",") // ""' 2>/dev/null || echo "") + echo "[stalled] $wt — agent idle for $((silent / 1000))s (threshold $((thr / 1000))s)${st:+ — agent state: $st}" + exit 1 +fi +exit 0 diff --git a/skills/orchestrate/scripts/orca-worker-start.sh b/skills/orchestrate/scripts/orca-worker-start.sh index 16c182e..e348766 100755 --- a/skills/orchestrate/scripts/orca-worker-start.sh +++ b/skills/orchestrate/scripts/orca-worker-start.sh @@ -12,6 +12,20 @@ # (same command shape as orca-spawn.sh), waits for `tui-idle`, and only then # binds the Dispatch with `worker-start --terminal`. # +# On re-entry this mode is idempotent, automatically — no flag to pass. +# Before creating anything it asks Orca whether this task's Dispatch already +# names a terminal that is still live on the worktree (`orchestration +# dispatch-show` joined against `terminal list --worktree`): +# live -> binds THAT terminal and creates nothing, saying so on +# stderr. --perm / --name are inert on this path: the agent +# is already running with the mode it was created with. +# not live -> unchanged: terminal create + tui-idle wait + bind. +# no record -> unchanged (this is the task's first start). +# can't tell-> exit 6, creates nothing, not retried automatically. +# To force a fresh agent, stop the old one first (`orca orchestration +# worker-stop`) and re-run. This is a check-then-create, so it protects a +# sequential re-entry — not two coordinators racing on one worktree. +# # REUSE MODE (--terminal ) # Bind this task's next phase (implement / rework) to the agent that already # holds the task's context. No new terminal, no env to re-inject. @@ -34,11 +48,15 @@ # ORCA_WORKER_START_DRYRUN print the orca commands instead of running them # ORCA_WORKER_START_JSON canned `worker-start --json` receipt (tests) # ORCA_WORKER_START_CREATE_JSON canned `terminal create --json` receipt (tests) +# ORCA_WORKER_START_DISPATCH_JSON canned `orchestration dispatch-show --json` (tests) +# ORCA_WORKER_START_TERMLIST_JSON canned `terminal list --json` (tests) # # On success prints: dispatch=, handle=, state=, setup= # Exit: 0 ok | 1 usage | 2 unsupported agent/placement for the escalation contract # | 3 receipt without a dispatch | 4 start failed (inspect stage/effects/ # residualResources; do NOT auto-retry) | 5 could not create the terminal +# | 6 could not determine whether a live agent terminal already exists — +# refused to create a second one (verify, then re-run or use --terminal) set -u ORCA="${ORCA_BIN:-orca}" @@ -129,8 +147,69 @@ dry="${ORCA_WORKER_START_DRYRUN:-}" rt="${LO_READY_TIMEOUT:-60}"; [ "$rt" -ge 1 ] 2>/dev/null || rt=60 -# --- worker mode: create the env-carrying agent terminal ourselves ------------ +reused="" + +# --- re-entry idempotency: never put a SECOND agent on one checkout ----------- +# SKILL.md's re-entry path restarts a worker the coordinator believes is dead by +# calling this script again with --worktree/--agent. If that liveness read was +# wrong, an unconditional `terminal create` lands a second agent on the same +# working tree, with the Dispatch bound to only one of them. So ask Orca first: +# does this task's Dispatch already name a terminal that is still live on this +# worktree? A probe that cannot answer is UNKNOWN, never "no agent" — creating +# the second agent is the damaging direction, so we refuse (exit 6). +# Both calls take `/dev/null) || dsp="" + fi + # ok:true with dispatch:null is a definite "no worker yet" (verified live); + # anything else means we could not read it — that is unknown, not none. + dsp_ok=$(printf '%s' "$dsp" | "$JQ" -r '.ok // false' 2>/dev/null || echo false) + if [ "$dsp_ok" != "true" ]; then + echo "orca-worker-start: cannot read the dispatch for '$task' — refusing to create a second agent on '$wt'; verify with \`orca orchestration dispatch-show --task $task\`, then re-run or bind the live terminal with --terminal" >&2 + exit 6 + fi + prev=$(printf '%s' "$dsp" | "$JQ" -r '.result.dispatch.assignee_handle // empty' 2>/dev/null) + + if [ -n "$prev" ]; then + # A recorded assignee is only a reuse target while it is STILL LIVE on the + # worktree we were asked to start on. + if [ -n "${ORCA_WORKER_START_TERMLIST_JSON:-}" ]; then + tl="$ORCA_WORKER_START_TERMLIST_JSON" + else + tl=$("$ORCA" terminal list --worktree "$wt" --json /dev/null) || tl="" + fi + tl_ok=$(printf '%s' "$tl" | "$JQ" -r '.ok // false' 2>/dev/null || echo false) + if [ "$tl_ok" != "true" ]; then + echo "orca-worker-start: cannot list terminals for '$wt' — refusing to create a second agent while '$prev' may still be live; verify with \`orca terminal list --worktree $wt\`" >&2 + exit 6 + fi + live=$(printf '%s' "$tl" | "$JQ" -r --arg h "$prev" \ + '[.result.terminals[]? | select(.handle==$h and .orphaned!=true and .connected==true)] | length' 2>/dev/null) + case "$live" in ''|*[!0-9]*) live=0 ;; esac + if [ "$live" -ge 1 ]; then + echo "orca-worker-start: worktree '$wt' already has a live agent terminal $prev — reusing it instead of creating a second agent (stop it first with \`orca orchestration worker-stop\` if you want a fresh one)" >&2 + term="$prev" + reused=1 + fi + fi + fi +fi + +# --- worker mode: create the env-carrying agent terminal ourselves ------------ +# (skipped when the probe above rebound us to the agent that is already there — +# the tui-idle wait below exists to pass a NEW CLI's trust screen, and a mid-task +# agent may never report idle) +if [ "$worker_mode" = 1 ] && [ -z "$reused" ]; then # Single-quote the values with embedded quotes escaped (`'\''`) so a path with # any metacharacter — including a quote — cannot break out of the command. esc_sq() { printf '%s' "$1" | sed "s/'/'\\\\''/g"; } diff --git a/skills/orchestrate/templates/session-prompt.md b/skills/orchestrate/templates/session-prompt.md index 16a7cbe..8bd915c 100644 --- a/skills/orchestrate/templates/session-prompt.md +++ b/skills/orchestrate/templates/session-prompt.md @@ -1,8 +1,19 @@ -# Session prompts — one line each, injected via `tmux send-keys -l` +# Session prompts — pick the set that matches your substrate -Substitute `{...}` then send as a SINGLE line (no newlines). Tokens: -`{TASK}` task id · `{STATUS_DIR}` abs path · `{SKILL}` orchestrate skill dir abs path · -`{INTEG}` integration branch · `{BRANCH}` this session's branch. +**tmux → §1–§4.** Injected via `tmux send-keys -l`: substitute `{...}`, then send as a +SINGLE line (no newlines). +**Orca → §O1–§O4.** NOT send-keys: each is the `--spec` body of one +`orca orchestration task-create`, one Task per task-phase, so it MAY span multiple +lines. Never tell an Orca worker to wait — it reports and ends its turn. + +Tokens: `{TASK}` task id · `{STATUS_DIR}` abs path · `{SKILL}` orchestrate skill dir abs path · +`{INTEG}` integration branch · `{BRANCH}` this session's branch · `{N}` rework round · +`{ORCA_TASK_ID}` Orca Task id from `task-create` (§O only) · +`{ORCA_DISPATCH_ID}` Orca Dispatch id from `orca-worker-start.sh` (§O only) — the +Dispatch is created *after* `task-create`, so on a task's first phase leave it +unsubstituted and the worker reads it from its own Orca dispatch context. + +**tmux substrate.** §1–§4 — one line each, `send-keys -l`. ## (1) Plan — injected at session launch @@ -20,6 +31,78 @@ Address the issues in .orchestration/reviews/{TASK}-r{N}.md via the loop-impleme Approved. Commit your changes on {BRANCH} with a conventional message (no push, no PR — the orchestrator merges into {INTEG} locally). Then run `STATUS_DIR={STATUS_DIR} sh {SKILL}/scripts/status-update.sh {TASK} done worktree=$PWD`. You may then stop. +**Orca substrate.** §O1–§O4 — one `--spec` per task-phase, delivered by Orca. Not +send-keys: a `--spec` MAY span multiple lines. Each phase is a separate Task, so no +prompt here says "wait" — the worker reports and ends its turn. + +## (O1) Plan — `--spec` of the plan Task + +You are the worker session for {TASK}. Treat .orchestration/briefs/{TASK}.md +`` as authority — especially ``, ``, and +``. Use the loop-implement skill but STOP after planning: run its +step 2 with the bundled `wiki-plan` skill (make every design decision grounded in a +`wiki/` page — record the decision->page map; leave nothing "as appropriate"), write the +resulting implementation plan to .orchestration/plans/{TASK}.md, then run +`STATUS_DIR={STATUS_DIR} sh {SKILL}/scripts/status-update.sh {TASK} plan_ready worktree=$PWD` +and report exactly once: +`orca orchestration send --type worker_done --subject "plan_ready: {TASK}" --body "" --task-id {ORCA_TASK_ID} --dispatch-id {ORCA_DISPATCH_ID} --outcome succeeded --files-modified ".orchestration/plans/{TASK}.md" --json` +(a failure is `--outcome failed`, never failure encoded only in prose). +Then END YOUR TURN. Do NOT write implementation code yet. + +## (O2) Implement — `--spec` of the implement Task + +Approved. Implement .orchestration/plans/{TASK}.md via the loop-implement skill: respect +`` (max 3 retries), write tests first, run them, self-review, and at step +6.5 you MUST call the test-quality-auditor subagent. Never touch anything in +``. Confirm EVERY `` item, then run +`STATUS_DIR={STATUS_DIR} sh {SKILL}/scripts/status-update.sh {TASK} impl_done worktree=$PWD` +and report exactly once: +`orca orchestration send --type worker_done --subject "impl_done: {TASK}" --body "" --task-id {ORCA_TASK_ID} --dispatch-id {ORCA_DISPATCH_ID} --outcome succeeded --files-modified "" --json` +(a failure is `--outcome failed`, never failure encoded only in prose). +Then END YOUR TURN. Do not commit, push, or PR — the orchestrator merges. + +## (O3) Rework — `--spec` of the rework Task + +Address the issues in .orchestration/reviews/{TASK}-r{N}.md via the loop-implement skill +(re-run step 6.5 audit; never weaken or skip tests). Then run +`STATUS_DIR={STATUS_DIR} sh {SKILL}/scripts/status-update.sh {TASK} impl_done worktree=$PWD` +and report exactly once: +`orca orchestration send --type worker_done --subject "impl_done: {TASK} r{N}" --body "" --task-id {ORCA_TASK_ID} --dispatch-id {ORCA_DISPATCH_ID} --outcome succeeded --files-modified "" --json` +(a failure is `--outcome failed`, never failure encoded only in prose). Then END YOUR TURN. + +## (O4) Merge-prep — `--spec` of the merge-prep Task + +Approved. Commit your changes on {BRANCH} with a conventional message (no push, no PR — +the orchestrator merges into {INTEG} locally). Then run +`STATUS_DIR={STATUS_DIR} sh {SKILL}/scripts/status-update.sh {TASK} done worktree=$PWD` +and report exactly once: +`orca orchestration send --type worker_done --subject "done: {TASK}" --body "" --task-id {ORCA_TASK_ID} --dispatch-id {ORCA_DISPATCH_ID} --outcome succeeded --files-modified "" --json` +(a failure is `--outcome failed`, never failure encoded only in prose). Then END YOUR TURN. + +## Orca worker protocol — REQUIRED block, append to every §O prompt above + +[1] Status stays durable — keep calling `{SKILL}/scripts/status-update.sh` at every + phase, exactly as the tmux path does. The status files remain the re-entry state; + the messages below sit on top of them, never instead of them. + +[2] Report the phase exactly once — the `--type worker_done` send written into your §O + prompt. One `worker_done` settles this Task, so plan / implement / rework are + separate Tasks. `--outcome failed` is how a failure is reported. + +[3] Forward a guardrails block instead of stalling on it — when a command is denied with + an escalation notice, send + `orca orchestration send --type escalation --subject "guardrails " --body "" --task-id {ORCA_TASK_ID} --dispatch-id {ORCA_DISPATCH_ID} --json`. + +[4] Blocking question — `orca orchestration ask --question "" --timeout-ms --json`, + then end your turn. Never open a local interactive prompt: no human is attached to + this session, so it blocks until the window expires with nothing to show for it. + +[5] `{ORCA_DISPATCH_ID}` is created by `orca-worker-start.sh`, after `task-create`. If the + orchestrator left it unsubstituted, use the dispatch id Orca gave you in this + dispatch's own context. + +[6] The Subagent usage protocol block below is REQUIRED for §O prompts too — append it. + --- ## Subagent usage protocol — REQUIRED block, append to every task prompt above diff --git a/tests/orca-wait.bats b/tests/orca-wait.bats index ca61d27..8fc87e1 100644 --- a/tests/orca-wait.bats +++ b/tests/orca-wait.bats @@ -11,6 +11,80 @@ done_msg() { # $1 = outcome, $2 = task id printf '{"result":{"count":1,"deliveryId":"delivery_1","messages":[{"id":"msg_1","type":"worker_done","from_handle":"term_a","subject":"s","body":"b","payload":"{\\"taskId\\":\\"%s\\",\\"dispatchId\\":\\"ctx_1\\",\\"outcome\\":\\"%s\\"}"}]}}' "$2" "$1" } +# --- the `orca` CLI test double ------------------------------------------------ +# Multi-window and multi-batch behavior cannot be observed through a single canned +# JSON string, so these tests replace the one boundary they do not own — the `orca` +# process — via the existing ORCA_BIN hook, and keep the script, jq and the real +# ack path real. Assertions below are always on what orca-wait.sh *did* (exit code, +# stdout, which deliveries it acked, how many windows it opened), never on the +# stub's canned bytes. +# +# The executable lives under the repo's gitignored .claude/tmp — never $TMPDIR or +# /tmp, where creating and chmod +x-ing an executable is forbidden by policy. +# Non-executable fixture data keeps using $BATS_TEST_TMPDIR, as the tests above do. +REPO_TMP() { printf '%s' "${BATS_TEST_DIRNAME}/../.claude/tmp"; } + +mk_stub() { # -> path of an executable `orca` test double + mkdir -p "$(REPO_TMP)" + local stub="$(REPO_TMP)/orca-stub-${BATS_TEST_NUMBER}" + cat > "$stub" <<'STUBEOF' +#!/bin/sh +# Test double for the `orca` CLI. All state under $STUB_DIR: +# calls window counter for `check --wait` +# windows the --timeout-ms value of each window, one per line +# .json canned Delivery returned by the nth window (absent -> empty) +# esc-at window number during which an escalation record is written +# tasklist-.json `task-list` output after the nth window (falls back to tasklist.json) +# acked one delivery id per successful ack, in order +# ack-fail if present, every ack fails +set -u +d="${STUB_DIR:?}" +case "${2:-}" in + check) + case "${3:-}" in + --wait) + n=$(cat "$d/calls" 2>/dev/null || echo 0); n=$((n + 1)); echo "$n" > "$d/calls" + prev=""; for a in "$@"; do + if [ "$prev" = "--timeout-ms" ]; then echo "$a" >> "$d/windows"; fi + prev="$a" + done + if [ -f "$d/esc-at" ] && [ "$(cat "$d/esc-at")" = "$n" ]; then + printf '{"taskId":"%s","rule":"%s","reason":"blocked"}' \ + "${STUB_ESC_TASK:-t_esc}" "${STUB_ESC_RULE:-sql_drop}" > "${STUB_ESC_DIR:?}/esc-$n.json" + fi + if [ -f "$d/$n.json" ]; then cat "$d/$n.json" + else echo '{"result":{"count":0,"messages":[]}}'; fi ;; + --ack) + if [ -f "$d/ack-fail" ]; then exit 1; fi + echo "${4:-}" >> "$d/acked"; echo '{"ok":true}' ;; + *) exit 1 ;; + esac ;; + task-list) + n=$(cat "$d/calls" 2>/dev/null || echo 0) + if [ -f "$d/tasklist-$n.json" ]; then cat "$d/tasklist-$n.json" + elif [ -f "$d/tasklist.json" ]; then cat "$d/tasklist.json" + else echo '{"result":{"tasks":[]}}'; fi ;; + *) exit 1 ;; +esac +STUBEOF + chmod +x "$stub" + printf '%s' "$stub" +} + +teardown() { rm -f "$(REPO_TMP)/orca-stub-${BATS_TEST_NUMBER}"; } + +@test "the orca test double drives the real check/ack/task-list path (no dry-run)" { + sd="$BATS_TEST_TMPDIR/sd"; mkdir -p "$sd" + printf '%s' "$(done_msg succeeded task_1)" > "$sd/1.json" + printf '{"result":{"tasks":[{"id":"task_1","status":"completed"}]}}' > "$sd/tasklist.json" + run env STUB_DIR="$sd" ORCA_BIN="$(mk_stub)" bash "$OW" 60000 task_1 + [ "$status" -eq 0 ] + [[ "$output" == *"ack delivery_1"* ]] + [[ "$output" == *"completed=1/1"* ]] + [ "$(cat "$sd/acked")" = "delivery_1" ] + [ "$(cat "$sd/calls")" = "1" ] +} + @test "worker_done succeeded: exit 0, message surfaced, delivery acked" { run env ORCA_WAIT_DRYRUN=1 ORCA_WAIT_CHECK_JSON="$(done_msg succeeded task_1)" bash "$OW" 60000 [ "$status" -eq 0 ] @@ -154,7 +228,218 @@ done_msg() { # $1 = outcome, $2 = task id [ "$status" -eq 0 ] } +# --- --until-all consumes batches until the whole Wave is completed ------------- + +@test "--until-all consumes successive batches and exits 0 once when all are completed" { + sd="$BATS_TEST_TMPDIR/sd"; mkdir -p "$sd" + printf '%s' "$(done_msg succeeded task_a)" > "$sd/1.json" + printf '%s' "$(done_msg succeeded task_b)" | sed 's/delivery_1/delivery_2/' > "$sd/2.json" + printf '{"result":{"tasks":[{"id":"task_a","status":"completed"}]}}' > "$sd/tasklist-1.json" + printf '{"result":{"tasks":[{"id":"task_a","status":"completed"},{"id":"task_b","status":"completed"}]}}' > "$sd/tasklist-2.json" + run env STUB_DIR="$sd" ORCA_BIN="$(mk_stub)" bash "$OW" --until-all 60000 task_a,task_b + [ "$status" -eq 0 ] + [[ "$output" == *"completed=1/2"* ]] # the first batch did NOT end the call + [[ "$output" == *"completed=2/2"* ]] + [ "$(cat "$sd/calls")" = "2" ] + [ "$(tr '\n' ',' < "$sd/acked")" = "delivery_1,delivery_2," ] +} + +@test "--until-all without a task id list is a usage error" { + run env ORCA_WAIT_DRYRUN=1 bash "$OW" --until-all 60000 + [ "$status" -eq 1 ] + [[ "$output" == *"needs the task id list"* ]] +} + +@test "an unknown flag is rejected before any window is opened (not read as the timeout)" { + sd="$BATS_TEST_TMPDIR/sd"; mkdir -p "$sd" + run env STUB_DIR="$sd" ORCA_BIN="$(mk_stub)" bash "$OW" --bogus 60000 task_a + [ "$status" -eq 1 ] + [[ "$output" == *"usage:"* ]] + [ ! -f "$sd/calls" ] +} + +@test "--until-all that does not complete inside the budget checkpoints at exit 2" { + sd="$BATS_TEST_TMPDIR/sd"; mkdir -p "$sd" + printf '%s' "$(done_msg succeeded task_a)" > "$sd/1.json" + printf '{"result":{"tasks":[{"id":"task_a","status":"completed"}]}}' > "$sd/tasklist.json" + run env STUB_DIR="$sd" ORCA_BIN="$(mk_stub)" ORCA_WAIT_RECHECK_MS=1000 \ + bash "$OW" --until-all 2000 task_a,task_b + [ "$status" -eq 2 ] + [[ "$output" == *"completed=1/2"* ]] + [ "$(cat "$sd/acked")" = "delivery_1" ] # the consumed batch stays consumed + [ "$(cat "$sd/calls")" = "2" ] +} + +# --- --until-all still stops dead on anything needing a coordinator decision ---- +# Batch 1 is a real completion (acked); batch 2 needs a decision and must be left +# unread so it replays. Asserting the ack log, not just the exit code, is the +# point: an implementation that acked the batch and then exited would still return +# the right code while silently dropping the event. + +_until_all_two_batches() { # $1 = JSON for batch 2 -> echoes the state dir + local sd="$BATS_TEST_TMPDIR/sd"; mkdir -p "$sd" + printf '%s' "$(done_msg succeeded task_a)" > "$sd/1.json" + printf '%s' "$1" > "$sd/2.json" + printf '{"result":{"tasks":[{"id":"task_a","status":"completed"}]}}' > "$sd/tasklist.json" + printf '%s' "$sd" +} + +@test "--until-all returns 3 immediately on a failed worker and does not ack that batch" { + sd="$(_until_all_two_batches "$(done_msg failed task_b | sed 's/delivery_1/delivery_2/')")" + run env STUB_DIR="$sd" ORCA_BIN="$(mk_stub)" bash "$OW" --until-all 60000 task_a,task_b + [ "$status" -eq 3 ] + [[ "$output" == *"task_b"* ]] + [[ "$(cat "$sd/acked")" == *"delivery_1"* ]] + [[ "$(cat "$sd/acked")" != *"delivery_2"* ]] +} + +@test "--until-all returns 5 immediately on an escalation and does not ack that batch" { + sd="$(_until_all_two_batches '{"result":{"count":1,"deliveryId":"delivery_2","messages":[{"id":"m_e","type":"escalation","from_handle":"term_b","subject":"guardrails ask: rm_rf"}]}}')" + run env STUB_DIR="$sd" ORCA_BIN="$(mk_stub)" bash "$OW" --until-all 60000 task_a,task_b + [ "$status" -eq 5 ] + [[ "$output" == *"rm_rf"* ]] + [[ "$(cat "$sd/acked")" == *"delivery_1"* ]] + [[ "$(cat "$sd/acked")" != *"delivery_2"* ]] +} + +@test "--until-all returns 6 immediately on a question and does not ack that batch" { + sd="$(_until_all_two_batches '{"result":{"count":1,"deliveryId":"delivery_2","messages":[{"id":"msg_q9","type":"question","from_handle":"term_c","subject":"which base branch?"}]}}')" + run env STUB_DIR="$sd" ORCA_BIN="$(mk_stub)" bash "$OW" --until-all 60000 task_a,task_b + [ "$status" -eq 6 ] + [[ "$output" == *"msg_q9"* ]] + [[ "$output" == *"reply"* ]] + [[ "$(cat "$sd/acked")" == *"delivery_1"* ]] + [[ "$(cat "$sd/acked")" != *"delivery_2"* ]] +} + +# --- the escalation dir is re-checked between blocking windows ------------------ + +@test "an escalation written AFTER the wait started surfaces within one interval" { + sd="$BATS_TEST_TMPDIR/sd"; esc="$BATS_TEST_TMPDIR/esc"; mkdir -p "$sd" "$esc" + echo 1 > "$sd/esc-at" # guardrails writes the record while window 1 is blocked + run env STUB_DIR="$sd" STUB_ESC_DIR="$esc" STUB_ESC_TASK=lo-7 STUB_ESC_RULE=sql_drop \ + GROUNDWORK_ESCALATION_DIR="$esc" ORCA_BIN="$(mk_stub)" ORCA_WAIT_RECHECK_MS=1000 \ + bash "$OW" 10000 + [ "$status" -eq 5 ] + [[ "$output" == *"lo-7:sql_drop"* ]] + [[ "$output" != *"ack"* ]] + # detected after ONE window, not after burning the whole 10-window budget + [ "$(cat "$sd/calls")" = "1" ] +} + +@test "the re-check loop consumes exactly the caller's budget, no more, no less" { + sd="$BATS_TEST_TMPDIR/sd"; mkdir -p "$sd" + run env STUB_DIR="$sd" ORCA_BIN="$(mk_stub)" ORCA_WAIT_RECHECK_MS=500 bash "$OW" 2000 + [ "$status" -eq 2 ] + [ "$(cat "$sd/calls")" = "4" ] + # the windows sum to exactly the caller's 2000ms — not extended, not truncated + [ "$(tr '\n' ',' < "$sd/windows")" = "500,500,500,500," ] + [[ "$output" == *"no message in 2000ms"* ]] +} + +@test "a re-check interval larger than the timeout yields exactly one clamped window" { + sd="$BATS_TEST_TMPDIR/sd"; mkdir -p "$sd" + run env STUB_DIR="$sd" ORCA_BIN="$(mk_stub)" ORCA_WAIT_RECHECK_MS=15000 bash "$OW" 1000 + [ "$status" -eq 2 ] + [ "$(cat "$sd/calls")" = "1" ] + # clamped to the remaining budget: the caller asked for 1000ms, not 15000ms + [ "$(tr '\n' ',' < "$sd/windows")" = "1000," ] +} + +@test "a zero timeout opens no window at all (boundary)" { + sd="$BATS_TEST_TMPDIR/sd"; mkdir -p "$sd" + run env STUB_DIR="$sd" ORCA_BIN="$(mk_stub)" bash "$OW" 0 + [ "$status" -eq 2 ] + [ ! -f "$sd/calls" ] +} + +@test "rejects a non-numeric or zero re-check interval (boundary)" { + run env ORCA_WAIT_DRYRUN=1 ORCA_WAIT_RECHECK_MS=abc bash "$OW" 60000 + [ "$status" -eq 1 ] + [[ "$output" == *"ORCA_WAIT_RECHECK_MS"* ]] + run env ORCA_WAIT_DRYRUN=1 ORCA_WAIT_RECHECK_MS=00 bash "$OW" 60000 + [ "$status" -eq 1 ] + [[ "$output" == *"ORCA_WAIT_RECHECK_MS"* ]] +} + +# --- a batch that carried no worker_done is not a completion -------------------- +# Heartbeats are delivered even though --types asks only for worker_done/escalation/ +# question (observed live). Exit 0 means "completions arrived — process them", so a +# liveness-only batch must not report it, or the coordinator collects artifacts that +# were never produced. + +hb_only() { # $1 = delivery id + printf '{"result":{"count":2,"deliveryId":"%s","messages":[{"type":"heartbeat","id":"m1","from_handle":"term_x","payload":"{\\"taskId\\":\\"task_1\\"}","subject":"alive"},{"type":"heartbeat","id":"m2","from_handle":"term_y","payload":"{\\"taskId\\":\\"task_2\\"}","subject":"alive"}]}}' "$1" +} + +@test "a heartbeat-only batch is consumed but never reported as a completion" { + run env ORCA_WAIT_DRYRUN=1 ORCA_WAIT_CHECK_JSON="$(hb_only delivery_hb)" \ + ORCA_WAIT_TASKLIST_JSON='{"result":{"tasks":[]}}' \ + bash "$OW" 1000 task_1,task_2 + [ "$status" -eq 2 ] + [[ "$output" == *"ack delivery_hb"* ]] # consumed: replaying liveness would livelock + [[ "$output" != *"completed="* ]] # and never counted as Wave progress +} + +@test "a heartbeat mixed with a real worker_done still completes normally (no regression)" { + mix='{"result":{"count":2,"deliveryId":"delivery_mix","messages":[{"type":"heartbeat","id":"m1","from_handle":"term_x","payload":"{\"taskId\":\"task_2\"}","subject":"alive"},{"id":"msg_1","type":"worker_done","from_handle":"term_a","subject":"s","payload":"{\"taskId\":\"task_1\",\"outcome\":\"succeeded\"}"}]}}' + run env ORCA_WAIT_DRYRUN=1 ORCA_WAIT_CHECK_JSON="$mix" \ + ORCA_WAIT_TASKLIST_JSON='{"result":{"tasks":[{"id":"task_1","status":"completed"}]}}' \ + bash "$OW" 60000 task_1,task_2 + [ "$status" -eq 0 ] + [[ "$output" == *"ack delivery_mix"* ]] + [[ "$output" == *"completed=1/2"* ]] +} + +@test "--until-all keeps waiting through a heartbeat-only batch (boundary)" { + sd="$BATS_TEST_TMPDIR/sd"; mkdir -p "$sd" + printf '%s' "$(hb_only delivery_hb)" > "$sd/1.json" + printf '%s' "$(done_msg succeeded task_a)" | sed 's/delivery_1/delivery_2/' > "$sd/2.json" + printf '{"result":{"tasks":[{"id":"task_a","status":"completed"}]}}' > "$sd/tasklist.json" + run env STUB_DIR="$sd" ORCA_BIN="$(mk_stub)" bash "$OW" --until-all 60000 task_a + [ "$status" -eq 0 ] + [[ "$output" == *"completed=1/1"* ]] + [[ "$output" != *"completed=0/1"* ]] # the noise batch reported no progress + [ "$(tr '\n' ',' < "$sd/acked")" = "delivery_hb,delivery_2," ] +} + @test "rejects a non-numeric timeout (boundary)" { run env ORCA_WAIT_DRYRUN=1 bash "$OW" "60000; rm -rf ~" [ "$status" -eq 1 ] } + +# Observed live while orchestrating this very change: a Delivery carried one +# worker_done from a PREVIOUS Wave's plan task plus one from an unrelated probe +# task, and orca-wait.sh exited 0 with completed=0/3 while all three of the +# current Wave's tasks were still running. "The batch had a worker_done" is not +# "this Wave progressed" — relevance must be judged against the caller's ids. +other_wave_done() { # $1 = delivery id + printf '{"result":{"count":2,"deliveryId":"%s","messages":[{"type":"worker_done","id":"m1","from_handle":"term_x","payload":"{\\"taskId\\":\\"task_oldwave\\",\\"outcome\\":\\"succeeded\\"}","subject":"plan_ready: old"},{"type":"worker_done","id":"m2","from_handle":"term_y","payload":"{\\"taskId\\":\\"task_probe\\",\\"outcome\\":\\"succeeded\\"}","subject":"probe"}]}}' "$1" +} + +@test "a batch whose only completions belong to OTHER tasks is not a completion here" { + run env ORCA_WAIT_DRYRUN=1 ORCA_WAIT_CHECK_JSON="$(other_wave_done delivery_ow)" \ + ORCA_WAIT_TASKLIST_JSON='{"result":{"tasks":[]}}' \ + bash "$OW" 1000 task_a,task_b,task_c + [ "$status" -eq 2 ] + [[ "$output" == *"ack delivery_ow"* ]] + [[ "$output" != *"completed="* ]] +} + +@test "--until-all keeps waiting through an other-wave completion batch" { + sd="$BATS_TEST_TMPDIR/sd"; mkdir -p "$sd" + printf '%s' "$(other_wave_done delivery_ow)" > "$sd/1.json" + printf '%s' "$(done_msg succeeded task_a)" | sed 's/delivery_1/delivery_2/' > "$sd/2.json" + printf '{"result":{"tasks":[{"id":"task_a","status":"completed"}]}}' > "$sd/tasklist.json" + run env STUB_DIR="$sd" ORCA_BIN="$(mk_stub)" bash "$OW" --until-all 60000 task_a + [ "$status" -eq 0 ] + [[ "$output" == *"completed=1/1"* ]] + [ "$(tr '\n' ',' < "$sd/acked")" = "delivery_ow,delivery_2," ] +} + +@test "with no task id list, relevance cannot be judged — any completion still counts" { + run env ORCA_WAIT_DRYRUN=1 ORCA_WAIT_CHECK_JSON="$(other_wave_done delivery_ow)" \ + bash "$OW" 1000 + [ "$status" -eq 0 ] + [[ "$output" == *"ack delivery_ow"* ]] +} diff --git a/tests/orca-worker-stalled.bats b/tests/orca-worker-stalled.bats new file mode 100644 index 0000000..c0953ba --- /dev/null +++ b/tests/orca-worker-stalled.bats @@ -0,0 +1,124 @@ +#!/usr/bin/env bats +# Tests for orca-worker-stalled.sh — the third worker state the alive/dead split +# cannot express. Driven entirely by canned `worktree ps` payloads plus a pinned +# clock, so no live Orca runtime is involved. +# +# The script is `#!/bin/sh`; invoke it with `sh`, not `bash`, or the shebang is +# overridden and a bashism would pass every test here while failing under dash in +# production. + +setup() { + WS="${BATS_TEST_DIRNAME}/../skills/orchestrate/scripts/orca-worker-stalled.sh" + WT=/repo/wt + NOW=1700000000000 +} + +ps_json() { # $1 = agent updatedAt, $2 = liveTerminalCount, $3 = agent state (optional) + printf '{"result":{"worktrees":[{"path":"%s","liveTerminalCount":%s,"hasAttachedPty":true,"lastOutputAt":%s,"agents":[{"state":"%s","updatedAt":%s}]}]}}' \ + "$WT" "$2" "$NOW" "${3:-working}" "$1" +} + +@test "recent output is progressing (exit 0)" { + run env ORCA_WORKTREE_PS_JSON="$(ps_json $((NOW - 1000)) 1)" ORCA_STALL_NOW_MS="$NOW" \ + sh "$WS" "$WT" + [ "$status" -eq 0 ] +} + +@test "silence past the threshold is a stall (exit 1) and names the worktree" { + run env ORCA_WORKTREE_PS_JSON="$(ps_json $((NOW - 900000)) 1)" ORCA_STALL_NOW_MS="$NOW" \ + sh "$WS" "$WT" + [ "$status" -eq 1 ] + [[ "$output" == *"[stalled]"* ]] + [[ "$output" == *"$WT"* ]] + [[ "$output" == *"900s"* ]] +} + +@test "the agent state is reported as context on a stall" { + run env ORCA_WORKTREE_PS_JSON="$(ps_json $((NOW - 900000)) 1 done)" ORCA_STALL_NOW_MS="$NOW" \ + sh "$WS" "$WT" + [ "$status" -eq 1 ] + [[ "$output" == *"agent state: done"* ]] +} + +@test "exactly at the threshold counts as stalled (boundary)" { + run env ORCA_WORKTREE_PS_JSON="$(ps_json $((NOW - 600000)) 1)" ORCA_STALL_NOW_MS="$NOW" \ + sh "$WS" "$WT" + [ "$status" -eq 1 ] +} + +@test "one millisecond under the threshold is not stalled (boundary)" { + run env ORCA_WORKTREE_PS_JSON="$(ps_json $((NOW - 599999)) 1)" ORCA_STALL_NOW_MS="$NOW" \ + sh "$WS" "$WT" + [ "$status" -eq 0 ] +} + +@test "ORCA_STALL_MS overrides the threshold" { + run env ORCA_WORKTREE_PS_JSON="$(ps_json $((NOW - 30000)) 1)" ORCA_STALL_NOW_MS="$NOW" \ + ORCA_STALL_MS=10000 sh "$WS" "$WT" + [ "$status" -eq 1 ] +} + +@test "no live terminal is NOT reported as a stall — that is the alive/dead question" { + run env ORCA_WORKTREE_PS_JSON='{"result":{"worktrees":[{"path":"/repo/wt","liveTerminalCount":0,"hasAttachedPty":false,"agents":[{"state":"x","updatedAt":1}]}]}}' \ + ORCA_STALL_NOW_MS="$NOW" sh "$WS" "$WT" + [ "$status" -eq 0 ] +} + +@test "an untracked worktree is unknown, never 'healthy' (exit 2)" { + run env ORCA_WORKTREE_PS_JSON='{"result":{"worktrees":[{"path":"/other","liveTerminalCount":1,"hasAttachedPty":true,"agents":[{"state":"x","updatedAt":1}]}]}}' \ + ORCA_STALL_NOW_MS="$NOW" sh "$WS" "$WT" + [ "$status" -eq 2 ] +} + +@test "no agents array at all is unknown, not stalled (exit 2)" { + run env ORCA_WORKTREE_PS_JSON='{"result":{"worktrees":[{"path":"/repo/wt","liveTerminalCount":1,"hasAttachedPty":true,"lastOutputAt":1}]}}' \ + ORCA_STALL_NOW_MS="$NOW" sh "$WS" "$WT" + [ "$status" -eq 2 ] +} + +@test "malformed payload is unknown, not stalled (exit 2)" { + run env ORCA_WORKTREE_PS_JSON='not json at all' ORCA_STALL_NOW_MS="$NOW" sh "$WS" "$WT" + [ "$status" -eq 2 ] +} + +@test "an empty payload is unknown (boundary)" { + run env ORCA_WORKTREE_PS_JSON=' ' ORCA_STALL_NOW_MS="$NOW" sh "$WS" "$WT" + [ "$status" -eq 2 ] +} + +@test "a sub-second future timestamp is precision, not skew — still progressing" { + run env ORCA_WORKTREE_PS_JSON="$(ps_json $((NOW + 800)) 1)" ORCA_STALL_NOW_MS="$NOW" \ + sh "$WS" "$WT" + [ "$status" -eq 0 ] +} + +@test "a far-future timestamp is real clock skew, not a stall (exit 2)" { + run env ORCA_WORKTREE_PS_JSON="$(ps_json $((NOW + 60000)) 1)" ORCA_STALL_NOW_MS="$NOW" \ + sh "$WS" "$WT" + [ "$status" -eq 2 ] +} + +@test "terminal repaint does not mask an idle agent (the lastOutputAt trap)" { + # lastOutputAt is fresh (a TUI spinner), the agent has not moved in 15 min. + run env ORCA_WORKTREE_PS_JSON="$(ps_json $((NOW - 900000)) 1)" ORCA_STALL_NOW_MS="$NOW" \ + sh "$WS" "$WT" + [ "$status" -eq 1 ] +} + +@test "a non-numeric threshold is refused rather than silently defaulted (error)" { + run env ORCA_WORKTREE_PS_JSON="$(ps_json $((NOW - 900000)) 1)" ORCA_STALL_NOW_MS="$NOW" \ + ORCA_STALL_MS=abc sh "$WS" "$WT" + [ "$status" -eq 2 ] +} + +@test "a zero threshold is refused (boundary — it would flag every worker)" { + run env ORCA_WORKTREE_PS_JSON="$(ps_json $NOW 1)" ORCA_STALL_NOW_MS="$NOW" \ + ORCA_STALL_MS=0 sh "$WS" "$WT" + [ "$status" -eq 2 ] +} + +@test "a missing worktree argument is a usage error" { + run sh "$WS" + [ "$status" -eq 2 ] + [[ "$output" == *"usage"* ]] +} diff --git a/tests/orca-worker-start.bats b/tests/orca-worker-start.bats index 46ddecd..0afacdd 100644 --- a/tests/orca-worker-start.bats +++ b/tests/orca-worker-start.bats @@ -4,8 +4,19 @@ # `worker-start --agent` cannot express); reuse mode must not re-create anything. setup() { + # These tests select worker mode by exporting GROUNDWORK_ESCALATION_DIR per + # case, and `run env FOO=bar ...` INHERITS the caller's environment. Run the + # suite inside a dev-loop worker (which always exports it) and every + # "no escalation env" case would silently take the worker-mode path instead. + # Declare the precondition here rather than depending on the ambient shell. + unset GROUNDWORK_ESCALATION_DIR GROUNDWORK_TASK_ID OWS="${BATS_TEST_DIRNAME}/../skills/orchestrate/scripts/orca-worker-start.sh" OK_RECEIPT='{"ok":true,"result":{"runId":"run_1","taskId":"task_1","dispatchId":"ctx_abc","state":"ready","stage":"input_accepted","setup":{"state":"running"},"effects":[{"kind":"terminal","role":"agent","action":"created","id":"term_agent"},{"kind":"dispatch_input","role":"agent","id":"term_agent","state":"accepted"}]}}' + # re-entry probe fixtures — payload shapes verified against the live Orca CLI: + # an unknown task answers ok:true/dispatch:null, so a first start is not a probe failure. + DSP_PREV='{"ok":true,"result":{"dispatch":{"assignee_handle":"term_prev"}}}' + DSP_NONE='{"ok":true,"result":{"dispatch":null}}' + TL_LIVE='{"ok":true,"result":{"terminals":[{"handle":"term_prev","orphaned":false,"connected":true}]}}' } # --- worker mode (escalation contract active) --------------------------------- @@ -84,6 +95,119 @@ setup() { [[ "$output" != *"dispatch="* ]] } +# --- re-entry idempotency (worker mode) --------------------------------------- +# SKILL.md restarts a "dead" worker with --worktree/--agent. If that liveness +# read was wrong, an unconditional `terminal create` puts a SECOND agent on the +# same checkout. The probe must bind the live one instead. + +@test "re-entry: a live agent terminal on the worktree is reused, not duplicated" { + run env ORCA_WORKER_START_DRYRUN=1 GROUNDWORK_ESCALATION_DIR=/e \ + ORCA_WORKER_START_DISPATCH_JSON="$DSP_PREV" \ + ORCA_WORKER_START_TERMLIST_JSON="$TL_LIVE" \ + bash "$OWS" --task task_1 --worktree "id:r::/wt" --agent claude + [ "$status" -eq 0 ] + [[ "$output" == *"[--worktree] [id:r::/wt] [--terminal] [term_prev]"* ]] + [[ "$output" == *"already has a live agent terminal term_prev"* ]] + [[ "$output" != *"[terminal] [create]"* ]] +} + +@test "re-entry: no dispatch on record (first start) still creates the terminal" { + run env ORCA_WORKER_START_DRYRUN=1 GROUNDWORK_ESCALATION_DIR=/e \ + ORCA_WORKER_START_DISPATCH_JSON="$DSP_NONE" \ + bash "$OWS" --task task_1 --worktree "id:r::/wt" --agent claude + [ "$status" -eq 0 ] + [[ "$output" == *"[terminal] [create]"* ]] + [[ "$output" != *"reusing it"* ]] +} + +@test "re-entry: a recorded but ORPHANED terminal is dead — create (boundary)" { + run env ORCA_WORKER_START_DRYRUN=1 GROUNDWORK_ESCALATION_DIR=/e \ + ORCA_WORKER_START_DISPATCH_JSON="$DSP_PREV" \ + ORCA_WORKER_START_TERMLIST_JSON='{"ok":true,"result":{"terminals":[{"handle":"term_prev","orphaned":true,"connected":true}]}}' \ + bash "$OWS" --task task_1 --worktree "id:r::/wt" --agent claude + [ "$status" -eq 0 ] + [[ "$output" == *"[terminal] [create]"* ]] + [[ "$output" != *"reusing it"* ]] +} + +@test "re-entry: a recorded but DISCONNECTED terminal is dead — create (boundary)" { + run env ORCA_WORKER_START_DRYRUN=1 GROUNDWORK_ESCALATION_DIR=/e \ + ORCA_WORKER_START_DISPATCH_JSON="$DSP_PREV" \ + ORCA_WORKER_START_TERMLIST_JSON='{"ok":true,"result":{"terminals":[{"handle":"term_prev","orphaned":false,"connected":false}]}}' \ + bash "$OWS" --task task_1 --worktree "id:r::/wt" --agent claude + [ "$status" -eq 0 ] + [[ "$output" == *"[terminal] [create]"* ]] + [[ "$output" != *"reusing it"* ]] +} + +@test "re-entry: an empty terminal list is not a reuse target (boundary)" { + run env ORCA_WORKER_START_DRYRUN=1 GROUNDWORK_ESCALATION_DIR=/e \ + ORCA_WORKER_START_DISPATCH_JSON="$DSP_PREV" \ + ORCA_WORKER_START_TERMLIST_JSON='{"ok":true,"result":{"terminals":[]}}' \ + bash "$OWS" --task task_1 --worktree "id:r::/wt" --agent claude + [ "$status" -eq 0 ] + [[ "$output" == *"[terminal] [create]"* ]] +} + +@test "re-entry: an assignee living on another worktree is not reused (boundary)" { + run env ORCA_WORKER_START_DRYRUN=1 GROUNDWORK_ESCALATION_DIR=/e \ + ORCA_WORKER_START_DISPATCH_JSON="$DSP_PREV" \ + ORCA_WORKER_START_TERMLIST_JSON='{"ok":true,"result":{"terminals":[{"handle":"term_other","orphaned":false,"connected":true}]}}' \ + bash "$OWS" --task task_1 --worktree "id:r::/wt" --agent claude + [ "$status" -eq 0 ] + [[ "$output" == *"[terminal] [create]"* ]] + [[ "$output" != *"term_prev"* ]] +} + +# --- re-entry: a probe that cannot answer fails closed (exit 6) --------------- +# "Unknown" must never be treated as "no live agent": that is the direction that +# silently puts two agents on one checkout. + +@test "re-entry: dispatch-show returning ok:false refuses to create (exit 6)" { + run env ORCA_WORKER_START_DRYRUN=1 GROUNDWORK_ESCALATION_DIR=/e \ + ORCA_WORKER_START_DISPATCH_JSON='{"ok":false,"error":{"code":"relay_unavailable"}}' \ + bash "$OWS" --task task_1 --worktree "id:r::/wt" --agent claude + [ "$status" -eq 6 ] + [[ "$output" == *"cannot read the dispatch for 'task_1'"* ]] + [[ "$output" != *"[terminal] [create]"* ]] + [[ "$output" != *"dispatch="* ]] +} + +@test "re-entry: a malformed dispatch receipt refuses to create (exit 6)" { + run env ORCA_WORKER_START_DRYRUN=1 GROUNDWORK_ESCALATION_DIR=/e \ + ORCA_WORKER_START_DISPATCH_JSON='not json at all' \ + bash "$OWS" --task task_1 --worktree "id:r::/wt" --agent claude + [ "$status" -eq 6 ] + [[ "$output" != *"[terminal] [create]"* ]] +} + +@test "re-entry: terminal list ok:false refuses while the assignee may be live" { + run env ORCA_WORKER_START_DRYRUN=1 GROUNDWORK_ESCALATION_DIR=/e \ + ORCA_WORKER_START_DISPATCH_JSON="$DSP_PREV" \ + ORCA_WORKER_START_TERMLIST_JSON='{"ok":false,"error":{"code":"selector_not_found"}}' \ + bash "$OWS" --task task_1 --worktree "id:r::/wt" --agent claude + [ "$status" -eq 6 ] + [[ "$output" == *"term_prev"* ]] + [[ "$output" != *"[terminal] [create]"* ]] +} + +@test "re-entry: a malformed terminal list refuses to create (exit 6)" { + run env ORCA_WORKER_START_DRYRUN=1 GROUNDWORK_ESCALATION_DIR=/e \ + ORCA_WORKER_START_DISPATCH_JSON="$DSP_PREV" \ + ORCA_WORKER_START_TERMLIST_JSON='garbage' \ + bash "$OWS" --task task_1 --worktree "id:r::/wt" --agent claude + [ "$status" -eq 6 ] + [[ "$output" != *"[terminal] [create]"* ]] +} + +@test "re-entry: a blank probe payload is unknown, not 'no worker' (boundary)" { + run env ORCA_WORKER_START_DRYRUN=1 GROUNDWORK_ESCALATION_DIR=/e \ + ORCA_WORKER_START_DISPATCH_JSON=' ' \ + bash "$OWS" --task task_1 --worktree "id:r::/wt" --agent claude + [ "$status" -eq 6 ] + [[ "$output" != *"[terminal] [create]"* ]] +} + # --- composed agent-first mode (no escalation contract requested) ------------- @test "no escalation env: uses Orca's agent-first worker-start with --setup run" { @@ -148,6 +272,28 @@ setup() { # --- reuse mode --------------------------------------------------------------- +@test "reuse mode: the re-entry probe never runs or overrides --terminal" { + run env ORCA_WORKER_START_DRYRUN=1 GROUNDWORK_ESCALATION_DIR=/e \ + ORCA_WORKER_START_DISPATCH_JSON="$DSP_PREV" \ + ORCA_WORKER_START_TERMLIST_JSON="$TL_LIVE" \ + bash "$OWS" --task task_2 --terminal term_given + [ "$status" -eq 0 ] + [[ "$output" == *"[--terminal] [term_given]"* ]] + [[ "$output" != *"term_prev"* ]] + [[ "$output" != *"reusing it"* ]] +} + +@test "composed agent-first mode is not probed either (no escalation env)" { + run env ORCA_WORKER_START_DRYRUN=1 \ + ORCA_WORKER_START_DISPATCH_JSON="$DSP_PREV" \ + ORCA_WORKER_START_TERMLIST_JSON="$TL_LIVE" \ + bash "$OWS" --task task_1 --worktree "id:r::/wt" --agent codex + [ "$status" -eq 0 ] + [[ "$output" == *"[--agent] [codex]"* ]] + [[ "$output" != *"[--terminal]"* ]] + [[ "$output" != *"term_prev"* ]] +} + @test "reuse mode: passes --terminal only and creates nothing" { run env ORCA_WORKER_START_DRYRUN=1 GROUNDWORK_ESCALATION_DIR=/e \ bash "$OWS" --task task_2 --terminal term_agent