From 2facd2e36a88af826d04bcc75e9cf22d413e5bb8 Mon Sep 17 00:00:00 2001 From: kjgbot Date: Sun, 20 Sep 2026 01:51:16 -0700 Subject: [PATCH 1/7] fix: harden agent transport recovery Session-Id: 01a0bdd5-1542-7fe1-b85c-ada48bf177d9 --- README.md | 4 + docs/SURFACE.md | 46 +++- .../software-factory/software-factory.flow.ts | 6 + kernel/DESIGN.md | 14 +- kernel/relayflowd-core/src/entry.rs | 12 +- kernel/relayflowd-core/src/machine.rs | 34 ++- .../src/machine/parallel_tests.rs | 7 +- .../relayflowd-core/src/machine/recovery.rs | 11 +- kernel/relayflowd-core/src/machine/tests.rs | 123 ++++++++++- kernel/relayflowd-core/src/retry.rs | 1 + kernel/relayflowd-core/src/spec.rs | 17 ++ kernel/relayflowd/src/server/tests.rs | 3 +- .../src/server/tests/agent/contract.rs | 2 +- .../relayflowd/src/server/tests/agent/pins.rs | 2 +- .../tests/crash_resume/pin_projection.rs | 12 +- kernel/relayflowd/tests/parallel_driver.rs | 6 +- packages/sdk/src/authored-root.ts | 11 +- packages/sdk/src/authored-worker-step.ts | 32 +++ packages/sdk/src/cli-transport-evidence.ts | 70 ++++++ packages/sdk/src/cli/step-failure.ts | 23 +- packages/sdk/src/compile.ts | 27 ++- packages/sdk/src/failure-kinds.ts | 7 + packages/sdk/src/pty-sidechannel.ts | 28 ++- packages/sdk/src/spec.ts | 3 + packages/sdk/src/step-fields.ts | 1 + packages/sdk/src/validate.ts | 6 + packages/sdk/src/worker-cli-relay.ts | 51 +++++ packages/sdk/src/worker-cli.ts | 204 ++++++++++-------- packages/sdk/src/worker.ts | 20 +- .../sdk/tests/agent-transcript-live.test.ts | 2 +- .../tests/authored-agent-permissions.test.ts | 15 ++ packages/sdk/tests/authored-flow.test.ts | 16 ++ packages/sdk/tests/authored-root.test.ts | 10 +- .../authored-run-failure-evidence.test.ts | 3 +- packages/sdk/tests/deterministic-llm.test.ts | 2 + packages/sdk/tests/pty-sidechannel.test.ts | 28 +++ packages/sdk/tests/spec-parity.test.ts | 19 ++ .../sdk/tests/step-failure-diagnostic.test.ts | 21 ++ packages/sdk/tests/verb-field-lint.test.ts | 1 + packages/sdk/tests/worker-cli.test.ts | 91 ++++++++ packages/surface/src/context.ts | 6 + 41 files changed, 839 insertions(+), 158 deletions(-) create mode 100644 packages/sdk/src/cli-transport-evidence.ts create mode 100644 packages/sdk/src/worker-cli-relay.ts diff --git a/README.md b/README.md index ba59bab97..3907ccbca 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,10 @@ export default flow('hello', async (f) => { console.log(greeting.trim()); const answer = await f.agent('greeter', { task: 'Reply with one short hello sentence. Do not use tools or modify files.', + // Semantic failures do not retry; one classified transport loss may. + maxIterations: 1, + transportRetries: 1, + recoveryMode: 'reset', }); console.log(answer.summary); f.done('success'); diff --git a/docs/SURFACE.md b/docs/SURFACE.md index 4620b56ea..a1e62c948 100644 --- a/docs/SURFACE.md +++ b/docs/SURFACE.md @@ -321,6 +321,37 @@ flow-wide `FlowHeader.workspace` / `tools.fs` scopes. The chief harness above remains an aspirational example; this option does not make that entire harness executable today. +### Agent retry and recovery controls + +`f.agent` separates semantic iteration from infrastructure recovery: + +```ts +await f.agent("reviewer", { + task: "Review the current change and write the verdict file.", + maxIterations: 2, // verification may reject one semantic result + transportRetries: 1, // one additional classified infrastructure attempt + recoveryMode: "inspect", // reset | inspect | manual +}); +``` + +The compatibility defaults are `maxIterations: 1`, `transportRetries: 1`, and +`recoveryMode: "reset"`. The bounded one-retry default preserves the existing +single-crash resume contract while removing the old unbounded infrastructure +loop; set `transportRetries: 0` to disable transport recovery. The SDK omits an +undeclared budget so legacy canonical specs and hashes stay unchanged, while +an explicit zero is retained. + +Transport retry is deliberately narrow. A signal close, a close without a +status, a bounded set of transient spawn errors, or the exact historical Codex +stdin-lifecycle failure is reported as `crashed` and may consume the transport +budget. An ordinary nonzero CLI exit remains `worker_error` and is terminal; +`maxIterations` never turns every nonzero exit into a retry. Every replacement +attempt reuses the step idempotency key. `reset` restores declared start pins, +`inspect` carries the dirty pins plus the prior transport trajectory, and +`manual` parks rather than redispatching. Journaled direct-transport evidence +includes bounded/redacted phase, cause, exit code, signal, OS error code, and +stderr tail. + ### Supported TypeScript LLM calls The local authored executor supports these signatures: @@ -656,10 +687,12 @@ kernel dispatch, including for authored `f.agent` calls. A subscriber sends `HELLO view\n`, `HELLO drive\n`, or `HELLO passthrough\n`, then receives live stdout/stderr bytes. View and passthrough are passive. Only drive forwards subsequent bytes to child stdin. -In this pipe-based slice, a drive greeting must arrive within 100ms of child -startup. Without one, the worker closes stdin so unattended and passive-view -agents receive EOF. Later drive greetings are rejected without marking human -intervention; a closed stdin pipe cannot be reopened. Supporting drive attachment +In this pipe-based slice, a drive greeting must arrive during a 100ms bounded +enrollment window **before** child startup. Only then is the child spawned with +a writable stdin pipe. Without one, the worker spawns with stdin ignored, so +unattended Codex never enters its "additional input from stdin" lifecycle and +passive viewers cannot change the input contract. Later drive greetings are +rejected without marking human intervention. Supporting drive attachment at arbitrary times requires a future terminal/session transport. Drive readers pause while child stdin writes flush, preserving input under backpressure. There is no backlog, terminal resize, or framing after the greeting. Socket @@ -715,8 +748,9 @@ Journal: /runs/.sqlite3 Each clause is present only when the journal holds the fact behind it; nothing is defaulted. The same fields appear as named keys on the `--json` diagnostic (`stepId`, `stepType`, `completionReason`, `attempt`, `maxIterations`, -`exitCode`, `stdoutTail`, `stderrTail`, `detail`, `transcriptPath`, `hint`, -`journalPath`), so the rendered line and the machine-readable record carry the +`transportRetries`, `exitCode`, `transportPhase`, `transportCause`, `signal`, +`errorCode`, `retryableTransport`, `stdoutTail`, `stderrTail`, `detail`, +`transcriptPath`, `hint`, `journalPath`), so the rendered line and the machine-readable record carry the same facts rather than the message being the only copy. `attempt=/` is read from the journal, not from the spec: `n` is the diff --git a/examples/software-factory/software-factory.flow.ts b/examples/software-factory/software-factory.flow.ts index 43c37a8f4..8224cec1b 100644 --- a/examples/software-factory/software-factory.flow.ts +++ b/examples/software-factory/software-factory.flow.ts @@ -45,6 +45,10 @@ export default flow("software-factory", { budget: { dollars: 10, wallcloc await f.agent("implementer", { cli: "claude", + // Continue from the journaled dirty workspace only for a classified + // transport loss; an ordinary nonzero CLI exit remains terminal. + transportRetries: 1, + recoveryMode: "inspect", task: `Implement this ticket in the current repository, on the current branch, with regression tests. Commit as you go.\n` + `Write a PR description to ${WORK}/summary.md (what changed, how it was verified). Do not touch ${WORK}/ otherwise.\n\nTicket:\n${ticket}`, }).gate({ type: "subprocess_gate", command: `test -s ${WORK}/summary.md` }); @@ -55,6 +59,8 @@ export default flow("software-factory", { budget: { dollars: 10, wallcloc // what it finds; it must end with an explicit verdict file, not prose. await f.agent("adversary", { cli: "claude", + transportRetries: 1, + recoveryMode: "inspect", task: `Review the diff against the base branch as an adversary: find bugs, missing tests, unsafe defaults, and scope creep. ` + `Fix what is mechanical and re-run the tests. Write ${WORK}/review.md with your findings, then write ${WORK}/review.passed ` + `ONLY if the change is ready for a human to merge; otherwise write ${WORK}/review.blocked with the blocking findings.`, diff --git a/kernel/DESIGN.md b/kernel/DESIGN.md index 30b9b8830..e88141368 100644 --- a/kernel/DESIGN.md +++ b/kernel/DESIGN.md @@ -58,6 +58,7 @@ One per attempt. Payload: | `pins.workspace` | agent steps: `[{surface, revision_id}]` — relayfile revision id per declared mount surface, or `{worktree_base_commit}` | | `pins.streams` | `[{stream, read_offset}]` — consumer offsets at attempt start | | `max_iterations` | from spec, echoed for legibility | +| `max_transport_retries` | additional attempts allowed after classified infrastructure loss; default one and omitted at that default | Deterministic/llm steps journal `pins.streams` only if they consume streams; `pins.workspace` is empty (no workspace). @@ -74,7 +75,7 @@ Payload: | `verification` | `{gate, verdict: pass\|fail, detail}` or null | | `end_pins` | agent steps: `{workspace: [{surface, revision_id}], streams: [{stream, read_offset}]}` — Appendix A rule 6: the next step's starting state **is** this | | `effects` | list of `{surface_path, idempotency_key}` dedupe keys recorded this attempt | -| `trajectory_tail` | agent failure only: worker-supplied tail injected into an `inspect` retry; `step.complete` rejects one over 16 KiB of canonical JSON | +| `trajectory_tail` | agent failure only: worker-supplied tail injected into an `inspect` retry; direct workers add bounded/redacted `transport{phase,cause,exit_code,signal,error_code?,retryable,stderr_tail}` evidence; `step.complete` rejects one over 16 KiB of canonical JSON | | `budget` | `{tokens_in, tokens_out, dollars, dollars_unmetered?}` — exact; zero for memoized replay by construction (no entry is written on replay). `dollars_unmetered: true` (omitted when false) marks tokens of unknown dollar cost: `dollars` is then metered cost only, dollar ceilings ignore the unknown part, token ceilings count it | | `completed_by` | `kernel` \| worker id — out-of-band completion uses the same entry, same discipline | | `next_attempt_at_ms` | when `disposition=retry`: computed backoff+jitter wake time | @@ -290,9 +291,14 @@ attempt's `budget` field. under the attempt's idempotency key; every writeback is a journaled `effect.recorded` deduped by `(step_id, idempotency_key, surface_path)`; `step.completed` pins end state, which defines the next step's start. Dead -attempt ⇒ recovery mode: `reset` restores pinned revisions and retries; -`inspect` retries inside the dirty workspace with the failed attempt's tail -injected; `manual` parks as `wait.human` with `diff_ref`. +attempt ⇒ recovery mode: `reset` restores pinned revisions before a permitted +retry; `inspect` resumes inside the dirty workspace with the failed attempt's +tail injected; `manual` parks as `wait.human` with `diff_ref`. Infrastructure retry +is a separate, explicit budget (`retry.max_transport_retries`, default one so a +single process loss remains resumable; set zero to disable). +Only `crashed` and `lease_expired` consume it. `worker_error`, timeout, budget, +cancellation, and an ordinary nonzero CLI exit are terminal regardless of the +budget; semantic verification retry remains bounded only by `max_iterations`. ### Memoized resume diff --git a/kernel/relayflowd-core/src/entry.rs b/kernel/relayflowd-core/src/entry.rs index 3dbb952cf..bc08608ce 100644 --- a/kernel/relayflowd-core/src/entry.rs +++ b/kernel/relayflowd-core/src/entry.rs @@ -204,6 +204,12 @@ pub struct AttemptStartedPayload { pub recovery_mode: Option, pub pins: Pins, pub max_iterations: u32, + #[serde(default, skip_serializing_if = "is_zero_u32")] + pub max_transport_retries: u32, +} + +fn is_zero_u32(value: &u32) -> bool { + *value == 0 } /// Runtime pins journaled per attempt (RFC Appendix A rules 2 and 6): the @@ -317,8 +323,10 @@ mod completion_reason_tests { /// skip `ALL`. #[test] fn all_covers_every_serialized_label() { - let labels: std::collections::HashSet<&str> = - CompletionReason::ALL.iter().map(|r| r.journal_label()).collect(); + let labels: std::collections::HashSet<&str> = CompletionReason::ALL + .iter() + .map(|r| r.journal_label()) + .collect(); assert_eq!( labels.len(), CompletionReason::ALL.len(), diff --git a/kernel/relayflowd-core/src/machine.rs b/kernel/relayflowd-core/src/machine.rs index fd1be7840..533809cf8 100644 --- a/kernel/relayflowd-core/src/machine.rs +++ b/kernel/relayflowd-core/src/machine.rs @@ -303,6 +303,7 @@ fn start_actions(state: &RunState, step: &StepSpec, attempt: u32, now_ms: i64) - recovery_mode, pins: pins.clone(), max_iterations: step.max_iterations, + max_transport_retries: step.retry.max_transport_retries, }, ); let execute = match step.step_type() { @@ -363,7 +364,23 @@ pub fn completion_actions( let verified = verification .as_ref() .is_some_and(|record| record.verdict == crate::entry::VerificationVerdict::Pass); - let may_retry = semantic_executions.saturating_add(1) < step.max_iterations; + let may_retry_semantic = semantic_executions.saturating_add(1) < step.max_iterations; + // `attempt` counts every start while `semantic_executions` counts only + // results a gate could judge. Their difference is therefore the number of + // infrastructure failures including this attempt. Keep that budget + // separate: a generic nonzero worker exit is not infrastructure, and a + // crash must not get an unbounded free loop merely because it consumed no + // semantic iteration. + let transport_failures = attempt.saturating_sub(semantic_executions); + let may_retry_transport = transport_failures <= step.retry.max_transport_retries; + let semantic_failure = matches!( + result.failure_reason, + None | Some(CompletionReason::VerificationFailed) + ); + let transport_failure = matches!( + result.failure_reason, + Some(CompletionReason::Crashed | CompletionReason::LeaseExpired) + ); // Preserve `result.output` for successful completions, and for FAILED // deterministic completions specifically — deterministic attempts journal // `{exit_code, stdout_tail, stderr_tail}` so the CLI can render the @@ -378,7 +395,8 @@ pub fn completion_actions( result.output, None, ) - } else if may_retry { + } else if (semantic_failure && may_retry_semantic) || (transport_failure && may_retry_transport) + { let key = idempotency_key(run_id, &step.id); let delay = backoff_delay_ms(&step.retry, &key, attempt); ( @@ -386,7 +404,11 @@ pub fn completion_actions( .failure_reason .unwrap_or(CompletionReason::VerificationFailed), Disposition::Retry, - if preserve_failure_output { result.output } else { Value::Null }, + if preserve_failure_output { + result.output + } else { + Value::Null + }, Some(now_ms.saturating_add(delay as i64)), ) } else { @@ -395,7 +417,11 @@ pub fn completion_actions( .failure_reason .unwrap_or(CompletionReason::RetriesExhausted), Disposition::StepDone, - if preserve_failure_output { result.output } else { Value::Null }, + if preserve_failure_output { + result.output + } else { + Value::Null + }, None, ) }; diff --git a/kernel/relayflowd-core/src/machine/parallel_tests.rs b/kernel/relayflowd-core/src/machine/parallel_tests.rs index 755bdb3d1..7b41c3339 100644 --- a/kernel/relayflowd-core/src/machine/parallel_tests.rs +++ b/kernel/relayflowd-core/src/machine/parallel_tests.rs @@ -13,7 +13,7 @@ fn parallel_spec() -> crate::RunSpec { "id": "lane-b", "type": "llm", "prompt": "research b", - "retry": {"initial_backoff_ms": 0, "max_backoff_ms": 0, "multiplier": 1, "jitter_percent": 0} + "retry": {"initial_backoff_ms": 0, "max_backoff_ms": 0, "multiplier": 1, "jitter_percent": 0, "max_transport_retries": 1} }, { "id": "join", @@ -25,7 +25,7 @@ fn parallel_spec() -> crate::RunSpec { "id": "lane-a", "type": "llm", "prompt": "research a", - "retry": {"initial_backoff_ms": 0, "max_backoff_ms": 0, "multiplier": 1, "jitter_percent": 0} + "retry": {"initial_backoff_ms": 0, "max_backoff_ms": 0, "multiplier": 1, "jitter_percent": 0, "max_transport_retries": 1} } ] })) @@ -50,18 +50,21 @@ fn parallel_agent_spec(overlapping: bool) -> crate::RunSpec { "id": "lane-b", "type": "agent", "instruction": "b", + "retry": {"max_transport_retries": 1}, "surfaces": {"workspace": [{"surface": "repo-b"}]} }, { "id": "lane-a", "type": "agent", "instruction": "a", + "retry": {"max_transport_retries": 1}, "surfaces": {"workspace": [{"surface": lane_a_surface}]} }, { "id": "join", "type": "agent", "instruction": "join", + "retry": {"max_transport_retries": 1}, "depends_on": ["lane-b", "lane-a"], "surfaces": {"workspace": [ {"surface": "repo-b"}, diff --git a/kernel/relayflowd-core/src/machine/recovery.rs b/kernel/relayflowd-core/src/machine/recovery.rs index e59c236c4..3786d0a24 100644 --- a/kernel/relayflowd-core/src/machine/recovery.rs +++ b/kernel/relayflowd-core/src/machine/recovery.rs @@ -85,13 +85,16 @@ pub fn abandonment_actions( .. } ); - let may_retry = runtime.semantic_executions < spec.max_iterations; + let transport_failures = runtime.attempts.saturating_sub(runtime.semantic_executions); + let may_retry = transport_failures <= spec.retry.max_transport_retries; // No retry delay for a dead leased attempt. This function records an // attempt that died WITHOUT producing a result a gate could judge -- which // is why, as the doc comment above says, it does not charge a semantic - // iteration either. Rate-limiting it is the same category error: the - // backoff curve exists to damp a step that keeps failing on its own merits, - // not one whose worker was killed. + // iteration. It is still bounded by the explicit transport retry budget; + // otherwise a permanently broken worker can redispatch forever while its + // semantic counter stays at zero. Rate-limiting remains a separate + // category: the backoff curve damps a step that keeps failing on its own + // merits, not one whose worker was killed. // // Leaving the delay in place also made recovery order a race, which is // issue #155. The dead lane sat in `Backoff` with `wake_at_ms` a few diff --git a/kernel/relayflowd-core/src/machine/tests.rs b/kernel/relayflowd-core/src/machine/tests.rs index 6add8aa32..61bbfe96a 100644 --- a/kernel/relayflowd-core/src/machine/tests.rs +++ b/kernel/relayflowd-core/src/machine/tests.rs @@ -63,8 +63,7 @@ fn verification_failure_schedules_a_durable_retry() { } #[test] -fn failed_deterministic_completion_preserves_exit_code_and_stderr( -) { +fn failed_deterministic_completion_preserves_exit_code_and_stderr() { // #292: failed attempts used to journal `output: null`, so the CLI // could not surface the actual exit code or stderr excerpt. Both the // retry branch and the terminal branch must now preserve the captured @@ -146,6 +145,7 @@ fn every_failed_run_terminates_with_declared_completion_reasons() { let spec = retrying_spec(); let mut step = spec.steps[0].clone(); step.max_iterations = 1; + step.retry.max_transport_retries = 0; let mut result = AttemptResult::successful(Value::Null, "test"); result.failure_reason = Some(reason); result.failure_detail = Some("declared test failure".to_owned()); @@ -290,7 +290,8 @@ fn crashed_attempt_does_not_consume_an_iteration() { // max_iterations 2: crash attempt 1, verification-fail the replacement // (attempt 2) — one semantic iteration must remain, so the step retries // instead of exhausting after a single semantic result. - let spec = retrying_spec(); + let mut spec = retrying_spec(); + spec.steps[0].retry.max_transport_retries = 1; let fresh = RunState::fold("run", spec.clone(), &[]).unwrap(); let Action::Append(started) = next_actions(&fresh, 10).remove(0) else { panic!("attempt 1 must journal its lease"); @@ -362,8 +363,8 @@ fn all_backing_off_steps_return_timers() { end_pins: None, effects: Vec::new(), trajectory_tail: None, - failure_reason: Some(CompletionReason::WorkerError), - failure_detail: Some("stub rejection".to_owned()), + failure_reason: None, + failure_detail: None, }; let mut entries = Vec::new(); for step in &spec.steps { @@ -395,7 +396,7 @@ fn agent_spec(mode: &str) -> crate::RunSpec { "instruction": "edit the workspace", "recovery_mode": mode, "max_iterations": 2, - "retry": {"initial_backoff_ms": 0, "max_backoff_ms": 0, "multiplier": 1, "jitter_percent": 0}, + "retry": {"initial_backoff_ms": 0, "max_backoff_ms": 0, "multiplier": 1, "jitter_percent": 0, "max_transport_retries": 1}, "surfaces": {"workspace": [{"surface": "repo"}]} }] })) @@ -458,6 +459,112 @@ fn reset_recovery_dispatches_the_original_pinned_revision() { ); } +#[test] +fn classified_transport_retry_is_bounded_and_preserves_pin_and_idempotency() { + let spec = agent_spec("reset"); + let pinned = workspace_pins("rev-clean"); + let started = started_agent(&spec, pinned.clone()); + let first_started: AttemptStartedPayload = + serde_json::from_value(started.payload.clone()).unwrap(); + assert_eq!(first_started.max_transport_retries, 1); + let running = RunState::fold("run", spec.clone(), std::slice::from_ref(&started)).unwrap(); + + let mut crashed = AttemptResult::successful(Value::Null, "worker"); + crashed.failure_reason = Some(CompletionReason::Crashed); + crashed.failure_detail = Some("direct transport closed without a status".to_owned()); + crashed.trajectory_tail = Some(json!({"transport": {"cause": "close_without_status"}})); + let first_failure = completion_actions( + "run", + &spec.steps[0], + 1, + running.steps["agent"].semantic_executions, + crashed.clone(), + 20, + ); + let Action::Append(first_completed) = &first_failure[0] else { + panic!() + }; + let first_payload: StepCompletedPayload = + serde_json::from_value(first_completed.payload.clone()).unwrap(); + assert_eq!(first_payload.completion_reason, CompletionReason::Crashed); + assert_eq!(first_payload.disposition, Disposition::Retry); + + let mut entries = vec![started]; + entries.extend(first_failure.into_iter().filter_map(|action| match action { + Action::Append(entry) => Some(entry), + _ => None, + })); + let backoff = RunState::fold("run", spec.clone(), &entries).unwrap(); + let Action::Append(woken) = + next_actions(&backoff, first_payload.next_attempt_at_ms.unwrap()).remove(0) + else { + panic!("transport retry timer must be journaled as completed") + }; + entries.push(woken); + let ready = RunState::fold("run", spec.clone(), &entries).unwrap(); + let retry = next_actions(&ready, first_payload.next_attempt_at_ms.unwrap()); + let Action::Append(second_started) = &retry[0] else { + panic!() + }; + let second_started: AttemptStartedPayload = + serde_json::from_value(second_started.payload.clone()).unwrap(); + let Action::Dispatch { + idempotency_key, + pins, + recovery, + .. + } = &retry[1] + else { + panic!() + }; + assert_eq!(idempotency_key, &first_started.idempotency_key); + assert_eq!(pins, &pinned); + assert_eq!( + recovery.as_ref().unwrap().restore_pins.as_ref(), + Some(&pinned) + ); + assert_eq!(second_started.pins, pinned); + + let terminal = completion_actions( + "run", + &spec.steps[0], + 2, + ready.steps["agent"].semantic_executions, + crashed, + 40, + ); + let Action::Append(terminal) = &terminal[0] else { + panic!() + }; + let terminal: StepCompletedPayload = serde_json::from_value(terminal.payload.clone()).unwrap(); + assert_eq!(terminal.completion_reason, CompletionReason::Crashed); + assert_eq!(terminal.disposition, Disposition::StepDone); + assert_eq!(terminal.next_attempt_at_ms, None); +} + +#[test] +fn ordinary_worker_error_is_not_retried_by_either_budget() { + let mut spec = agent_spec("reset"); + spec.steps[0].max_iterations = 4; + spec.steps[0].retry.max_transport_retries = 4; + let mut failed = AttemptResult::successful(Value::Null, "worker"); + failed.failure_reason = Some(CompletionReason::WorkerError); + failed.failure_detail = Some("CLI rejected the task".to_owned()); + let actions = completion_actions("run", &spec.steps[0], 1, 0, failed, 20); + assert_eq!( + actions.len(), + 1, + "a semantic worker error must not schedule a blind retry" + ); + let Action::Append(completed) = &actions[0] else { + panic!() + }; + let completed: StepCompletedPayload = + serde_json::from_value(completed.payload.clone()).unwrap(); + assert_eq!(completed.completion_reason, CompletionReason::WorkerError); + assert_eq!(completed.disposition, Disposition::StepDone); +} + #[test] fn inspect_recovery_injects_the_dirty_pin_completion_reason_and_tail() { let spec = agent_spec("inspect"); @@ -473,7 +580,7 @@ fn inspect_recovery_injects_the_dirty_pin_completion_reason_and_tail() { end_pins: Some(dirty.clone()), effects: vec![], trajectory_tail: Some(json!(["edited file"])), - failure_reason: Some(CompletionReason::WorkerError), + failure_reason: Some(CompletionReason::Crashed), failure_detail: Some("stub rejection".to_owned()), }; let completed = completion_actions( @@ -503,7 +610,7 @@ fn inspect_recovery_injects_the_dirty_pin_completion_reason_and_tail() { assert_eq!(pins, &dirty); assert_eq!( recovery.previous_completion_reason, - Some(CompletionReason::WorkerError) + Some(CompletionReason::Crashed) ); assert_eq!(recovery.trajectory_tail, Some(json!(["edited file"]))); assert!(recovery.restore_pins.is_none()); diff --git a/kernel/relayflowd-core/src/retry.rs b/kernel/relayflowd-core/src/retry.rs index ed8550fa9..edc3e44b7 100644 --- a/kernel/relayflowd-core/src/retry.rs +++ b/kernel/relayflowd-core/src/retry.rs @@ -42,6 +42,7 @@ mod tests { max_backoff_ms: 5_000, multiplier: 2, jitter_percent: 20, + max_transport_retries: 1, }; let first = backoff_delay_ms(&policy, "stable", 3); assert_eq!(first, backoff_delay_ms(&policy, "stable", 3)); diff --git a/kernel/relayflowd-core/src/spec.rs b/kernel/relayflowd-core/src/spec.rs index bc5f4dae3..cb4ef0dae 100644 --- a/kernel/relayflowd-core/src/spec.rs +++ b/kernel/relayflowd-core/src/spec.rs @@ -674,6 +674,14 @@ pub struct RetryPolicy { pub multiplier: u32, #[serde(default = "default_jitter_percent")] pub jitter_percent: u8, + /// Additional attempts allowed only after infrastructure loss + /// (`crashed`/`lease_expired`). Semantic failures never consume this + /// budget, and ordinary worker nonzero exits never qualify for it. + #[serde( + default = "default_max_transport_retries", + skip_serializing_if = "is_default_max_transport_retries" + )] + pub max_transport_retries: u32, } impl Default for RetryPolicy { @@ -683,6 +691,7 @@ impl Default for RetryPolicy { max_backoff_ms: default_max_backoff_ms(), multiplier: default_multiplier(), jitter_percent: default_jitter_percent(), + max_transport_retries: default_max_transport_retries(), } } } @@ -715,6 +724,14 @@ fn default_jitter_percent() -> u8 { 20 } +fn default_max_transport_retries() -> u32 { + 1 +} + +fn is_default_max_transport_retries(value: &u32) -> bool { + *value == default_max_transport_retries() +} + #[derive(Debug, Error, PartialEq, Eq)] pub enum SpecError { #[error("step {step} has invalid memory: {detail}")] diff --git a/kernel/relayflowd/src/server/tests.rs b/kernel/relayflowd/src/server/tests.rs index c960de3c9..6feb85da0 100644 --- a/kernel/relayflowd/src/server/tests.rs +++ b/kernel/relayflowd/src/server/tests.rs @@ -57,7 +57,8 @@ fn start_llm_run(data_dir: &Path, hub: &Arc) -> String { "initial_backoff_ms": 10, "max_backoff_ms": 10, "multiplier": 1, - "jitter_percent": 0 + "jitter_percent": 0, + "max_transport_retries": 1 } }] }); diff --git a/kernel/relayflowd/src/server/tests/agent/contract.rs b/kernel/relayflowd/src/server/tests/agent/contract.rs index 01ff7ffe3..2040001a6 100644 --- a/kernel/relayflowd/src/server/tests/agent/contract.rs +++ b/kernel/relayflowd/src/server/tests/agent/contract.rs @@ -334,7 +334,7 @@ fn a_replacement_worker_that_never_reported_the_pinned_surface_is_not_dispatched "type": "agent", "instruction": "edit", "max_iterations": 3, - "retry": {"initial_backoff_ms": 0, "max_backoff_ms": 0, "multiplier": 1, "jitter_percent": 0}, + "retry": {"initial_backoff_ms": 0, "max_backoff_ms": 0, "multiplier": 1, "jitter_percent": 0, "max_transport_retries": 1}, "surfaces": {"workspace": [{"surface": "repo"}]} }] }); diff --git a/kernel/relayflowd/src/server/tests/agent/pins.rs b/kernel/relayflowd/src/server/tests/agent/pins.rs index db296116d..df898499b 100644 --- a/kernel/relayflowd/src/server/tests/agent/pins.rs +++ b/kernel/relayflowd/src/server/tests/agent/pins.rs @@ -260,7 +260,7 @@ fn a_replacement_worker_at_a_different_revision_is_not_dispatched_the_stale_pins // already be standing at the pinned revision. "recovery_mode": "inspect", "max_iterations": 1, - "retry": {"initial_backoff_ms": 0, "max_backoff_ms": 0, "multiplier": 1, "jitter_percent": 0}, + "retry": {"initial_backoff_ms": 0, "max_backoff_ms": 0, "multiplier": 1, "jitter_percent": 0, "max_transport_retries": 1}, "surfaces": {"workspace": [{"surface": "repo"}]} }] }); diff --git a/kernel/relayflowd/tests/crash_resume/pin_projection.rs b/kernel/relayflowd/tests/crash_resume/pin_projection.rs index 157cc9807..eb4c45e3a 100644 --- a/kernel/relayflowd/tests/crash_resume/pin_projection.rs +++ b/kernel/relayflowd/tests/crash_resume/pin_projection.rs @@ -1,6 +1,6 @@ //! Rejected agent evidence cannot advance either durable or live pin state. -use relayflowd_core::{CompletionReason, EntryType, StepCompletedPayload}; +use relayflowd_core::{CompletionReason, Disposition, EntryType, StepCompletedPayload}; use serde_json::json; use super::{ @@ -9,7 +9,7 @@ use super::{ }; #[test] -fn rejected_completion_cannot_forge_inspect_retry_pins_over_the_real_socket() { +fn rejected_completion_cannot_forge_pins_or_trigger_a_blind_retry() { let fixture = LlmFixture::parallel("rejected-pin-projection"); let _server = ServerGuard::start(&fixture); let socket = fixture.socket(); @@ -66,12 +66,7 @@ fn rejected_completion_cannot_forge_inspect_retry_pins_over_the_real_socket() { }), ) .unwrap(); - assert_eq!(rejected["status"], "parked"); - - let retry = worker.event("step.dispatch").unwrap(); - assert_eq!(retry["attempt"], 2); - assert_eq!(retry["pins"], first["pins"]); - assert_eq!(retry["recovery"]["mode"], "inspect"); + assert_eq!(rejected["status"], "failed"); let rejected_fact = journal_entries(&fixture.data_dir) .unwrap() @@ -84,5 +79,6 @@ fn rejected_completion_cannot_forge_inspect_retry_pins_over_the_real_socket() { .unwrap(); let payload: StepCompletedPayload = serde_json::from_value(rejected_fact.payload).unwrap(); assert_eq!(payload.completion_reason, CompletionReason::WorkerError); + assert_eq!(payload.disposition, Disposition::StepDone); assert_eq!(payload.end_pins, None); } diff --git a/kernel/relayflowd/tests/parallel_driver.rs b/kernel/relayflowd/tests/parallel_driver.rs index af9f82ba3..a0b3c4147 100644 --- a/kernel/relayflowd/tests/parallel_driver.rs +++ b/kernel/relayflowd/tests/parallel_driver.rs @@ -358,7 +358,7 @@ fn backpressured_or_mismatched_lane_does_not_drop_a_later_dispatch() { .start(parallel_llm_spec(), "test", None) .unwrap() .status, - RunStatus::Parked + RunStatus::Failed ); assert_eq!( dispatcher @@ -366,8 +366,8 @@ fn backpressured_or_mismatched_lane_does_not_drop_a_later_dispatch() { .iter() .map(|dispatch| (dispatch.step_id.as_str(), dispatch.attempt)) .collect::>(), - [("lane-b", 1), ("lane-a", 1), ("lane-b", 2), ("lane-a", 2)], - "a compatible replacement must receive due retries without an external resume" + [("lane-b", 1), ("lane-a", 1)], + "a pin mismatch must reach the later lane, then fail closed without a blind retry" ); } diff --git a/packages/sdk/src/authored-root.ts b/packages/sdk/src/authored-root.ts index eb8adc78c..c91e1825c 100644 --- a/packages/sdk/src/authored-root.ts +++ b/packages/sdk/src/authored-root.ts @@ -240,11 +240,10 @@ async function driveRoot( } /** - * A returned body failure is deterministic for this root attempt. Complete - * every kernel retry with the same declared failure so the run reaches its - * durable `step_failed` terminal instead of being stranded waiting for a - * worker after the CLI has already returned. A process crash never enters - * this path: the running attempt remains recoverable by `run.resume`. + * A returned body failure is deterministic for this root attempt and is + * completed once as terminal `worker_error`. A process crash never enters + * this path: the running attempt remains recoverable by `run.resume` under + * the root's separate transport retry budget. */ async function terminalizeRootFailure( peer: JournalClient, @@ -290,7 +289,7 @@ function rootSpec(metadata: AuthoredRootMetadata, stream: string) { id: 'authored-root', type: 'agent', instruction: canonicalize(metadata), surfaces: { streams: [{ stream }] }, - recoveryMode: 'reset', maxIterations: 8, + recoveryMode: 'reset', maxIterations: 1, transportRetries: 7, }], })); } diff --git a/packages/sdk/src/authored-worker-step.ts b/packages/sdk/src/authored-worker-step.ts index bd85fac07..ab9a7a201 100644 --- a/packages/sdk/src/authored-worker-step.ts +++ b/packages/sdk/src/authored-worker-step.ts @@ -157,6 +157,29 @@ export function authoredWorkerRunner( `f.agent options.transport must be 'direct' or 'relay' (got ${JSON.stringify(options.transport)}).`, ); } + if (options.maxIterations !== undefined + && (!Number.isSafeInteger(options.maxIterations) || options.maxIterations < 1)) { + throw new AuthoredFlowExecutionError( + 'agent_cli_unresolved', + `f.agent options.maxIterations must be a positive integer (got ${JSON.stringify(options.maxIterations)}).`, + ); + } + if (options.transportRetries !== undefined + && (!Number.isSafeInteger(options.transportRetries) || options.transportRetries < 0)) { + throw new AuthoredFlowExecutionError( + 'agent_cli_unresolved', + `f.agent options.transportRetries must be a non-negative integer (got ${JSON.stringify(options.transportRetries)}).`, + ); + } + if (options.recoveryMode !== undefined + && options.recoveryMode !== 'reset' + && options.recoveryMode !== 'inspect' + && options.recoveryMode !== 'manual') { + throw new AuthoredFlowExecutionError( + 'agent_cli_unresolved', + `f.agent options.recoveryMode must be 'reset', 'inspect', or 'manual' (got ${JSON.stringify(options.recoveryMode)}).`, + ); + } const permissions = options.permissions; const permissionsSnapshot = permissions === undefined ? undefined : snapshotJsonValue(permissions, 'f.agent options.permissions') as unknown as PermissionsSpec; @@ -169,6 +192,9 @@ export function authoredWorkerRunner( ...(options.model === undefined ? {} : { model: options.model }), ...(options.cwd === undefined ? {} : { cwd: options.cwd }), ...(options.transport === undefined ? {} : { transport: options.transport }), + ...(options.maxIterations === undefined ? {} : { maxIterations: options.maxIterations }), + ...(options.transportRetries === undefined ? {} : { transportRetries: options.transportRetries }), + ...(options.recoveryMode === undefined ? {} : { recoveryMode: options.recoveryMode }), ...(verification === undefined ? {} : { verification }), }); if (typeof output !== 'object' || output === null || Array.isArray(output)) { @@ -260,7 +286,13 @@ function stepDetails( ...(found.completionReason === undefined ? {} : { completionReason: found.completionReason }), ...(found.attempt === undefined ? {} : { attempt: found.attempt }), ...(found.maxIterations === undefined ? {} : { maxIterations: found.maxIterations }), + ...(found.transportRetries === undefined ? {} : { transportRetries: found.transportRetries }), ...(found.exitCode === undefined ? {} : { exitCode: found.exitCode }), + ...(found.transportPhase === undefined ? {} : { transportPhase: found.transportPhase }), + ...(found.transportCause === undefined ? {} : { transportCause: found.transportCause }), + ...(found.signal === undefined ? {} : { signal: found.signal }), + ...(found.errorCode === undefined ? {} : { errorCode: found.errorCode }), + ...(found.retryableTransport === undefined ? {} : { retryableTransport: found.retryableTransport }), ...(found.stdoutTail === undefined ? {} : { stdoutTail: found.stdoutTail }), ...(found.stderrTail === undefined ? {} : { stderrTail: found.stderrTail }), ...(found.detail === undefined ? {} : { detail: found.detail }), diff --git a/packages/sdk/src/cli-transport-evidence.ts b/packages/sdk/src/cli-transport-evidence.ts new file mode 100644 index 000000000..375540d38 --- /dev/null +++ b/packages/sdk/src/cli-transport-evidence.ts @@ -0,0 +1,70 @@ +import { boundedText, redactText } from './agent-transcript.js'; + +export type CliTransportPhase = 'spawn' | 'close' | 'abort' | 'timeout' | 'result_exit_grace'; +export type CliTransportCause = + | 'exited' + | 'nonzero_exit' + | 'signal' + | 'close_without_status' + | 'spawn_error' + | 'process_error' + | 'lease_lost' + | 'timeout' + | 'result_exit_timeout' + | 'codex_stdin_lifecycle'; + +export interface CliTransportEvidence { + phase: CliTransportPhase; + cause: CliTransportCause; + exit_code: number | null; + signal: NodeJS.Signals | null; + /** OS/libuv error code, never an unbounded error message. */ + error_code?: string; + /** True only for a closed set of infrastructure failures. */ + retryable: boolean; + /** Redacted UTF-8 tail with its truncation made explicit. */ + stderr_tail: string; +} + +/** Infrastructure classification is deliberately narrower than "nonzero". */ +export function agentCompletionReason( + result: { exit_code: number | null; transport?: CliTransportEvidence }, +): 'success' | 'crashed' | 'timeout' | 'worker_error' { + if (result.exit_code === 0) return 'success'; + if (result.transport?.retryable === true) return 'crashed'; + if (result.transport?.cause === 'timeout') return 'timeout'; + return 'worker_error'; +} + +const TRANSPORT_STDERR_MAX_BYTES = 2 * 1024; +const RETRYABLE_SPAWN_ERROR_CODES = new Set(['EAGAIN', 'EMFILE', 'ENFILE', 'ENOMEM', 'ETXTBSY']); + +export function isRetryableSpawnError(code: string | undefined): boolean { + return code !== undefined && RETRYABLE_SPAWN_ERROR_CODES.has(code); +} + +export function transportEvidence( + value: { + phase: CliTransportPhase; + cause: CliTransportCause; + exitCode: number | null; + signal: NodeJS.Signals | null; + stderr: string; + errorCode?: string; + retryable: boolean; + }, + env: NodeJS.ProcessEnv, +): CliTransportEvidence { + const bounded = boundedText(redactText(value.stderr, env), TRANSPORT_STDERR_MAX_BYTES, 'transport stderr: '); + const errorCode = value.errorCode !== undefined && /^[A-Z0-9_-]{1,64}$/.test(value.errorCode) + ? value.errorCode : undefined; + return { + phase: value.phase, + cause: value.cause, + exit_code: value.exitCode, + signal: value.signal, + ...(errorCode === undefined ? {} : { error_code: errorCode }), + retryable: value.retryable, + stderr_tail: bounded.text, + }; +} diff --git a/packages/sdk/src/cli/step-failure.ts b/packages/sdk/src/cli/step-failure.ts index d04e10ba1..a2b886268 100644 --- a/packages/sdk/src/cli/step-failure.ts +++ b/packages/sdk/src/cli/step-failure.ts @@ -41,6 +41,7 @@ export async function stepFailureDetails( // enforcing (relayflowd-core/src/entry.rs `AttemptStartedPayload`). They are // collected on the same walk and reported only when the journal held them. const budgets = new Map(); + const transportBudgets = new Map(); while (true) { const { entries } = await client.journalRead(runId, fromSeq, 100); if (entries.length === 0) break; @@ -58,6 +59,10 @@ export async function stepFailureDetails( if (typeof maxIterations === 'number' && Number.isSafeInteger(maxIterations) && maxIterations > 0) { budgets.set(stepId, maxIterations); } + const transportRetries = record(entry['payload'])?.['max_transport_retries']; + if (typeof transportRetries === 'number' && Number.isSafeInteger(transportRetries) && transportRetries >= 0) { + transportBudgets.set(stepId, transportRetries); + } continue; } if (entry['entry_type'] !== 'step.completed') continue; @@ -76,12 +81,14 @@ export async function stepFailureDetails( const stepType = snapshot.steps[stepId]?.type; const attempt = entry['attempt']; const maxIterations = budgets.get(stepId); + const transportRetries = transportBudgets.get(stepId); failures.set(stepId, { stepId, completionReason, ...(stepType === undefined ? {} : { stepType }), ...(typeof attempt === 'number' && Number.isSafeInteger(attempt) && attempt > 0 ? { attempt } : {}), ...(maxIterations === undefined ? {} : { maxIterations }), + ...(transportRetries === undefined ? {} : { transportRetries }), ...evidence(payload), }); } @@ -114,7 +121,13 @@ export function renderStepEvidence(details: StepFailedDetails): string { + (details.stepType === undefined ? '' : ` (${details.stepType})`) + ` completionReason: ${details.completionReason}` + renderAttempt(details) + + (details.transportRetries === undefined ? '' : ` transportRetries=${details.transportRetries}`) + (details.exitCode === undefined ? '' : ` exit=${details.exitCode}`) + + (details.signal === undefined ? '' : ` signal=${details.signal}`) + + (details.transportCause === undefined ? '' : ` transport=${details.transportCause}`) + + (details.transportPhase === undefined ? '' : ` phase=${details.transportPhase}`) + + (details.errorCode === undefined ? '' : ` error_code=${details.errorCode}`) + + (details.retryableTransport === undefined ? '' : ` retryable=${details.retryableTransport}`) + '.' + (details.detail === undefined ? '' : `\nDetail: ${details.detail}`) + (details.stdoutTail ? `\nStdout (last 1,024 bytes):\n${details.stdoutTail}` : '') @@ -189,9 +202,10 @@ function evidence(payload: Record): Partial typeof detail === 'string' ? parsed(detail) : undefined, ].filter((candidate): candidate is Record => candidate !== undefined); const structured = candidates.find(processShaped) ?? candidates[0]; - const exitCode = structured?.['exit_code']; + const transport = record(record(payload['trajectory_tail'])?.['transport']); + const exitCode = structured?.['exit_code'] ?? transport?.['exit_code']; const stdout = structured?.['stdout_tail']; - const stderr = structured?.['stderr_tail']; + const stderr = structured?.['stderr_tail'] ?? transport?.['stderr_tail']; const structuredShape = structured !== undefined && processShaped(structured); const transcript = record(record(payload['trajectory_tail'])?.['transcript']); const failure = record(transcript?.['failure']); @@ -204,6 +218,11 @@ function evidence(payload: Record): Partial ? tail(excerpt) : undefined; return { ...(typeof exitCode === 'number' && Number.isSafeInteger(exitCode) ? { exitCode } : {}), + ...(typeof transport?.['phase'] === 'string' ? { transportPhase: tail(transport['phase']) } : {}), + ...(typeof transport?.['cause'] === 'string' ? { transportCause: tail(transport['cause']) } : {}), + ...(typeof transport?.['signal'] === 'string' ? { signal: tail(transport['signal']) } : {}), + ...(typeof transport?.['error_code'] === 'string' ? { errorCode: tail(transport['error_code']) } : {}), + ...(typeof transport?.['retryable'] === 'boolean' ? { retryableTransport: transport['retryable'] } : {}), ...(typeof stdout === 'string' && stdout.length > 0 ? { stdoutTail: tail(stdout) } : {}), ...(typeof stderr === 'string' ? { stderrTail: tail(stderr) } : {}), // Keep the daemon's account only when it was NOT just a render of the diff --git a/packages/sdk/src/compile.ts b/packages/sdk/src/compile.ts index 4f454b437..a349ee725 100644 --- a/packages/sdk/src/compile.ts +++ b/packages/sdk/src/compile.ts @@ -182,6 +182,7 @@ function compileStep(step: StepSpec): StepSpec { ? { dependsOn: step.input === undefined ? step.dependsOn : [...new Set([...(step.dependsOn ?? []), ...bindingDependencies(step.input)])] } : {}), ...(step.input !== undefined ? { input: step.input } : {}), maxIterations, + ...(step.transportRetries !== undefined ? { transportRetries: step.transportRetries } : {}), ...(step.memory !== undefined ? { memory: step.memory } : {}), ...(step.requirements !== undefined ? { requirements: step.requirements } : {}), }; @@ -424,7 +425,7 @@ function kernelStepToAuthoring(value: unknown, at: string): unknown { const unionKeys = [ 'id', 'type', 'depends_on', 'max_iterations', 'retry', 'verification', 'memory', 'requirements', 'input', 'command', 'timeout_ms', 'lease_ms', 'prompt', 'model', 'cli', 'instruction', - 'recovery_mode', 'surfaces', 'permissions', + 'recovery_mode', 'surfaces', 'permissions', 'cwd', 'transport', ] as const; const step = requireKernelObject(value, unionKeys, at); const type = step['type']; @@ -434,10 +435,11 @@ function kernelStepToAuthoring(value: unknown, at: string): unknown { : type === 'llm' ? ['prompt', 'model', 'cli'] as const : type === 'agent' - ? ['instruction', 'cli', 'model', 'recovery_mode', 'surfaces', 'permissions'] as const + ? ['instruction', 'cli', 'model', 'recovery_mode', 'surfaces', 'permissions', 'cwd', 'transport'] as const : []; assertKernelKeys(step, [...commonKeys, ...typeKeys], at); - if (step['retry'] !== undefined) validateAuthoringRetryDefaults(step['retry'], `${at}.retry`); + const transportRetries = step['retry'] === undefined + ? undefined : validateAuthoringRetryDefaults(step['retry'], `${at}.retry`); const dependsOn = step['depends_on']; const common = { id: step['id'], @@ -448,6 +450,7 @@ function kernelStepToAuthoring(value: unknown, at: string): unknown { ? { dependsOn } : {}), ...(step['max_iterations'] !== undefined ? { maxIterations: step['max_iterations'] } : {}), + ...(transportRetries === undefined ? {} : { transportRetries }), ...kernelVerificationToAuthoring(type, step['verification'], `${at}.verification`), ...(step['memory'] !== undefined ? { memory: kernelMemoryToAuthoring(step['memory'], `${at}.memory`) } : {}), }; @@ -467,7 +470,7 @@ function kernelStepToAuthoring(value: unknown, at: string): unknown { ...common, instruction: step['instruction'], ...(step['recovery_mode'] !== undefined ? { recoveryMode: step['recovery_mode'] } : {}), - ...copyDefined(step, ['cli', 'model', 'surfaces']), + ...copyDefined(step, ['cli', 'model', 'surfaces', 'cwd', 'transport']), ...(step['permissions'] !== undefined ? { permissions: kernelPermissionsToAuthoring(step['permissions'], `${at}.permissions`) } : {}), @@ -476,9 +479,9 @@ function kernelStepToAuthoring(value: unknown, at: string): unknown { return common; } -function validateAuthoringRetryDefaults(value: unknown, at: string): void { +function validateAuthoringRetryDefaults(value: unknown, at: string): number | undefined { const retry = requireKernelObject(value, [ - 'initial_backoff_ms', 'max_backoff_ms', 'multiplier', 'jitter_percent', + 'initial_backoff_ms', 'max_backoff_ms', 'multiplier', 'jitter_percent', 'max_transport_retries', ], at); for (const [field, expected] of Object.entries(KERNEL_RETRY_DEFAULTS)) { if (retry[field] !== expected) { @@ -487,6 +490,12 @@ function validateAuthoringRetryDefaults(value: unknown, at: string): void { ]); } } + const transportRetries = retry['max_transport_retries']; + if (transportRetries === undefined) return undefined; + if (typeof transportRetries !== 'number' || !Number.isSafeInteger(transportRetries) || transportRetries < 0) { + throw new CompileError([`${at}.max_transport_retries must be a non-negative integer`]); + } + return transportRetries; } function kernelVerificationToAuthoring( @@ -583,7 +592,11 @@ function toKernelStep(step: StepSpec): KernelStepSpec { : [...new Set([...(step.dependsOn ?? []), ...bindingDependencies(step.input)])], ...(step.input !== undefined ? { input: step.input } : {}), max_iterations: step.maxIterations ?? 1, - retry: { ...KERNEL_RETRY_DEFAULTS }, + retry: { + ...KERNEL_RETRY_DEFAULTS, + ...(step.transportRetries !== undefined + ? { max_transport_retries: step.transportRetries } : {}), + }, verification: toKernelVerification(step), ...(step.requirements !== undefined ? { requirements: { ...Object.fromEntries(Object.entries(step.requirements).filter(([key]) => key !== 'expectedDurationMs')), diff --git a/packages/sdk/src/failure-kinds.ts b/packages/sdk/src/failure-kinds.ts index 05487a909..6a132ba73 100644 --- a/packages/sdk/src/failure-kinds.ts +++ b/packages/sdk/src/failure-kinds.ts @@ -175,7 +175,14 @@ export interface StepFailedDetails { * `retries_exhausted`. */ maxIterations?: number; + /** Additional classified infrastructure retries declared for the step. */ + transportRetries?: number; exitCode?: number; + transportPhase?: string; + transportCause?: string; + signal?: string; + errorCode?: string; + retryableTransport?: boolean; /** Terminal-safe UTF-8 excerpt, at most 1,024 bytes. */ stdoutTail?: string; /** Terminal-safe UTF-8 excerpt, at most 1,024 bytes. */ diff --git a/packages/sdk/src/pty-sidechannel.ts b/packages/sdk/src/pty-sidechannel.ts index 552a995ae..8934db609 100644 --- a/packages/sdk/src/pty-sidechannel.ts +++ b/packages/sdk/src/pty-sidechannel.ts @@ -32,6 +32,9 @@ export async function openSidechannel( ) { const peers = new Map(); let closed = false; + let driveConnected = false; + let announceDrive!: () => void; + const driveConnection = new Promise(resolve => { announceDrive = resolve; }); const server = createServer(socket => { if (peers.size >= 16) { socket.destroy(); return; } peers.set(socket, false); @@ -52,7 +55,11 @@ export async function openSidechannel( socket.setTimeout(0); peers.set(socket, true); // Passthrough is a passive raw-byte view in this initial slice. - if (mode === 'drive') context.onDrive(); + if (mode === 'drive') { + driveConnected = true; + announceDrive(); + context.onDrive(); + } bytes = hello.subarray(end + 1); hello = Buffer.alloc(0); } @@ -89,6 +96,25 @@ export async function openSidechannel( if (ready && !peer.write(bytes)) peer.destroy(); } }, + /** + * Wait for a drive peer before the CLI is spawned. + * + * A direct headless CLI must not be handed a pipe merely because a + * sidechannel exists: Codex treats any non-TTY stdin as an additional + * prompt source and waits for its lifecycle. The caller uses this bounded + * enrollment window to choose `pipe` only when a driver actually joined; + * view and passthrough peers never change the child's stdin contract. + */ + async waitForDrive(timeoutMs: number): Promise { + if (driveConnected) return true; + let timer: NodeJS.Timeout | undefined; + await Promise.race([ + driveConnection, + new Promise(resolve => { timer = setTimeout(resolve, timeoutMs); }), + ]); + if (timer !== undefined) clearTimeout(timer); + return driveConnected; + }, close() { if (closed) return; closed = true; diff --git a/packages/sdk/src/spec.ts b/packages/sdk/src/spec.ts index 822ddf664..b064374ef 100644 --- a/packages/sdk/src/spec.ts +++ b/packages/sdk/src/spec.ts @@ -206,6 +206,8 @@ export interface BaseStepSpec { dependsOn?: string[]; /** Semantic retry bound (kernel DESIGN.md §1.2 `max_iterations`). Default 1. */ maxIterations?: number; + /** Additional attempts after classified infrastructure loss. Default 1. */ + transportRetries?: number; } export interface OutputBinding { @@ -410,6 +412,7 @@ export interface KernelRetryPolicy { max_backoff_ms: number; multiplier: number; jitter_percent: number; + max_transport_retries?: number; } /** diff --git a/packages/sdk/src/step-fields.ts b/packages/sdk/src/step-fields.ts index d51e1a273..27cf8965a 100644 --- a/packages/sdk/src/step-fields.ts +++ b/packages/sdk/src/step-fields.ts @@ -25,6 +25,7 @@ export const STEP_COMMON_FIELDS = [ 'input', 'verification', 'maxIterations', + 'transportRetries', 'memory', 'requirements', ] as const; diff --git a/packages/sdk/src/validate.ts b/packages/sdk/src/validate.ts index 09b733c1a..ff370ae88 100644 --- a/packages/sdk/src/validate.ts +++ b/packages/sdk/src/validate.ts @@ -397,6 +397,12 @@ class Validator { if (st['maxIterations'] !== undefined && !isPosInt(st['maxIterations'])) { this.fail(`${at}.maxIterations: expected a positive integer`); } + if (st['transportRetries'] !== undefined + && (typeof st['transportRetries'] !== 'number' + || !Number.isSafeInteger(st['transportRetries']) + || st['transportRetries'] < 0)) { + this.fail(`${at}.transportRetries: expected a non-negative integer`); + } if (type === 'deterministic') { this.validateDeterministic(st as unknown as DeterministicStepSpec, at); diff --git a/packages/sdk/src/worker-cli-relay.ts b/packages/sdk/src/worker-cli-relay.ts new file mode 100644 index 000000000..2ed0ad235 --- /dev/null +++ b/packages/sdk/src/worker-cli-relay.ts @@ -0,0 +1,51 @@ +import { redactRelayError } from './redact.js'; +import { + runAgentRelayTask, + AgentRelayTransportError, +} from './agent-relay-transport.js'; +import type { CliAdapterKind } from './cli-adapter.js'; +import { requirePricedUsage } from './worker-usage.js'; +import type { WorkerCliResult } from './worker-cli.js'; + +/** Journal identity and durable dispatch storage used by the Relay task transport. */ +export interface AgentRelayContext { + runId: string; + stepId: string; + idempotencyKey: string; + dataDir?: string; + resultSchema?: unknown; +} + +/** Wait under the same worker lease for an authoritative task receipt. */ +export async function runViaAgentRelay( + kind: CliAdapterKind, instruction: string, wakeContext: unknown, + model: string | undefined, context: AgentRelayContext | undefined, + worker_cwd: string | undefined, signal: AbortSignal | undefined, +): Promise { + try { + if (kind === 'relayflows-wrapper-v1') throw new Error('Relay task transport does not support same-process wrappers.'); + if (!context?.dataDir) throw new Error('Relay task transport requires a durable data directory and journal dispatch identity.'); + const task = instruction + (wakeContext === undefined ? '' : `\n\nWake context (journaled):\n${JSON.stringify(wakeContext)}`) + + '\n\nReport the final task output with the injected agent_result tool and final=true. Wait for its successful durable acknowledgment before exiting.'; + const received = await runAgentRelayTask({ + cli: kind, task, model, worker_cwd, result_schema: context.resultSchema, + runId: context.runId, stepId: context.stepId, idempotencyKey: context.idempotencyKey, + dataDir: context.dataDir, + }, { signal }); + const receipt = { ...received, error: received.error === null ? null : redactRelayError(received.error) }; + const accounting = receipt.task_execution.accounting; + const result: WorkerCliResult = { + relay_task: receipt, exit_code: receipt.status === 'completed' ? 0 : 1, + stdout_tail: receipt.status === 'completed' ? JSON.stringify(receipt.output) : '', + stderr_tail: receipt.status === 'failed' ? `Relay task failed: ${receipt.error}` : '', + ...(accounting?.tokens_input === undefined ? {} : { tokens_input: accounting.tokens_input }), + ...(accounting?.tokens_output === undefined ? {} : { tokens_output: accounting.tokens_output }), + }; + return requirePricedUsage(result, model); + } catch (error) { + signal?.throwIfAborted(); + const detail = error instanceof AgentRelayTransportError ? error.message + : error instanceof Error ? error.message : 'Relay task transport failed'; + return { exit_code: null, stdout_tail: '', stderr_tail: redactRelayError(detail) }; + } +} diff --git a/packages/sdk/src/worker-cli.ts b/packages/sdk/src/worker-cli.ts index 4b95d30dc..b245f579b 100644 --- a/packages/sdk/src/worker-cli.ts +++ b/packages/sdk/src/worker-cli.ts @@ -3,7 +3,14 @@ import { resolve, sep } from 'node:path'; import { diffWorkspaceFiles, snapshotWorkspaceFiles } from './agent-artifacts.js'; import { claudeResultOutcome, decodeProviderResult, decodeWrapperResult, requirePricedUsage } from './worker-usage.js'; import { openSidechannel, type SidechannelContext } from './pty-sidechannel.js'; -import { openTranscriptWriter, transcriptPath, type TranscriptDigest, type TranscriptFile, type TranscriptWriter } from './agent-transcript.js'; +import { + openTranscriptWriter, + redactText, + transcriptPath, + type TranscriptDigest, + type TranscriptFile, + type TranscriptWriter, +} from './agent-transcript.js'; import { spawn } from 'node:child_process'; import { StringDecoder } from 'node:string_decoder'; import { childStop } from './child-stop.js'; @@ -21,14 +28,20 @@ import { type WrapperSessionLimits, } from './wrapper-session.js'; import { wrapperEnvironment } from './wrapper-runtime.js'; -import { redactRelayError } from './redact.js'; import { applyStepEnvironment } from './step-env.js'; import { TAIL_CLOSE_TIMEOUT_MS, openTranscriptTail, type TranscriptTailWriter } from './transcript-tail.js'; +import type { AgentTransport } from './agent-relay-transport.js'; import { - runAgentRelayTask, - AgentRelayTransportError, - type AgentTransport, -} from './agent-relay-transport.js'; + isRetryableSpawnError, + transportEvidence, + type CliTransportCause, + type CliTransportEvidence, +} from './cli-transport-evidence.js'; +import { runViaAgentRelay, type AgentRelayContext } from './worker-cli-relay.js'; + +export type { CliTransportCause, CliTransportEvidence, CliTransportPhase } from './cli-transport-evidence.js'; +export { agentCompletionReason } from './cli-transport-evidence.js'; +export type { AgentRelayContext } from './worker-cli-relay.js'; /** Present only when a dispatched agent step carries a journaled wake context. */ export const WAKE_CONTEXT_ENV = 'RELAYFLOW_WAKE_CONTEXT'; @@ -47,6 +60,12 @@ export interface WorkerCliResult { exit_code: number | null; stdout_tail: string; stderr_tail: string; + /** + * Bounded, redacted process-lifecycle evidence for a direct CLI spawn. + * This is journaled in `trajectory_tail`, including when parsed JSON output + * would otherwise discard the process wrapper. + */ + transport?: CliTransportEvidence; /** * Files the agent created or changed under its working directory, * cwd-relative POSIX paths, sorted. Measured by the worker that spawned the @@ -68,17 +87,6 @@ export interface WorkerCliResult { transcript?: TranscriptDigest; } -/** - * Journal identity and durable dispatch storage used by the Relay task transport. - */ -export interface AgentRelayContext { - runId: string; - stepId: string; - idempotencyKey: string; - dataDir?: string; - resultSchema?: unknown; -} - export async function runAgentCli( cli: string, instruction: string, @@ -175,7 +183,9 @@ export async function runAgentCli( const args = [...invocation.args]; args.splice(args.length - 1, 0, ...(kind === 'claude' ? ['--output-format', 'stream-json', '--verbose'] : ['--json'])); const completion = kind === 'claude' ? claudeResultOutcome : undefined; - return requirePricedUsage(decodeProviderResult(await spawnInvocation(cli, { ...invocation, args }, env, signal, sidechannel, cwd, completion), kind, env), effectiveModel); + return requirePricedUsage(decodeProviderResult(await spawnInvocation( + cli, { ...invocation, args }, env, signal, sidechannel, cwd, completion, kind, + ), kind, env), effectiveModel); } } @@ -231,40 +241,6 @@ function serializedByDirectory(directory: string, task: () => Promise): Pr return run; } -/** Wait under the same worker lease for an authoritative task receipt. */ -async function runViaAgentRelay( - kind: CliAdapterKind, instruction: string, wakeContext: unknown, - model: string | undefined, context: AgentRelayContext | undefined, - worker_cwd: string | undefined, signal: AbortSignal | undefined, -): Promise { - try { - if (kind === 'relayflows-wrapper-v1') throw new Error('Relay task transport does not support same-process wrappers.'); - if (!context?.dataDir) throw new Error('Relay task transport requires a durable data directory and journal dispatch identity.'); - const task = instruction + (wakeContext === undefined ? '' : `\n\nWake context (journaled):\n${JSON.stringify(wakeContext)}`) - + '\n\nReport the final task output with the injected agent_result tool and final=true. Wait for its successful durable acknowledgment before exiting.'; - const received = await runAgentRelayTask({ - cli: kind, task, model, worker_cwd, result_schema: context.resultSchema, - runId: context.runId, stepId: context.stepId, idempotencyKey: context.idempotencyKey, - dataDir: context.dataDir, - }, { signal }); - const receipt = { ...received, error: received.error === null ? null : redactRelayError(received.error) }; - const accounting = receipt.task_execution.accounting; - const result: WorkerCliResult = { - relay_task: receipt, exit_code: receipt.status === 'completed' ? 0 : 1, - stdout_tail: receipt.status === 'completed' ? JSON.stringify(receipt.output) : '', - stderr_tail: receipt.status === 'failed' ? `Relay task failed: ${receipt.error}` : '', - ...(accounting?.tokens_input === undefined ? {} : { tokens_input: accounting.tokens_input }), - ...(accounting?.tokens_output === undefined ? {} : { tokens_output: accounting.tokens_output }), - }; - return requirePricedUsage(result, model); - } catch (error) { - signal?.throwIfAborted(); - const detail = error instanceof AgentRelayTransportError ? error.message - : error instanceof Error ? error.message : 'Relay task transport failed'; - return { exit_code: null, stdout_tail: '', stderr_tail: redactRelayError(detail) }; - } -} - /** * How long a CLI that has reported its final result may take to exit. Claude * Code in print mode waits, after its result, for every background task it @@ -282,9 +258,15 @@ async function spawnInvocation( sidechannel?: SidechannelContext, cwd?: string, completion?: (line: string) => { failed: boolean } | undefined, + kind?: CliAdapterKind, ): Promise { - let writeInput: (bytes: Buffer) => Promise = async () => false; - let canDrive = () => false; + const pendingInput: Buffer[] = []; + let acceptingDrive = true; + let writeInput: (bytes: Buffer) => Promise = async bytes => { + pendingInput.push(Buffer.from(bytes)); + return true; + }; + let canDrive = () => acceptingDrive; let driven = false; // The transcript file lives beside the PTY socket and is named by attempt. // Its absence (no data dir, no attempt, unwritable dir) costs the step @@ -298,6 +280,12 @@ async function spawnInvocation( ...sidechannel, onDrive() { driven = true; sidechannel.onDrive(); }, }, bytes => writeInput(bytes), () => canDrive()); + // Decide the child's stdin shape before spawn. An unattended Codex process + // gets `/dev/null` from `ignore`, so it never enters the "additional input + // from stdin" path. Only an already-enrolled drive peer gets a writable + // pipe. View/passthrough retain live output without changing stdin. + driven = channel === undefined ? false : await channel.waitForDrive(100); + acceptingDrive = false; // Tee the transcript into bounded tail files beside the socket. Evidence // for `flows status --tail`, never the record; a failure here is a warning. const tails = sidechannel === undefined ? undefined : openTails(sidechannel); @@ -312,24 +300,26 @@ async function spawnInvocation( // `reapOnExit` — reaches the whole agent tree, lease-bound or not. const ownsGroup = process.platform !== 'win32'; const child = spawn(cli, invocation.args, { - stdio: ['pipe', 'pipe', 'pipe'], env, + stdio: [driven ? 'pipe' : 'ignore', 'pipe', 'pipe'], env, detached: ownsGroup, ...(cwd === undefined ? {} : { cwd }), }); - child.stdin.on('error', () => {}); - if (channel === undefined) child.stdin.end(); - canDrive = () => !child.stdin.destroyed && !child.stdin.writableEnded; - // A pipe cannot be reopened after EOF. Give startup subscribers a bounded - // chance to opt into drive, then let unattended/view-only CLIs read EOF. - const inputTimer = channel === undefined ? undefined : setTimeout(() => { - if (!driven) child.stdin.end(); - }, 100); - writeInput = bytes => new Promise(resolve => { - if (!canDrive()) { resolve(false); return; } - // write(false) still accepts the bytes. The completion callback waits - // until they flush; the sidechannel pauses its reader in the meantime. - child.stdin.write(bytes, error => resolve(!error)); - }); + const stdin = child.stdin; + stdin?.on('error', () => {}); + canDrive = () => stdin !== null && !stdin.destroyed && !stdin.writableEnded; + let inputQueue = Promise.resolve(true); + writeInput = bytes => { + inputQueue = inputQueue.then(previousAccepted => { + if (!previousAccepted || !canDrive()) return false; + return new Promise(resolve => { + // write(false) still accepts the bytes. The completion callback + // waits until they flush; the sidechannel pauses its reader. + stdin!.write(bytes, error => resolve(!error)); + }); + }); + return inputQueue; + }; + for (const bytes of pendingInput.splice(0)) void writeInput(bytes); const stop = childStop(child, ownsGroup); const release = ownsGroup ? reapOnExit(stop) : () => {}; const stdout: Buffer[] = []; @@ -342,7 +332,6 @@ async function spawnInvocation( const finish = (result: WorkerCliResult, discardTranscript = false): void => { if (settled) return; settled = true; - if (inputTimer !== undefined) clearTimeout(inputTimer); channel?.close(); if (timer !== undefined) clearTimeout(timer); if (graceTimer !== undefined) clearTimeout(graceTimer); @@ -382,7 +371,11 @@ async function spawnInvocation( }; const onAbort = (): void => { stop.kill(); - finish({ exit_code: null, stdout_tail: '', stderr_tail: 'Agent execution aborted: lease ownership lost.' }, true); + const transport = transportEvidence({ + phase: 'abort', cause: 'lease_lost', exitCode: null, signal: null, + stderr: 'Agent execution aborted: lease ownership lost.', retryable: false, + }, env); + finish({ exit_code: null, stdout_tail: '', stderr_tail: 'Agent execution aborted: lease ownership lost.', transport }, true); }; /** * Same invariant as `wrapper-session.ts`: `'close'` and `'error'` are @@ -409,14 +402,22 @@ async function spawnInvocation( if (graceTimer !== undefined || settled) return; graceTimer = setTimeout(() => { stop.terminate(); + const exitCode = outcome.failed ? 1 : 0; + const stderrText = `${Buffer.concat(stderr).toString('utf8')}\nCLI reported its final result but had not exited ${RESULT_EXIT_GRACE_MS}ms later; its process tree was stopped.`.trim(); + const transport = transportEvidence({ + phase: 'result_exit_grace', cause: 'result_exit_timeout', exitCode, signal: null, + stderr: stderrText, + retryable: false, + }, env); finish({ - exit_code: outcome.failed ? 1 : 0, + exit_code: exitCode, stdout_tail: Buffer.concat(stdout).toString('utf8'), - stderr_tail: `${Buffer.concat(stderr).toString('utf8')}\nCLI reported its final result but had not exited ${RESULT_EXIT_GRACE_MS}ms later; its process tree was stopped.`.trim(), + stderr_tail: redactText(stderrText, env), + transport, }); }, RESULT_EXIT_GRACE_MS); }; - child.stdout.on('data', (chunk: Buffer) => { + child.stdout!.on('data', (chunk: Buffer) => { stdout.push(chunk); channel?.publish(chunk); // The tails take raw bytes, so they are fed before any line splitting @@ -435,29 +436,60 @@ async function spawnInvocation( if (outcome !== undefined) onResult(outcome); } }); - child.stderr.on('data', (chunk: Buffer) => { stderr.push(chunk); channel?.publish(chunk); tails?.stderr.append(chunk); }); - child.once('error', (error) => finishOnChildExit({ - exit_code: null, - stdout_tail: Buffer.concat(stdout).toString('utf8'), - stderr_tail: error.message, - })); + child.stderr!.on('data', (chunk: Buffer) => { stderr.push(chunk); channel?.publish(chunk); tails?.stderr.append(chunk); }); + child.once('error', (error) => { + const code = typeof (error as NodeJS.ErrnoException).code === 'string' + ? (error as NodeJS.ErrnoException).code : undefined; + const retryable = isRetryableSpawnError(code); + const transport = transportEvidence({ + phase: 'spawn', cause: child.pid === undefined ? 'spawn_error' : 'process_error', + exitCode: null, signal: null, stderr: error.message, errorCode: code, retryable, + }, env); + finishOnChildExit({ + exit_code: null, + stdout_tail: Buffer.concat(stdout).toString('utf8'), + stderr_tail: redactText(error.message, env), + transport, + }); + }); child.once('error', () => { if (child.pid === undefined) release(); }); child.once('close', release); - child.once('close', (code) => finishOnChildExit({ - exit_code: code, - stdout_tail: Buffer.concat(stdout).toString('utf8'), - stderr_tail: Buffer.concat(stderr).toString('utf8'), - })); + child.once('close', (code, signalName) => { + const stderrText = Buffer.concat(stderr).toString('utf8'); + const codexStdinLifecycle = kind === 'codex' && code === 1 + && stderrText.trim() === 'Reading additional input from stdin...'; + const cause: CliTransportCause = codexStdinLifecycle ? 'codex_stdin_lifecycle' + : signalName !== null ? 'signal' + : code === null ? 'close_without_status' + : code === 0 ? 'exited' : 'nonzero_exit'; + const retryable = codexStdinLifecycle || signalName !== null || code === null; + const transport = transportEvidence({ + phase: 'close', cause, exitCode: code, signal: signalName, stderr: stderrText, retryable, + }, env); + finishOnChildExit({ + exit_code: code, + stdout_tail: Buffer.concat(stdout).toString('utf8'), + stderr_tail: redactText(stderrText, env), + transport, + }); + }); if (invocation.timeoutMs > 0) { timer = setTimeout(() => { // The stop outlives this settle on purpose: `finish` resolves the step, // but only the forced group kill releases the pipes a leaked descendant // is holding, and until they are released `flows run` cannot exit. stop.terminate(); + const timeoutMessage = `CLI invocation timed out after ${invocation.timeoutMs}ms.`; + const transport = transportEvidence({ + phase: 'timeout', cause: 'timeout', exitCode: null, signal: null, + stderr: `${Buffer.concat(stderr).toString('utf8')}\n${timeoutMessage}`, + retryable: false, + }, env); finish({ exit_code: null, stdout_tail: Buffer.concat(stdout).toString('utf8'), - stderr_tail: `CLI invocation timed out after ${invocation.timeoutMs}ms.`, + stderr_tail: timeoutMessage, + transport, }); }, invocation.timeoutMs); } diff --git a/packages/sdk/src/worker.ts b/packages/sdk/src/worker.ts index cb59599b0..3e86df9f3 100644 --- a/packages/sdk/src/worker.ts +++ b/packages/sdk/src/worker.ts @@ -5,12 +5,19 @@ import type { JournalClient } from './journal-client.js'; import type { Pins, StepDispatchEvent } from './protocol.js'; import type { KernelAgentStep } from './spec.js'; import { runAgentCli } from './worker-cli.js'; +import { agentCompletionReason } from './cli-transport-evidence.js'; import { resolveCliModel } from './cli-adapter.js'; import { withWorkerLease } from './worker-lease.js'; import { workerInstruction } from './worker-input.js'; import { helperCall } from './yaml-helpers.js'; import { completeHelperDispatch } from './yaml-helper-effect.js'; -import { ARTIFACT_PATHS_MAX, boundTranscriptDigest, type TranscriptDigest } from './agent-transcript.js'; +import { + ARTIFACT_PATHS_MAX, + boundTranscriptDigest, + boundedText, + redactText, + type TranscriptDigest, +} from './agent-transcript.js'; export { MODEL_ENV, WAKE_CONTEXT_ENV } from './worker-cli.js'; @@ -119,7 +126,7 @@ export class AgentWorker extends EventEmitter { dataDir: this.options.dataDir, resultSchema: spec.verification?.json_schema }) : Promise.resolve({ exit_code: null, stdout_tail: '', stderr_tail: 'agent step has no declared CLI' })); const { result, usage } = workerSpend(completed, effectiveModel); - const completionReason = result.exit_code === 0 ? 'success' : 'worker_error'; + const completionReason = agentCompletionReason(result); // Output shape: if the CLI's stdout parses as JSON, promote THAT // as the step's `output` value so `json_schema` verification @@ -147,7 +154,13 @@ export class AgentWorker extends EventEmitter { // `transcript` is not part of the wrapper either: it is evidence about the // attempt, journaled in `trajectory_tail` below on success and failure // alike, where the kernel already accepts and bounds it (16 KiB). - const { transcript, ...wrapper } = result; + const { transcript, transport, ...rawWrapper } = result; + // `stderr_tail` is evidence, not authored output. Redact and bound it + // before it reaches either the worker-failure render or the journal. + const wrapper = { + ...rawWrapper, + stderr_tail: boundedText(redactText(rawWrapper.stderr_tail), 2 * 1024, 'worker stderr: ').text, + }; const output = result.relay_task?.status === 'completed' && result.exit_code === 0 ? result.relay_task.output : parseJsonOutput(result.stdout_tail) ?? wrapper; const trajectoryTail = { @@ -156,6 +169,7 @@ export class AgentWorker extends EventEmitter { task_execution: result.relay_task.task_execution, error: result.relay_task.error, } }), ...(transcript === undefined ? {} : { transcript: transcriptDigest(transcript, dispatch.attempt, result) }), + ...(transport === undefined ? {} : { transport }), }; await this.client.stepComplete( diff --git a/packages/sdk/tests/agent-transcript-live.test.ts b/packages/sdk/tests/agent-transcript-live.test.ts index f94f912f1..966b84e98 100644 --- a/packages/sdk/tests/agent-transcript-live.test.ts +++ b/packages/sdk/tests/agent-transcript-live.test.ts @@ -197,7 +197,7 @@ describe('the transcript digest through the built CLI, a real daemon and the loc // `attempt=1/1` is read off the journal, not the spec: this step had one // attempt against a budget of one, which is what `retries_exhausted`-style // reasons mean and what the bare reason could not say. - expect(failed.message).toContain('Step "agent-1" (agent) completionReason: worker_error attempt=1/1 exit=1.'); + expect(failed.message).toContain('Step "agent-1" (agent) completionReason: worker_error attempt=1/1 transportRetries=1 exit=1'); expect(failed.message).toContain('\nDetail: gave up: [redacted:FAKE_TOKEN]\n'); const transcriptPath = /\nTranscript: (\S+attempt-1\.transcript\.jsonl)\n/.exec(failed.message)?.[1]; expect(transcriptPath).toBe(join(f.root, 'data', 'runs', report.runId, 'steps', 'agent-1', 'attempt-1.transcript.jsonl')); diff --git a/packages/sdk/tests/authored-agent-permissions.test.ts b/packages/sdk/tests/authored-agent-permissions.test.ts index 70ca71c3f..eff962eea 100644 --- a/packages/sdk/tests/authored-agent-permissions.test.ts +++ b/packages/sdk/tests/authored-agent-permissions.test.ts @@ -101,6 +101,21 @@ process.stdout.write('unused'); expect(step.permissions).toEqual({ access_preset: 'readonly' }); }); + it('lowers explicit semantic, transport, and recovery controls without changing defaults', async () => { + const declared = await capture({ + task: 'x', maxIterations: 3, transportRetries: 2, recoveryMode: 'inspect', + }); + expect(declared).toMatchObject({ + max_iterations: 3, + retry: { max_transport_retries: 2 }, + recovery_mode: 'inspect', + }); + + const defaults = await capture({ task: 'x' }); + expect(defaults).toMatchObject({ max_iterations: 1, recovery_mode: 'reset' }); + expect(defaults.retry).not.toHaveProperty('max_transport_retries'); + }); + it('reads the outer permissions property once', async () => { const getter = vi.fn(() => ({ fileGlobs: ['drafts/**'] })); const step = await capture({ task: 'x', get permissions() { return getter(); } }); diff --git a/packages/sdk/tests/authored-flow.test.ts b/packages/sdk/tests/authored-flow.test.ts index ebe1fb449..19739637b 100644 --- a/packages/sdk/tests/authored-flow.test.ts +++ b/packages/sdk/tests/authored-flow.test.ts @@ -217,6 +217,22 @@ describe('authored flow journal executor', () => { } }); + it('refuses invalid f.agent retry and recovery controls before contacting the journal', async () => { + const disconnectedJournal = new JournalClient('/journal-must-not-be-contacted'); + for (const [field, value, message] of [ + ['maxIterations', 0, 'positive integer'], + ['transportRetries', -1, 'non-negative integer'], + ['recoveryMode', 'continue', "'reset', 'inspect', or 'manual'"], + ] as const) { + await expect(executeAuthoredFlow(flow(`agent-invalid-${field}`, async f => { + await f.agent('worker', { task: 'x', [field]: value } as never); + f.done('success'); + }), disconnectedJournal)).rejects.toMatchObject({ + code: 'agent_cli_unresolved', message: expect.stringContaining(message), + }); + } + }); + it('rejects invalid raw headers before the executor can contact the journal', async () => { const disconnectedJournal = new JournalClient('/journal-must-not-be-contacted'); diff --git a/packages/sdk/tests/authored-root.test.ts b/packages/sdk/tests/authored-root.test.ts index bee894829..b0ab31108 100644 --- a/packages/sdk/tests/authored-root.test.ts +++ b/packages/sdk/tests/authored-root.test.ts @@ -63,9 +63,7 @@ class RootPeer extends EventEmitter { ): Promise { this.completions.push({ attempt, reason }); if (reason === 'success') return outcome(runId, 'completed', 'success'); - if (attempt >= 8) return outcome(runId, 'failed', 'step_failed'); - queueMicrotask(() => this.emit('step.dispatch', dispatch(runId, attempt + 1))); - return outcome(runId, 'parked', null); + return outcome(runId, 'failed', 'step_failed'); } } @@ -224,7 +222,7 @@ describe('durable authored root', () => { expect(journal.peer.completions).toEqual([{ attempt: 2, reason: 'success' }]); }); - it('drives declared root retries to a durable terminal after a body failure', async () => { + it('terminalizes a returned body failure without replaying semantic side effects', async () => { const loaded = await fixture(true); const journal = new RootJournal(); @@ -232,9 +230,7 @@ describe('durable authored root', () => { loaded, journal as unknown as JournalClient, undefined, { dataDir: '/unused', admissionKey: 'failure' }, )).rejects.toThrow('child failed'); - expect(journal.peer.completions).toEqual(Array.from({ length: 8 }, (_, index) => ({ - attempt: index + 1, reason: 'worker_error', - }))); + expect(journal.peer.completions).toEqual([{ attempt: 1, reason: 'worker_error' }]); }); it('renews the authored root lease while its body is still running', async () => { diff --git a/packages/sdk/tests/authored-run-failure-evidence.test.ts b/packages/sdk/tests/authored-run-failure-evidence.test.ts index 0160b9a5e..a40f18530 100644 --- a/packages/sdk/tests/authored-run-failure-evidence.test.ts +++ b/packages/sdk/tests/authored-run-failure-evidence.test.ts @@ -23,7 +23,8 @@ async function openRoot(journal: JournalClient): Promise { steps: [{ id: 'authored-root', type: 'agent', instruction: '{}', surfaces: { streams: [{ stream: 'evidence-root-stream' }] }, - recovery_mode: 'reset', max_iterations: 8, + recovery_mode: 'reset', max_iterations: 1, + retry: { max_transport_retries: 7 }, }], } as never); return outcome.run_id; diff --git a/packages/sdk/tests/deterministic-llm.test.ts b/packages/sdk/tests/deterministic-llm.test.ts index 158ed0fef..dc4770601 100644 --- a/packages/sdk/tests/deterministic-llm.test.ts +++ b/packages/sdk/tests/deterministic-llm.test.ts @@ -91,6 +91,7 @@ steps: instruction: "Edit the repo per the plan." recoveryMode: inspect maxIterations: 2 + transportRetries: 1 surfaces: workspace: - surface: repo @@ -111,6 +112,7 @@ describe('compile: agent step (ladder rung c, Appendix A surface)', () => { expect(act.instruction).toBe('Edit the repo per the plan.'); expect(act.recoveryMode).toBe('inspect'); expect(act.maxIterations).toBe(2); + expect(act.transportRetries).toBe(1); expect(act.surfaces?.workspace).toEqual([{ surface: 'repo' }]); expect(act.surfaces?.streams).toEqual([{ stream: 'results' }]); expect(act.surfaces?.external).toEqual(['pr://github/example']); diff --git a/packages/sdk/tests/pty-sidechannel.test.ts b/packages/sdk/tests/pty-sidechannel.test.ts index 5e6fa50c7..36a3b65bb 100644 --- a/packages/sdk/tests/pty-sidechannel.test.ts +++ b/packages/sdk/tests/pty-sidechannel.test.ts @@ -97,6 +97,34 @@ process.stdin.on('end', () => { clearTimeout(watchdog); process.stdout.write('eo } finally { peer?.destroy(); } }); +it('unattended Codex receives closed stdin before startup instead of entering its additional-input lifecycle', async () => { + const dataDir = dir(); + const cli = join(dataDir, 'codex'); + writeFileSync(cli, `#!/usr/bin/env node +let ended = false; +process.stdin.resume(); +process.stdin.on('end', () => { + ended = true; + process.stdout.write('stdin-closed-before-startup'); +}); +setTimeout(() => { + if (ended) process.exit(0); + process.stderr.write('Reading additional input from stdin...'); + process.exit(1); +}, 50); +`, { mode: 0o755 }); + + const result = await runAgentCli(cli, 'test', undefined, 'pty-test-model', undefined, undefined, 'agent', { + dataDir, runId: 'r', stepId: 's', attempt: 1, onDrive() {}, + }); + + expect(result).toMatchObject({ + exit_code: 0, + stdout_tail: 'stdin-closed-before-startup', + transport: { phase: 'close', cause: 'exited', exit_code: 0, signal: null, retryable: false }, + }); +}); + it('rejects drive after EOF without marking human intervention', async () => { const dataDir = dir(); const cli = join(dataDir, 'claude'); diff --git a/packages/sdk/tests/spec-parity.test.ts b/packages/sdk/tests/spec-parity.test.ts index 48e25ac2d..13518a864 100644 --- a/packages/sdk/tests/spec-parity.test.ts +++ b/packages/sdk/tests/spec-parity.test.ts @@ -4,6 +4,7 @@ import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; import { compileAndHash, + compileSpec, compileYaml, compileYamlToCanonicalJson, toKernelSpec, @@ -60,6 +61,24 @@ describe('spec parity: one dialect at the SDK<->kernel boundary', () => { expect(() => kernelToAuthoring(kernel)).toThrow('retry.initial_backoff_ms'); }); + it('round-trips the explicit transport retry budget without changing legacy defaults', () => { + const flow = compileYaml(fixture('hello-ladder.flow.yaml')); + const agent = flow.steps.find(step => step.type === 'agent')!; + agent.transportRetries = 2; + const kernel = toKernelSpec(flow); + expect(kernel.steps.find(step => step.type === 'agent')!.retry.max_transport_retries).toBe(2); + expect(kernelToAuthoring(kernel)).toEqual(compileSpec(flow)); + }); + + it('preserves an explicit zero transport retry budget', () => { + const flow = compileYaml(fixture('hello-ladder.flow.yaml')); + const agent = flow.steps.find(step => step.type === 'agent')!; + agent.transportRetries = 0; + const kernel = toKernelSpec(flow); + expect(kernel.steps.find(step => step.type === 'agent')!.retry.max_transport_retries).toBe(0); + expect(kernelToAuthoring(kernel).steps.find(step => step.type === 'agent')!.transportRetries).toBe(0); + }); + // The trigger dialect. `toKernelSpec` used to spread `flow.triggers` through // untouched, so an event subscription reached the kernel in camelCase and // `relayflowd` -- `#[serde(deny_unknown_fields)]` over snake_case -- refused diff --git a/packages/sdk/tests/step-failure-diagnostic.test.ts b/packages/sdk/tests/step-failure-diagnostic.test.ts index 7172944e5..8830287d1 100644 --- a/packages/sdk/tests/step-failure-diagnostic.test.ts +++ b/packages/sdk/tests/step-failure-diagnostic.test.ts @@ -200,6 +200,27 @@ describe('step failure diagnostic', () => { expect(diagnostic.message).toContain('Transcript: /data/runs/run-failed/steps/fail-command/attempt-2.transcript.jsonl'); }); + it('renders structured transport cause, signal, retryability, and redacted stderr', async () => { + const { client } = stub([[{ + seq: 1, entry_type: 'step.completed', step_id: 'fail-command', attempt: 2, + payload: { + completionReason: 'crashed', disposition: 'step_done', output: null, + trajectory_tail: { transport: { + phase: 'close', cause: 'signal', exit_code: null, signal: 'SIGKILL', + retryable: true, stderr_tail: 'connection reset [redacted:API_TOKEN]', + } }, + verification: { gate: 'execution', verdict: 'fail', detail: 'worker reported crashed' }, + }, + }]], 'agent'); + const diagnostic = (await classify(client)).report.diagnostics.at(-1) as RunDiagnostic; + expect(diagnostic).toMatchObject({ + completionReason: 'crashed', attempt: 2, + transportPhase: 'close', transportCause: 'signal', signal: 'SIGKILL', + retryableTransport: true, stderrTail: 'connection reset [redacted:API_TOKEN]', + }); + expect(diagnostic.message).toContain('signal=SIGKILL transport=signal phase=close retryable=true'); + }); + it('does not print a stderr excerpt twice', async () => { const { client } = stub([[{ seq: 1, entry_type: 'step.completed', step_id: 'fail-command', diff --git a/packages/sdk/tests/verb-field-lint.test.ts b/packages/sdk/tests/verb-field-lint.test.ts index b4255713a..e4c09be2c 100644 --- a/packages/sdk/tests/verb-field-lint.test.ts +++ b/packages/sdk/tests/verb-field-lint.test.ts @@ -189,6 +189,7 @@ describe('closed per-verb step fields', () => { 'input', 'verification', 'maxIterations', + 'transportRetries', // Added by #221 (gate 5 slice 1). `memory` is common rather than // verb-specific: any step kind may declare a pack, so it generates no // foreign-field pairs. diff --git a/packages/sdk/tests/worker-cli.test.ts b/packages/sdk/tests/worker-cli.test.ts index 748f07725..e619d0e2e 100644 --- a/packages/sdk/tests/worker-cli.test.ts +++ b/packages/sdk/tests/worker-cli.test.ts @@ -79,6 +79,97 @@ process.stdout.write(JSON.stringify({ type: 'result', result: 'default-model-ok' }); }); +describe('direct transport lifecycle evidence', () => { + it('classifies only the exact Codex stdin lifecycle signature as retryable', async () => { + const directory = makeDirectory(); + const codex = makeWrapper(directory, 'codex', ` +process.stderr.write('Reading additional input from stdin...'); +process.exit(1); +`); + + const result = await runAgentCli(codex, 'do the task', undefined, 'unpriced-test-model'); + + expect(result).toMatchObject({ + exit_code: 1, + transport: { + phase: 'close', cause: 'codex_stdin_lifecycle', exit_code: 1, + signal: null, retryable: true, + }, + }); + expect(result.stderr_tail).toBe('Reading additional input from stdin...'); + }); + + it('records a signal close separately from an ordinary nonzero exit', async () => { + const directory = makeDirectory(); + const signaled = makeWrapper(directory, 'codex', `process.kill(process.pid, 'SIGTERM');`); + const signaledResult = await runAgentCli(signaled, 'task', undefined, 'unpriced-test-model'); + expect(signaledResult).toMatchObject({ + exit_code: null, + transport: { phase: 'close', cause: 'signal', signal: 'SIGTERM', retryable: true }, + }); + + const rejected = makeWrapper(directory, 'claude', `process.stderr.write('bad request ' + process.env.TRANSPORT_TEST_TOKEN); process.exit(7);`); + const rejectedResult = await withEnvironment({ TRANSPORT_TEST_TOKEN: 'transport-secret-123' }, () => + runAgentCli(rejected, 'task', undefined, 'unpriced-test-model')); + expect(rejectedResult).toMatchObject({ + exit_code: 7, + transport: { phase: 'close', cause: 'nonzero_exit', signal: null, retryable: false }, + }); + expect(rejectedResult.stderr_tail).toContain('[redacted:TRANSPORT_TEST_TOKEN]'); + expect(rejectedResult.stderr_tail).not.toContain('transport-secret-123'); + }); + + it('records a spawn error code without treating a missing executable as transient', async () => { + const directory = makeDirectory(); + const result = await runAgentCli(join(directory, 'codex'), 'task', undefined, 'unpriced-test-model'); + expect(result).toMatchObject({ + exit_code: null, + transport: { + phase: 'spawn', cause: 'spawn_error', error_code: 'ENOENT', + signal: null, retryable: false, + }, + }); + }); + + it('journals classified lifecycle evidence and reports crashed instead of generic worker_error', async () => { + const directory = makeDirectory(); + const codex = makeWrapper(directory, 'codex', ` +process.stderr.write('Reading additional input from stdin...'); +process.exit(1); +`); + const client = new EventEmitter() as EventEmitter & Record; + client.workerAttach = async () => ({}); + client.stepHeartbeat = async () => ({ lease_deadline_ms: Date.now() + 30_000 }); + let complete!: (args: unknown[]) => void; + const completed = new Promise(resolve => { complete = resolve; }); + client.stepComplete = async (...args: unknown[]) => { complete(args); return {}; }; + const worker = new AgentWorker(client as unknown as JournalClient, { + workerId: 'worker', pins: { workspace: [], streams: [] }, dataDir: join(directory, 'data'), + }); + const errors: unknown[] = []; + worker.on('error', error => errors.push(error)); + await worker.attach(); + client.emit('step.dispatch', { + run_id: 'run', step_id: 'agent', step_type: 'agent', attempt: 1, + idempotency_key: 'stable', lease_id: 'lease', lease_deadline_ms: Date.now() + 30_000, + pins: { workspace: [], streams: [] }, + spec: { cli: codex, instruction: 'task', model: 'unpriced-test-model' }, + }); + const args = await completed; + await worker.close(); + expect(errors).toEqual([]); + expect(args[4]).toBe('crashed'); + expect(args[5]).toMatchObject({ + trajectory_tail: { + transport: { + phase: 'close', cause: 'codex_stdin_lifecycle', exit_code: 1, + signal: null, retryable: true, + }, + }, + }); + }); +}); + describe('step discovery environment', () => { const NAMES = ['RELAYFLOW_DATA_DIR', 'RELAYFLOW_RUN_ID', 'RELAYFLOW_STEP_ID', 'RELAYFLOW_ATTEMPT', 'RELAYFLOW_WAKE_CONTEXT', 'RELAYFLOW_MODEL']; diff --git a/packages/surface/src/context.ts b/packages/surface/src/context.ts index 38b48ce74..1f95f56b2 100644 --- a/packages/surface/src/context.ts +++ b/packages/surface/src/context.ts @@ -40,6 +40,12 @@ export interface AgentOptions { * as a first-class workspace participant that DMs can steer. */ transport?: 'direct' | 'relay'; + /** Semantic executions allowed when verification rejects output. Default 1. */ + maxIterations?: number; + /** Additional attempts allowed only after classified transport loss. Default 1. */ + transportRetries?: number; + /** Appendix A workspace recovery for a transport retry. Default `reset`. */ + recoveryMode?: 'reset' | 'inspect' | 'manual'; } export interface LlmOptions { From f3bd47fe02d5113bc5c119208d44b66895c30c8e Mon Sep 17 00:00:00 2001 From: kjgbot Date: Sun, 20 Sep 2026 01:55:37 -0700 Subject: [PATCH 2/7] fix: publish transport retry schema Session-Id: 01a0bdd5-1542-7fe1-b85c-ada48bf177d9 --- packages/schema/flows.schema.json | 30 ++++++++++++++++++++++++++++ packages/schema/tests/parity.test.ts | 3 +++ scripts/schema-constraints.mjs | 2 ++ 3 files changed, 35 insertions(+) diff --git a/packages/schema/flows.schema.json b/packages/schema/flows.schema.json index 92fee9384..a5da6ae76 100644 --- a/packages/schema/flows.schema.json +++ b/packages/schema/flows.schema.json @@ -1014,6 +1014,12 @@ "description": "Semantic retry bound (kernel DESIGN.md §1.2 `max_iterations`). Default 1.", "type": "integer", "minimum": 1 + }, + "transportRetries": { + "title": "transportRetries", + "description": "Additional attempts after classified infrastructure loss. Default 1.", + "type": "integer", + "minimum": 0 } }, "required": [ @@ -1082,6 +1088,12 @@ "type": "integer", "minimum": 1 }, + "transportRetries": { + "title": "transportRetries", + "description": "Additional attempts after classified infrastructure loss. Default 1.", + "type": "integer", + "minimum": 0 + }, "command": { "title": "command", "description": "command in the Relayflows spec.", @@ -1191,6 +1203,12 @@ "type": "integer", "minimum": 1 }, + "transportRetries": { + "title": "transportRetries", + "description": "Additional attempts after classified infrastructure loss. Default 1.", + "type": "integer", + "minimum": 0 + }, "prompt": { "title": "prompt", "description": "prompt in the Relayflows spec.", @@ -1310,6 +1328,12 @@ "type": "integer", "minimum": 1 }, + "transportRetries": { + "title": "transportRetries", + "description": "Additional attempts after classified infrastructure loss. Default 1.", + "type": "integer", + "minimum": 0 + }, "instruction": { "title": "instruction", "description": "instruction in the Relayflows spec.", @@ -1861,6 +1885,12 @@ "title": "jitter_percent", "description": "jitter_percent in the Relayflows spec.", "type": "number" + }, + "max_transport_retries": { + "title": "max_transport_retries", + "description": "max_transport_retries in the Relayflows spec.", + "type": "integer", + "minimum": 0 } }, "required": [ diff --git a/packages/schema/tests/parity.test.ts b/packages/schema/tests/parity.test.ts index bb6a32766..0e56a4eb0 100644 --- a/packages/schema/tests/parity.test.ts +++ b/packages/schema/tests/parity.test.ts @@ -55,6 +55,9 @@ const cases: Array<[string, unknown, boolean]> = [ ['empty command', flow({ command: '' }), false], ['positive timeout', flow({ timeoutMs: 0 }), false], ['fractional retry', flow({ maxIterations: 1.5 }), false], + ['negative transport retry', flow({ transportRetries: -1 }), false], + ['fractional transport retry', flow({ transportRetries: 1.5 }), false], + ['zero transport retry', flow({ transportRetries: 0 }), true], ['wrong step field', flow({ prompt: 'hello' }), false], ['nonzero exit gate', flow({ verification: { type: 'exit_code', expect: 1 } }), false], ['legacy zero exit gate', flow({ verification: { type: 'exit_code', expect: 0 } }), true], diff --git a/scripts/schema-constraints.mjs b/scripts/schema-constraints.mjs index 7c4417566..35d1f214a 100644 --- a/scripts/schema-constraints.mjs +++ b/scripts/schema-constraints.mjs @@ -23,9 +23,11 @@ export function applyConstraints(defs, version) { } for (const type of ['BaseStepSpec', 'DeterministicStepSpec', 'LlmStepSpec', 'AgentStepSpec']) { property(type, 'maxIterations', positive); + property(type, 'transportRetries', integer); Object.assign(defs[type].properties.dependsOn.items, nonempty); property(type, 'input', { propertyNames: { type: 'string', pattern: '\\S' } }); } + property('KernelRetryPolicy', 'max_transport_retries', integer); property('DeterministicStepSpec', 'timeoutMs', positive); for (const field of ['maxTokensIn', 'maxTokensOut']) property('BudgetSpec', field, integer); property('BudgetSpec', 'maxDollars', decimal); From 2e784eaf8d72b81950178000a9c769607de2c5be Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 20 Sep 2026 02:52:43 -0700 Subject: [PATCH 3/7] fix(kernel): park manual recovery on worker-reported transport loss, repair torn parks Review of #501 (history lens, Cursor Bugbot) found `completion_actions` retried every budget-eligible `crashed`/`lease_expired` without reading the agent step's `recovery_mode`; only the kernel-noticed death (`abandonment_actions`) honoured `manual`. The same dead attempt therefore parked or redispatched depending on who noticed it first, against RFC-0001 Appendix A rule 4. - `completion_actions` parks a `manual` agent step on a worker-reported transport loss (`disposition: park` + `wait.human`), at any transport budget. It takes the journaled `start_pins` so the diff is anchored on the kernel's pin, never the worker's `end_pins` claim. The `wait.human` is built by one shared `manual_park_wait` for both producers. - A park is two appends; dying between them left a permanent, unanswerable park (raised in independent review). `state::park_placeholder_wait_id` names the placeholder and `recovery_actions_filtered` journals the missing `wait.human` on resume, once. Closes the same latent gap on the abandonment path. - `all_backing_off_steps_return_timers` regains a retryable failure precondition (`crashed`; `worker_error` is terminal since the budget split). - `step.attempt.started.max_transport_retries` is always journaled, like `max_iterations`; the old `skip_serializing_if` omitted the explicit zero that explains why a lost process was not retried, and contradicted DESIGN.md. DESIGN.md reconciled and now ties the SDK classifier, the completion-reason alphabet and the kernel disposition together. Tests: core unit (both producers, torn-park repair, journal field), in-process engine crash injection between the two appends, and a real-daemon protocol test (crashed / lease_expired / budget 0, silence probe, SIGKILL + resume, human answer redispatches on the pinned revision). Mutation red/green and full kernel + SDK runs captured in kernel/evidence/501/. Co-Authored-By: Claude Opus 5 (1M context) Session-Id: 144d3b43-0019-4de3-988a-7cd9ba4fc148 --- kernel/DESIGN.md | 15 +- kernel/evidence/501/README.md | 80 ++++ .../evidence/501/clippy-all-targets-warn.txt | 255 ++++++++++ kernel/evidence/501/clippy.txt | 213 ++++++++ kernel/evidence/501/green-kernel.txt | 438 +++++++++++++++++ .../501/green-sdk-bundle-pristine.txt | 22 + kernel/evidence/501/green-sdk.txt | 453 ++++++++++++++++++ kernel/evidence/501/green-targeted.txt | 54 +++ kernel/evidence/501/red-repair-only.txt | 64 +++ kernel/evidence/501/red.txt | 107 +++++ kernel/evidence/501/sdk-typecheck-build.txt | 19 + kernel/relayflowd-core/src/entry.rs | 58 ++- kernel/relayflowd-core/src/lib.rs | 2 +- kernel/relayflowd-core/src/machine.rs | 36 +- .../src/machine/parallel_tests.rs | 5 +- .../relayflowd-core/src/machine/recovery.rs | 98 +++- kernel/relayflowd-core/src/machine/tests.rs | 235 ++++++++- kernel/relayflowd-core/src/state.rs | 11 +- kernel/relayflowd-core/src/state/tests.rs | 4 +- kernel/relayflowd-core/tests/memoization.rs | 1 + kernel/relayflowd/src/engine/drive.rs | 9 +- kernel/relayflowd/src/engine/input.rs | 4 +- kernel/relayflowd/src/engine/memory.rs | 4 +- kernel/relayflowd/src/engine/remote.rs | 1 + kernel/relayflowd/src/exec_det.rs | 2 +- kernel/relayflowd/tests/budget_gate.rs | 2 +- kernel/relayflowd/tests/crash_resume.rs | 2 + .../tests/crash_resume/manual_recovery.rs | 192 ++++++++ kernel/relayflowd/tests/manual_recovery.rs | 305 ++++++++++++ 29 files changed, 2641 insertions(+), 50 deletions(-) create mode 100644 kernel/evidence/501/README.md create mode 100644 kernel/evidence/501/clippy-all-targets-warn.txt create mode 100644 kernel/evidence/501/clippy.txt create mode 100644 kernel/evidence/501/green-kernel.txt create mode 100644 kernel/evidence/501/green-sdk-bundle-pristine.txt create mode 100644 kernel/evidence/501/green-sdk.txt create mode 100644 kernel/evidence/501/green-targeted.txt create mode 100644 kernel/evidence/501/red-repair-only.txt create mode 100644 kernel/evidence/501/red.txt create mode 100644 kernel/evidence/501/sdk-typecheck-build.txt create mode 100644 kernel/relayflowd/tests/crash_resume/manual_recovery.rs create mode 100644 kernel/relayflowd/tests/manual_recovery.rs diff --git a/kernel/DESIGN.md b/kernel/DESIGN.md index e88141368..39df677c9 100644 --- a/kernel/DESIGN.md +++ b/kernel/DESIGN.md @@ -58,7 +58,7 @@ One per attempt. Payload: | `pins.workspace` | agent steps: `[{surface, revision_id}]` — relayfile revision id per declared mount surface, or `{worktree_base_commit}` | | `pins.streams` | `[{stream, read_offset}]` — consumer offsets at attempt start | | `max_iterations` | from spec, echoed for legibility | -| `max_transport_retries` | additional attempts allowed after classified infrastructure loss; default one and omitted at that default | +| `max_transport_retries` | from spec, echoed for legibility like `max_iterations`; additional attempts allowed after classified infrastructure loss (`crashed` / `lease_expired`). Always present — an explicit `0` is the value that explains why a lost process was not retried, so the journal never omits it (the spec's canonical form omits its default of one; the journal does not) | Deterministic/llm steps journal `pins.streams` only if they consume streams; `pins.workspace` is empty (no workspace). @@ -300,6 +300,19 @@ Only `crashed` and `lease_expired` consume it. `worker_error`, timeout, budget, cancellation, and an ordinary nonzero CLI exit are terminal regardless of the budget; semantic verification retry remains bounded only by `max_iterations`. +The kernel learns of a dead attempt two ways, and the recovery mode applies to +both: the kernel notices an abandoned lease (`abandonment_actions`), or the +worker reports its own loss through `step.complete` with `crashed` / +`lease_expired` (`completion_actions`). Under `manual` either path journals +`step.completed` with `disposition: park` and then the `wait.human` a human +answers, each its own append. A process death between the two leaves the step +folded to the placeholder wait id `park--` with nothing to +answer; resume recognises that placeholder and journals the missing +`wait.human` from the same journaled facts (last completion reason, start +pins), once. The SDK decides *which* transport losses are `crashed` +(`worker-cli.ts` classifies, `cli-transport-evidence.ts` maps to the reason); +the kernel decides what a `crashed` completion *does* (budget, recovery mode). + ### Memoized resume `resume(run_id)` re-executes nothing that finished. Algorithm: diff --git a/kernel/evidence/501/README.md b/kernel/evidence/501/README.md new file mode 100644 index 000000000..5ff118672 --- /dev/null +++ b/kernel/evidence/501/README.md @@ -0,0 +1,80 @@ +# PR #501 review follow-up — `manual` recovery for worker-reported transport loss + +Review-swarm history lens and Cursor Bugbot both found the same defect in +`f3bd47fe`: `completion_actions` retried every budget-eligible `crashed` / +`lease_expired` completion without reading the agent step's `recovery_mode`. +Only the kernel-noticed death (`abandonment_actions`) honoured `manual`, so +the same dead attempt parked or redispatched depending on who noticed it +first — contradicting RFC-0001 Appendix A rule 4. + +## What changed + +1. **`completion_actions` parks a `manual` agent step on a worker-reported + `crashed` / `lease_expired`** (`Disposition::Park` + `wait.human`), for any + transport budget including zero. The `wait.human` construction is one + function, `recovery::manual_park_wait`, shared with `abandonment_actions` + so the two producers cannot drift. `completion_actions` gained a + `start_pins` parameter so the `diff_ref` is anchored on the journaled + start pin, never the worker's `end_pins` claim. +2. **Torn parks are repaired on resume.** A park is two appends, each its own + transaction (raised by the independent reviewer, confirmed by external + probe). A step folded to the placeholder `park--` + (`state::park_placeholder_wait_id`) with no `wait.human` after it now has + the wait journaled by `recovery_actions_filtered` — once, from the same + journaled facts. This closes the same latent gap on the pre-existing + abandonment path. +3. **`all_backing_off_steps_return_timers`** regained a retryable failure + precondition (`Crashed`; `worker_error` is terminal since the budget + split) so it exercises failure backoff again. +4. **`step.attempt.started.max_transport_retries` is always journaled** + (`entry.rs` dropped `skip_serializing_if = is_zero_u32`). `kernel/DESIGN.md` + said "omitted at default (1)"; the code omitted zero — the one value that + explains why a lost process was not retried. Now it matches its sibling + `max_iterations`: always present. DESIGN.md updated, plus a paragraph tying + the SDK classifier, the completion-reason alphabet and the kernel + disposition together. + +## Evidence (literal commands + full output) + +| file | what | +|---|---| +| `red.txt` | both production changes mutated off (`manual_park = false && …`, repair guard `false && …`): 5 regression tests fail with the original symptom — `disposition: retry`, second dispatch | +| `red-repair-only.txt` | repair alone mutated off: torn-park tests fail (0 `wait.human` where 1 expected) | +| `green-targeted.txt` | files restored byte-for-byte (`sha256sum -c` OK), same commands pass | +| `green-kernel.txt` | `cargo test --workspace`, exit 0 | +| `clippy.txt` | `cargo clippy --workspace --all-targets -- -D warnings`: exits 101 on pre-existing findings only (`schema.rs:125`, `spec.rs:77`, `memoization.rs:102` as in the PR body, plus pre-existing test-target findings); `clippy-all-targets-warn.txt` lists every warning location — none on lines this change added | +| `sdk-typecheck-build.txt` | surface built + packed + installed `--no-save` into sdk (documented flow), then `npm run typecheck && npm run typecheck:tests && npm run build`, exit 0 | +| `green-sdk.txt` | `RELAYFLOWD_BIN= npx vitest run`: 154 files / 2410 tests pass; 2 environmental failures explained below | +| `green-sdk-bundle-pristine.txt` | `tests/bundle.test.ts` re-run from a pristine `npm ci`: 23/23 pass, exit 0 | + +The two failures in `green-sdk.txt` are environmental, not from this change: + +- `tests/bundle.test.ts` — `REFUSED [bundle_invalid] package-lock.json: + node_modules/@agent-relay/cli-surface does not match its pinned version`. + The documented `--no-save` surface override resolved `cli-surface` to + 12.4.0 over the lockfile's 12.2.4; the bundle builder refuses a non-pristine + tree by design. Green from a pristine `npm ci` (file above). +- `tests/authored-node-runtime.test.ts` — `beforeAll` pins `bun --version` + to exactly `1.4.0`; this machine has 1.4.2. The PR body excluded this + standalone suite for a different toolchain reason (node flag). + +Regression tests added: + +- `relayflowd-core/src/machine/tests.rs`: + `manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching` + (crashed + lease_expired), `reset_recovery_still_retries_a_worker_reported_transport_loss`, + `recovery_journals_the_wait_human_a_torn_manual_park_never_wrote` (both producers) +- `relayflowd-core/src/entry.rs`: `max_transport_retries_is_always_journaled`, + `a_pre_field_attempt_started_still_reads` +- `relayflowd/tests/manual_recovery.rs` (in-process engine, mock worker): + park survives reopen + resume with no redispatch and answers to a human on + the pinned revision (crashed / lease_expired / budget 0); crash injected + between the park's two appends is repaired on resume, idempotently +- `relayflowd/tests/crash_resume/manual_recovery.rs` (real `relayflowd serve` + binary over the protocol socket): same three cases, silence probe for + `step.dispatch`, daemon SIGKILL + restart + `resume`, journal unchanged, + `event.emit` answer redispatches attempt 2 on `rev-0` with the same + idempotency key + +`rustfmt --check` drift is unchanged from the PR head (20 files, none touched +by this change beyond formatting the lines it added). diff --git a/kernel/evidence/501/clippy-all-targets-warn.txt b/kernel/evidence/501/clippy-all-targets-warn.txt new file mode 100644 index 000000000..60ed06121 --- /dev/null +++ b/kernel/evidence/501/clippy-all-targets-warn.txt @@ -0,0 +1,255 @@ +$ cd kernel && cargo clippy --workspace --all-targets # warnings not denied, to reach every crate; filtered below to files this change touches + Checking relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) +warning: this `if` statement can be collapsed + --> relayflowd-core/src/schema.rs:125:13 + | +125 | / if let Some(target) = resolve(base_uri, reference, &resources, &anchors) { +126 | | if schema.pointer(&target).is_some() { +127 | | here.push(target); +128 | | } +129 | | } + | |_____________^ + | + = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.98.0/index.html#collapsible_if + = note: `#[warn(clippy::collapsible_if)]` on by default +help: collapse nested if block + | +125 ~ if let Some(target) = resolve(base_uri, reference, &resources, &anchors) +126 ~ && schema.pointer(&target).is_some() { +127 | here.push(target); +128 ~ } + | + +warning: this `if` statement can be collapsed + --> relayflowd-core/src/spec.rs:77:9 + | +77 | / if let Some(budget) = &self.budget { +78 | | if budget +79 | | .max_dollars +80 | | .as_deref() +... | +91 | | } + | |_________^ + | + = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.98.0/index.html#collapsible_if +help: collapse nested if block + | +77 ~ if let Some(budget) = &self.budget +78 ~ && (budget +79 | .max_dollars +... +84 | .as_ref() +85 ~ .is_some_and(|p| !crate::memory::valid_decimal(&p.dollars))) +86 | { +... +89 | )); +90 ~ } + | + +warning: this `if` statement can be collapsed + --> relayflowd-core/src/memoization.rs:102:9 + | +102 | / if let Action::Append(start) = &action { +103 | | if start.entry_type == EntryType::StepAttemptStarted && start.attempt == Some(1) { +104 | | let step = state +105 | | .spec +... | +138 | | } + | |_________^ + | + = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.98.0/index.html#collapsible_if +help: collapse nested if block + | +102 ~ if let Action::Append(start) = &action +103 ~ && start.entry_type == EntryType::StepAttemptStarted && start.attempt == Some(1) { +104 | let step = state +... +136 | } +137 ~ } + | + +warning: using `chunks_exact` with a constant chunk size + --> relayflowd-core/src/machine/parallel_tests.rs:118:42 + | +118 | for (pair, expected_step) in actions.chunks_exact(2).zip(["lane-b", "lane-a"]) { + | ^^^^^^^^^^^^^^^ help: consider using `as_chunks` instead: `as_chunks::<2>().0.iter()` + | + = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.98.0/index.html#chunks_exact_to_as_chunks + = note: `#[warn(clippy::chunks_exact_to_as_chunks)]` on by default + +warning: using `chunks_exact` with a constant chunk size + --> relayflowd-core/src/machine/parallel_tests.rs:226:44 + | +226 | for (pair, expected_step) in restarted.chunks_exact(2).zip(["lane-b", "lane-a"]) { + | ^^^^^^^^^^^^^^^ help: consider using `as_chunks` instead: `as_chunks::<2>().0.iter()` + | + = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.98.0/index.html#chunks_exact_to_as_chunks + +warning: `relayflowd-core` (lib) generated 3 warnings (run `cargo clippy --fix --lib -p relayflowd-core -- ` to apply 3 suggestions) + Checking relayflowd-journal v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-journal) +warning: items after a test module + --> relayflowd-core/src/state/budget.rs:88:1 + | + 88 | mod tests { + | ^^^^^^^^^ +... +139 | impl RunState { + | ^^^^^^^^^^^^^ + | + = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.98.0/index.html#items_after_test_module + = note: `#[warn(clippy::items_after_test_module)]` on by default + = help: move the items to before the test module was defined + +warning: unnecessary use of `clone` to create a slice from a reference + --> relayflowd-core/tests/memoization.rs:81:52 + | +81 | let folded = RunState::fold("new", state.spec, &[entry.clone()]).unwrap(); + | ^^^^^^^^^^^^^^^^ help: try: `std::slice::from_ref(entry)` + | + = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.98.0/index.html#cloned_ref_to_slice_refs + = note: `#[warn(clippy::cloned_ref_to_slice_refs)]` on by default + +warning: `relayflowd-core` (test "memoization") generated 1 warning (run `cargo clippy --fix --test "memoization" -p relayflowd-core -- ` to apply 1 suggestion) +warning: `relayflowd-core` (lib test) generated 6 warnings (3 duplicates) (run `cargo clippy --fix --lib -p relayflowd-core --tests -- ` to apply 3 suggestions) + Checking relayflowd v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd) +warning: match can be simplified with `.unwrap_or_default()` + --> relayflowd/src/engine/remote.rs:612:13 + | +612 | / match String::from_utf8(writer.buf) { +613 | | Ok(text) => text, +614 | | Err(_) => String::new(), +615 | | } + | |_____________^ help: replace it with: `String::from_utf8(writer.buf).unwrap_or_default()` + | + = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.98.0/index.html#manual_unwrap_or_default + = note: `#[warn(clippy::manual_unwrap_or_default)]` on by default + +warning: very complex type used. Consider factoring parts into `type` definitions + --> relayflowd/src/engine/wake.rs:62:23 + | +62 | inbox_resume: Option<&dyn Fn(&str) -> Result>, + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.98.0/index.html#type_complexity + = note: `#[warn(clippy::type_complexity)]` on by default + +warning: this `if` statement can be collapsed + --> relayflowd/src/engine.rs:319:9 + | +319 | / if started.is_err() { +320 | | if let Some(key) = admission_key { +... | +332 | | } + | |_________^ + | + = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.98.0/index.html#collapsible_if + = note: `#[warn(clippy::collapsible_if)]` on by default +help: collapse nested if block + | +319 ~ if started.is_err() +320 ~ && let Some(key) = admission_key { +321 | // Once registered, this run is the durable receipt even if a +... +330 | } +331 ~ } + | + +warning: file opened with `create`, but `truncate` behavior not defined + --> relayflowd/src/server/lifecycle.rs:47:10 + | +47 | .create(true) + | ^^^^^^^^^^^^- help: add: `.truncate(true)` + | + = help: if you intend to overwrite an existing file entirely, call `.truncate(true)` + = help: if you instead know that you may want to keep some parts of the old file, call `.truncate(false)` + = help: alternatively, use `.append(true)` to append to the file instead of overwriting it + = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.98.0/index.html#suspicious_open_options + = note: `#[warn(clippy::suspicious_open_options)]` on by default + +warning: this function has too many arguments (8/7) + --> relayflowd/src/server/session.rs:113:5 + | +113 | / pub fn attach_worker( +114 | | &self, +115 | | connection_id: u64, +116 | | worker_id: String, +... | +121 | | writer: Writer, +122 | | ) { + | |_____^ + | + = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.98.0/index.html#too_many_arguments + = note: `#[warn(clippy::too_many_arguments)]` on by default + +warning: `relayflowd` (lib test) generated 5 warnings (run `cargo clippy --fix --lib -p relayflowd --tests -- ` to apply 2 suggestions) +warning: `relayflowd` (lib) generated 5 warnings (5 duplicates) +warning: this `if` statement can be collapsed + --> relayflowd/tests/daemon_lifecycle.rs:62:9 + | +62 | / if path.exists() { +63 | | if let Ok(bytes) = std::fs::read(&path) { +64 | | if let Ok(connection) = serde_json::from_slice::(&bytes) { +65 | | if connection["pid"].as_i64() != Some(excluding_pid as i64) { +... | +73 | | } + | |_________^ + | + = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.98.0/index.html#collapsible_if + = note: `#[warn(clippy::collapsible_if)]` on by default +help: collapse nested if block + | +62 ~ if path.exists() +63 ~ && let Ok(bytes) = std::fs::read(&path) { +64 | if let Ok(connection) = serde_json::from_slice::(&bytes) { +... +71 | } +72 ~ } + | + +warning: this `if` statement can be collapsed + --> relayflowd/tests/daemon_lifecycle.rs:63:13 + | +63 | / if let Ok(bytes) = std::fs::read(&path) { +64 | | if let Ok(connection) = serde_json::from_slice::(&bytes) { +65 | | if connection["pid"].as_i64() != Some(excluding_pid as i64) { +66 | | let socket = connection["socket_path"].as_str().unwrap(); +... | +72 | | } + | |_____________^ + | + = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.98.0/index.html#collapsible_if +help: collapse nested if block + | +63 ~ if let Ok(bytes) = std::fs::read(&path) +64 ~ && let Ok(connection) = serde_json::from_slice::(&bytes) { +65 | if connection["pid"].as_i64() != Some(excluding_pid as i64) { +... +70 | } +71 ~ } + | + +warning: this `if` statement can be collapsed + --> relayflowd/tests/daemon_lifecycle.rs:64:17 + | +64 | / if let Ok(connection) = serde_json::from_slice::(&bytes) { +65 | | if connection["pid"].as_i64() != Some(excluding_pid as i64) { +66 | | let socket = connection["socket_path"].as_str().unwrap(); +67 | | UnixStream::connect(socket) +... | +71 | | } + | |_________________^ + | + = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.98.0/index.html#collapsible_if +help: collapse nested if block + | +64 ~ if let Ok(connection) = serde_json::from_slice::(&bytes) +65 ~ && connection["pid"].as_i64() != Some(excluding_pid as i64) { +66 | let socket = connection["socket_path"].as_str().unwrap(); +... +69 | return connection; +70 ~ } + | + +warning: `relayflowd` (test "daemon_lifecycle") generated 3 warnings (run `cargo clippy --fix --test "daemon_lifecycle" -p relayflowd -- ` to apply 3 suggestions) + Finished `dev` profile [unoptimized + debuginfo] target(s) in 2.96s +exit_code=0 diff --git a/kernel/evidence/501/clippy.txt b/kernel/evidence/501/clippy.txt new file mode 100644 index 000000000..89b4f604b --- /dev/null +++ b/kernel/evidence/501/clippy.txt @@ -0,0 +1,213 @@ +$ cd kernel && cargo clippy --workspace --all-targets -- -D warnings + Checking stable_deref_trait v1.2.1 + Checking cfg-if v1.0.4 + Checking smallvec v1.15.2 + Checking writeable v0.6.4 + Checking memchr v2.8.3 + Checking litemap v0.8.3 + Checking utf8_iter v1.0.4 + Checking typenum v1.20.1 + Checking once_cell v1.21.4 + Checking libc v0.2.189 + Checking serde_core v1.0.229 + Checking zerocopy v0.8.56 + Checking icu_properties_data v2.3.0 + Checking icu_normalizer_data v2.3.0 + Checking scopeguard v1.2.0 + Checking regex-syntax v0.8.11 + Checking zmij v1.0.23 + Checking borrow-or-share v0.2.4 + Checking itoa v1.0.18 + Checking num-traits v0.2.19 + Checking bit-vec v0.8.0 + Checking ref-cast v1.0.27 + Checking percent-encoding v2.3.2 + Checking uuid v1.26.0 + Checking zerofrom v0.1.8 + Checking outref v0.5.2 + Checking vsimd v0.8.0 + Checking lock_api v0.4.14 + Checking lazy_static v1.5.0 + Checking bytecount v0.6.9 + Checking num-cmp v0.1.0 + Checking cpufeatures v0.2.17 + Checking yoke v0.8.3 + Checking base64 v0.22.1 + Checking thiserror v2.0.20 + Checking ryu-js v1.0.3 + Checking foldhash v0.1.5 + Checking bitflags v2.13.1 + Checking utf8parse v0.2.2 + Checking libsqlite3-sys v0.35.0 + Checking is_terminal_polyfill v1.70.2 + Checking colorchoice v1.0.5 + Checking anstyle v1.0.14 + Checking bit-set v0.8.0 + Checking fallible-streaming-iterator v0.1.9 + Checking fallible-iterator v0.3.0 + Checking anstyle-query v1.1.5 + Checking anstyle-parse v1.0.0 + Checking strsim v0.11.1 + Checking clap_lex v1.1.0 + Checking zerovec v0.11.8 + Checking zerotrie v0.2.5 + Checking hashbrown v0.15.5 + Checking linux-raw-sys v0.12.1 + Checking anyhow v1.0.104 + Checking fastrand v2.5.0 + Checking anstream v1.0.0 + Checking uuid-simd v0.8.0 + Checking rustix v1.1.4 + Checking aho-corasick v1.1.5 + Checking clap_builder v4.6.6 + Checking generic-array v0.14.7 + Checking hashlink v0.10.0 + Checking num-integer v0.1.47 + Checking num-complex v0.4.6 + Checking tinystr v0.8.4 + Checking potential_utf v0.1.6 + Checking block-buffer v0.10.4 + Checking crypto-common v0.1.7 + Checking rusqlite v0.37.0 + Checking icu_collections v2.3.0 + Checking icu_locale_core v2.3.0 + Checking digest v0.10.7 + Checking num-bigint v0.4.8 + Checking num-iter v0.1.46 + Checking getrandom v0.3.4 + Checking parking_lot_core v0.9.12 + Checking getrandom v0.4.3 + Checking wait-timeout v0.2.1 + Checking rand_core v0.9.5 + Checking sha2 v0.10.9 + Checking parking_lot v0.12.5 + Checking serde v1.0.229 + Checking serde_json v1.0.151 + Checking icu_provider v2.3.1 + Checking regex-automata v0.4.18 + Checking icu_normalizer v2.3.0 + Checking icu_properties v2.3.0 + Checking num-rational v0.4.2 + Checking fluent-uri v0.3.2 + Checking email_address v0.2.9 + Checking num v0.4.3 + Checking fraction v0.15.4 + Checking tempfile v3.27.0 + Checking clap v4.6.6 + Checking idna_adapter v1.2.2 + Checking idna v1.1.0 + Checking ppv-lite86 v0.2.21 + Checking ahash v0.8.12 + Checking referencing v0.33.0 + Checking rand_chacha v0.9.0 + Checking rand v0.9.5 + Checking regex v1.13.1 + Checking fancy-regex v0.16.2 + Checking ulid v1.2.1 + Checking jsonschema v0.33.0 + Checking relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) +error: this `if` statement can be collapsed + --> relayflowd-core/src/schema.rs:125:13 + | +125 | / if let Some(target) = resolve(base_uri, reference, &resources, &anchors) { +126 | | if schema.pointer(&target).is_some() { +127 | | here.push(target); +128 | | } +129 | | } + | |_____________^ + | + = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.98.0/index.html#collapsible_if + = note: `-D clippy::collapsible-if` implied by `-D warnings` + = help: to override `-D warnings` add `#[allow(clippy::collapsible_if)]` +help: collapse nested if block + | +125 ~ if let Some(target) = resolve(base_uri, reference, &resources, &anchors) +126 ~ && schema.pointer(&target).is_some() { +127 | here.push(target); +128 ~ } + | + +error: this `if` statement can be collapsed + --> relayflowd-core/src/spec.rs:77:9 + | +77 | / if let Some(budget) = &self.budget { +78 | | if budget +79 | | .max_dollars +80 | | .as_deref() +... | +91 | | } + | |_________^ + | + = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.98.0/index.html#collapsible_if +help: collapse nested if block + | +77 ~ if let Some(budget) = &self.budget +78 ~ && (budget +79 | .max_dollars +... +84 | .as_ref() +85 ~ .is_some_and(|p| !crate::memory::valid_decimal(&p.dollars))) +86 | { +... +89 | )); +90 ~ } + | + +error: this `if` statement can be collapsed + --> relayflowd-core/src/memoization.rs:102:9 + | +102 | / if let Action::Append(start) = &action { +103 | | if start.entry_type == EntryType::StepAttemptStarted && start.attempt == Some(1) { +104 | | let step = state +105 | | .spec +... | +138 | | } + | |_________^ + | + = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.98.0/index.html#collapsible_if +help: collapse nested if block + | +102 ~ if let Action::Append(start) = &action +103 ~ && start.entry_type == EntryType::StepAttemptStarted && start.attempt == Some(1) { +104 | let step = state +... +136 | } +137 ~ } + | + +error: could not compile `relayflowd-core` (lib) due to 3 previous errors +warning: build failed, waiting for other jobs to finish... +error: using `chunks_exact` with a constant chunk size + --> relayflowd-core/src/machine/parallel_tests.rs:118:42 + | +118 | for (pair, expected_step) in actions.chunks_exact(2).zip(["lane-b", "lane-a"]) { + | ^^^^^^^^^^^^^^^ help: consider using `as_chunks` instead: `as_chunks::<2>().0.iter()` + | + = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.98.0/index.html#chunks_exact_to_as_chunks + = note: `-D clippy::chunks-exact-to-as-chunks` implied by `-D warnings` + = help: to override `-D warnings` add `#[allow(clippy::chunks_exact_to_as_chunks)]` + +error: using `chunks_exact` with a constant chunk size + --> relayflowd-core/src/machine/parallel_tests.rs:226:44 + | +226 | for (pair, expected_step) in restarted.chunks_exact(2).zip(["lane-b", "lane-a"]) { + | ^^^^^^^^^^^^^^^ help: consider using `as_chunks` instead: `as_chunks::<2>().0.iter()` + | + = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.98.0/index.html#chunks_exact_to_as_chunks + +error: items after a test module + --> relayflowd-core/src/state/budget.rs:88:1 + | + 88 | mod tests { + | ^^^^^^^^^ +... +139 | impl RunState { + | ^^^^^^^^^^^^^ + | + = help: for further information visit https://rust-lang.github.io/rust-clippy/rust-1.98.0/index.html#items_after_test_module + = note: `-D clippy::items-after-test-module` implied by `-D warnings` + = help: to override `-D warnings` add `#[allow(clippy::items_after_test_module)]` + = help: move the items to before the test module was defined + +error: could not compile `relayflowd-core` (lib test) due to 6 previous errors +exit_code=101 diff --git a/kernel/evidence/501/green-kernel.txt b/kernel/evidence/501/green-kernel.txt new file mode 100644 index 000000000..cca0036b7 --- /dev/null +++ b/kernel/evidence/501/green-kernel.txt @@ -0,0 +1,438 @@ +$ cd kernel && cargo test --workspace + Compiling relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) + Compiling relayflowd-journal v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-journal) + Compiling relayflowd v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd) + Finished `test` profile [unoptimized + debuginfo] target(s) in 2.02s + Running unittests src/lib.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/relayflowd-f043db0bb3534a16) + +running 53 tests +test engine::remote::worker_failure_detail_tests::a_null_or_blank_output_yields_no_detail ... ok +test engine::remote::worker_failure_detail_tests::a_string_output_is_carried_verbatim_and_trimmed ... ok +test engine::remote::worker_failure_detail_tests::a_non_string_output_is_rendered_rather_than_dropped ... ok +test engine::remote::worker_failure_detail_tests::an_output_at_the_boundary_is_not_truncated ... ok +test engine::boot_identity_tests::every_engine_in_this_process_shares_one_boot_id ... ok +test engine::remote::worker_failure_detail_tests::truncation_does_not_split_a_multi_byte_char ... ok +test server::liveness::tests::sweep_id_buckets_by_the_interval ... ok +test server::channels::tests::unknown_verb_never_falls_through_to_receive ... ok +test server::client::tests::resume_waits_while_the_heartbeat_renewed_lease_is_live ... ok +test engine::remote::worker_failure_detail_tests::render_is_bounded_before_allocation_for_hostile_string ... ok +test server::tests::agent::contract::an_oversized_trajectory_tail_is_refused_at_step_complete ... ok +test server::tests::agent::eligibility::required_streams_must_be_held_before_worker_registration ... ok +test engine::remote::worker_failure_detail_tests::render_is_bounded_before_allocation_for_hostile_json ... ok +test exec_det::tests::timeout_has_an_explicit_completion_reason ... ok +test exec_det::tests::captures_deterministic_output ... ok +test server::tests::agent::contract::an_agent_worker_attaching_without_pins_is_refused_at_attach ... ok +test exec_det::tests::failed_command_evidence_survives_completion ... ok +test server::tests::hello_enforces_protocol_version ... ok +test exec_det::tests::lease_override_bounds_execution_and_preserves_command_timeout ... ok +test server::tests::run_resume_asks_the_registry_instead_of_treating_an_orphan_file_as_a_run ... ok +test engine::wake::claim_guard_tests::a_disarmed_guard_leaves_the_claim_alone ... ok +test server::liveness::tests::sweep_pass_healthy_subscription_is_a_noop ... ok +test server::tests::run_start_fails_closed_on_an_unknown_verification_key ... ok +test server::liveness::tests::sweep_pass_latches_after_journaling_and_next_bucket_is_empty ... ok +test engine::wake::claim_guard_tests::a_panic_between_claim_and_register_still_releases ... ok +test engine::wake::claim_guard_tests::a_guard_only_releases_its_own_run ... ok +test socket_path::tests::deep_data_dir_produces_short_socket_path ... ok +test socket_path::tests::different_data_dirs_yield_different_sockets ... ok +test socket_path::tests::relative_and_absolute_data_dirs_agree ... ok +test socket_path::tests::same_data_dir_yields_same_socket ... ok +test engine::wake::claim_guard_tests::an_armed_guard_releases_the_claim_when_dropped ... ok +test server::tests::run_start_refuses_invalid_admission_keys ... ok +test server::tests::run_resume_refuses_a_journal_that_never_recorded_its_run ... ok +test server::tests::agent::contract::an_agent_worker_missing_a_declared_surface_parks_the_run_instead_of_erroring ... ok +test server::tests::agent::contract::agent_without_a_compatible_worker_parks_without_starting ... ok +test server::tests::agent::eligibility::required_streams_keep_ordinary_steps_off_conversation_workers ... ok +test engine::boot_identity_tests::prior_boot_registered_undriven_admission_is_recovered_by_start_retry ... ok +test server::tests::run_start_admission_key_recovers_the_same_run_and_refuses_spec_drift ... ok +test server::tests::run_resume_adopts_a_real_journal_whose_registry_row_is_missing ... ok +test engine::boot_identity_tests::failure_after_workspace_binding_releases_admission_without_exposing_effects ... ok +test server::tests::agent::pins::reset_worker_reporting_a_revision_other_than_its_pin_fails_closed_as_worker_error ... ok +test server::tests::agent::contract::an_llm_completion_claiming_an_effect_fails_closed_with_the_reason_journaled ... ok +test server::tests::run_resume_refuses_a_valid_journal_that_belongs_to_another_run ... ok +test server::tests::agent::contract::a_replacement_worker_that_never_reported_the_pinned_surface_is_not_dispatched_to ... ok +test server::tests::agent::contract::a_transcript_digest_at_its_budget_rides_trajectory_tail_verbatim ... ok +test server::tests::a_failed_disconnect_journal_append_is_retained_and_retried_not_dropped ... ok +test server::tests::agent::contract::human_intervention_is_durable_and_resume_requires_explicit_override ... ok +test server::tests::stopped_heartbeats_past_the_deadline_journal_lease_expired_and_release_the_step ... ok +test server::tests::agent::pins::consecutive_agent_steps_on_different_surfaces_each_start_from_their_own_pins ... ok +test server::tests::step_wait_parks_the_attempt_and_a_human_answer_redispatches_it ... ok +test exec_det::tests::timeout_kills_the_whole_process_group ... ok +test server::tests::agent::pins::a_replacement_worker_at_a_different_revision_is_not_dispatched_the_stale_pins ... ok +test server::tests::an_entry_appended_during_watch_registration_is_delivered_exactly_once ... ok + +test result: ok. 53 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.57s + + Running unittests src/main.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/relayflowd-6e3681176306c99e) + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/budget_gate.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/budget_gate-9607ae7c2db11b74) + +running 10 tests +test metering_flag_is_additive_on_the_wire ... ok +test prior_spend_metering_flag_is_additive_and_fails_closed_for_older_kernels ... ok +test daily_windows_reset_and_exact_limits_do_not_refuse ... ok +test carried_metered_dollars_still_stop_the_continuing_run ... ok +test unmetered_tokens_still_cross_a_token_ceiling ... ok +test unmetered_usage_may_not_claim_priced_dollars ... ok +test carried_prior_spend_keeps_unknown_dollar_cost_unmetered ... ok +test crossing_completion_is_durable_and_next_step_is_refused ... ok +test unmetered_spend_is_journaled_as_unknown_and_never_crosses_a_dollar_ceiling ... ok +test deterministic_spend_and_wallclock_limit_gate_parallel_batch_starts ... ok + +test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running tests/crash_resume.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/crash_resume-d2d102ec3e89b705) + +running 41 tests +test agent::resume_without_a_worker_parks_immediately_instead_of_timing_out ... ok +test concurrency::run_start_dispatches_every_independent_lane_before_any_completion ... ok +test channels::channels_reject_foreign_workers_stale_attempts_and_invalid_acknowledgements ... ok +test agent::rung_c_sigkill_after_final_effect_replays_results_without_redispatch ... ok +test agent::rung_c_sigkill_between_agent_completion_and_final_effect_memoizes_the_agent ... ok +test llm::serve_plumbs_watch_events_and_replayable_stream_verbs ... ok +test concurrency::live_resume_leaves_an_active_lease_running ... ok +test concurrency::concurrent_resumes_lease_exactly_one_attempt ... ok +test concurrency::server_restart_recovers_every_parallel_lease_without_duplicate_success ... ok +test concurrency::cancel_closes_the_lease_and_rejects_a_late_completion ... ok +test agent::rung_c_crash_between_effect_election_and_the_provider_call_performs_it_exactly_once ... ok +test concurrency::cancel_and_completion_race_has_one_terminal_fact ... ok +test llm::failing_llm_verification_schedules_a_durable_retry_and_succeeds ... ok +test agent::rung_c_reset_sigkill_mid_edit_restores_pins_dedupes_effect_and_explains_attempts ... ok +test pin_projection::rejected_completion_cannot_forge_pins_or_trigger_a_blind_retry ... ok +test memory::memory_sigkill_after_injection_replays_pack_and_charges_it_once ... ok +test llm::llm_verification_exhaustion_is_a_declared_failure_kind ... ok +test sigkill_mid_step_replaces_and_explains_the_dead_attempt ... ok +test protocol_admission::every_mutating_run_verb_refuses_terminal_before_changing_state ... ok +test sigkill_after_cancel_request_resumes_to_one_canceled_fact ... ok +test llm::sigkill_after_the_final_rung_b_effect_resumes_without_redispatching_llm ... ok +test llm::worker_killed_while_holding_a_lease_is_explained_and_released_on_cli_resume ... ok +test llm::completed_llm_output_is_memoized_when_serve_dies_during_the_next_step ... ok +test worker_capacity::two_workers_receive_a_deterministic_fair_capacity_bounded_batch ... ok +test sigkill_under_serve_resumes_the_socket_started_run ... ok +test surface_identity::aliases_are_rejected_and_external_ancestors_serialize_over_real_sockets ... ok +test placement::declared_placement_keeps_one_source_tree_across_resume ... ok +test agent::rung_c_sigkill_boundaries_resume_only_unfinished_steps_via_real_cli ... ok +test workspace_identity::workspace_aliases_are_refused_and_canonical_subtrees_serialize_over_real_sockets ... ok +test placement::sigkill_before_first_step_preserves_the_submitted_workspace ... ok +test placement::sigkill_mid_step_keeps_the_route_and_source_tree ... ok +test llm::sigkill_under_serve_mid_llm_releases_the_lease_and_finishes_via_cli_resume ... ok +test parallel_lifecycle::terminal_failure_drains_or_explains_every_live_sibling ... ok +test worker_capacity::default_capacity_one_reopens_only_after_durable_completion_or_crash ... ok +test sigkill_sweep_covers_every_hello_step_boundary ... ok +test llm::sigkill_sweep_covers_before_and_between_the_rung_b_steps ... ok +test parallel_lifecycle::overlapping_agent_conflict_survives_server_crash_and_resume ... ok +test parallel_lifecycle::overlapping_agent_lanes_serialize_while_disjoint_lanes_merge_in_either_order ... ok +test channels::channels_sigkill_resume_redelivers_unacked_messages_with_exactly_once_effects ... ok +test manual_recovery::manual_recovery_parks_a_worker_reported_loss_across_daemon_restart_until_a_human_answers ... ok +test parallel_lifecycle::renewed_parallel_leases_survive_the_original_grant_and_remain_distinct ... ok + +test result: ok. 41 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 36.42s + + Running tests/daemon_lifecycle.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/daemon_lifecycle-b705da9761b2a254) + +running 6 tests +test deep_data_dir_still_binds ... ok +test sigkill_leaves_a_stale_file_with_a_dead_pid ... ok +test clean_shutdown_removes_advertisement_and_socket ... ok +test connection_file_is_published_only_after_the_socket_is_live ... ok +test a_second_serve_on_a_served_data_dir_refuses_and_the_first_keeps_serving ... ok +test a_sigkilled_daemons_successor_starts_cleanly ... ok + +test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + + Running tests/event_wake.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/event_wake-b98778c1ad4de872) + +running 3 tests +test matching_event_wakes_once_with_fresh_context ... ok +test two_racing_deliveries_of_one_event_produce_exactly_one_run ... ok +test a_resumed_run_dispatches_the_original_wake_context ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + + Running tests/hn_monitor_integration.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/hn_monitor_integration-059c38eb4828898f) + +running 1 test +test hn_story_event_wakes_monitor_once_with_story_context ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running tests/input_binding.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/input_binding-4f132e6302508de8) + +running 2 tests +test binding_schema_is_additive_and_fails_closed ... ok +test sigkill_before_consumer_resolves_original_journal_output_without_reexecuting_source ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.06s + + Running tests/invalid_schema_preflight.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/invalid_schema_preflight-9c08869ea58d283f) + +running 3 tests +test invalid_json_schema_is_refused_before_journal_or_command ... ok +test unbounded_json_schema_is_refused_before_journal_or_command ... ok +test legitimately_recursive_json_schema_still_starts ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 3.34s + + Running tests/manual_recovery.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/manual_recovery-6793123a1244aad2) + +running 2 tests +test a_park_torn_between_its_two_appends_is_repaired_on_resume ... ok +test manual_park_of_a_worker_reported_loss_survives_reopen_and_answers_to_a_human ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s + + Running tests/memoization.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/memoization-e5537edfeea8b814) + +running 3 tests +test refuses_missing_wrong_flow_and_unreadable_journal_before_creating_run ... ok +test reused_prefix_survives_restart_without_source_and_never_mutates_prior ... ok +test actual_changed_input_invalidates_consumer_even_with_identical_consumer_spec ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s + + Running tests/memory.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/memory-fe5dbc6738ee15ff) + +running 5 tests +test rejected_journal_fact_releases_reservation_and_never_dispatches ... ok +test llm_dispatch_receives_same_pack_after_resume_without_provider ... ok +test over_budget_and_provider_errors_fail_without_dispatch_or_charge ... ok +test replay_and_resume_need_no_provider_and_script_receives_recorded_pack ... ok +test semantic_retry_reuses_memory_without_a_second_charge ... ok + +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running tests/memory_epoch.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/memory_epoch-627e50d1c86c39e4) + +running 1 test +test epoch_carries_pack_and_exact_charge_and_refuses_duplicate_injection ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + + Running tests/parallel_driver.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/parallel_driver-7a554ae88aa529f9) + +running 4 tests +test stop_after_one_holds_for_an_independent_deterministic_batch ... ok +test backpressured_or_mismatched_lane_does_not_drop_a_later_dispatch ... ok +test pause_before_second_independent_step_holds_the_driver_boundary ... ok +test crash_boundaries_resume_the_real_driver_with_one_effect_per_lane ... ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s + + Running tests/placement_pins.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/placement_pins-839e136752b4e5af) + +running 3 tests +test unsupported_local_pty_is_refused_before_an_earlier_step_can_run ... ok +test default_worker_pins_the_declared_worktree_base_commit_and_refuses_missing_source ... ok +test a_resumed_attempt_keeps_the_original_pin_after_the_worktree_head_moves ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.06s + + Running tests/placement_routing.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/placement_routing-9cb694324a7d7bf1) + +running 3 tests +test a_failed_routing_append_never_starts_or_dispatches_work ... ok +test crash_between_routing_and_start_does_not_redecide ... ok +test worker_retry_consumes_the_original_routing_fact ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + + Running tests/routing_diagnostics.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/routing_diagnostics-7807262e1a1167ec) + +running 2 tests +test duplicate_routes_have_a_distinct_diagnostic_and_leave_the_original_fact_intact ... ok +test malformed_routes_name_the_same_field_at_append_replay_and_epoch_replay ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/spec_review_routing.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/spec_review_routing-032df51688a72750) + +running 4 tests +test attempt_scoped_route_is_rejected_at_append_and_replay ... ok +test malformed_epoch_routes_are_rejected_before_commit ... ok +test epoch_cannot_drop_or_replace_a_durable_route ... ok +test workspace_pin_peels_tags_and_refuses_non_commit_objects ... ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s + + Running tests/subscription_liveness.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/subscription_liveness-c3bbfd58dcafb336) + +running 3 tests +test submit_event_upserts_subscription_row_and_sweep_flags_it_stale_after_budget ... ok +test stale_transition_is_journaled_as_subscription_stale_entry_in_the_last_known_run ... ok +test a_fresh_arrival_re_arms_the_latch_and_the_next_silence_can_stale_again ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + + Running tests/trigger_watcher.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/trigger_watcher-0504d7ed791030e6) + +running 3 tests +test retains_bad_and_unregistered_events_while_consuming_filter_nonmatches ... ok +test failed_archive_retries_the_same_durable_run ... ok +test journals_payload_and_filename_key_then_archives_and_dedupes_replay ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + + Running unittests src/lib.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/relayflowd_core-e734d1d7b8cb7b12) + +running 72 tests +test clock::tests::simulated_clock_is_explicitly_advanced ... ok +test entry::completion_reason_tests::every_journal_label_matches_serialized ... ok +test entry::completion_reason_tests::all_covers_every_serialized_label ... ok +test entry::attempt_started_tests::a_pre_field_attempt_started_still_reads ... ok +test channel::tests::send_retry_is_stable_and_conflicting_content_is_rejected ... ok +test entry::attempt_started_tests::max_transport_retries_is_always_journaled ... ok +test channel::tests::malformed_payloads_and_invalid_new_channel_appends_leave_state_unchanged ... ok +test journal::tests::memory_journal_assigns_sequences_and_rolls_epochs ... ok +test channel::tests::forged_deliveries_and_acknowledgements_fail_closed ... ok +test channel::tests::delivery_replay_and_independent_acknowledged_offsets ... ok +test machine::parallel_tests::every_declared_mutable_surface_participates_in_conflict_selection ... ok +test machine::tests::every_reason_label_matches_its_serialized_form ... ok +test machine::parallel_tests::failed_run_drains_open_siblings_before_terminal_entry ... ok +test machine::parallel_tests::machine_starts_every_runnable_step_in_authored_order ... ok +test machine::tests::deterministic_lease_rejects_invalid_and_foreign_fields ... ok +test machine::tests::crashed_attempt_does_not_consume_an_iteration ... ok +test machine::parallel_tests::overlapping_agent_surfaces_are_serialized_in_authored_order ... ok +test machine::tests::cancel_request_closes_the_active_lease_before_the_terminal_fact ... ok +test machine::tests::all_backing_off_steps_return_timers ... ok +test machine::tests::machine_starts_runnable_step_with_stable_effect_key ... ok +test machine::tests::every_failed_run_terminates_with_declared_completion_reasons ... ok +test machine::tests::ordinary_worker_error_is_not_retried_by_either_budget ... ok +test machine::tests::durable_cancel_request_outranks_crash_recovery ... ok +test machine::tests::manual_recovery_parks_needs_human_and_never_redispatches ... ok +test machine::tests::inspect_recovery_injects_the_dirty_pin_completion_reason_and_tail ... ok +test machine::tests::successful_memo_is_never_scheduled_again ... ok +test machine::tests::repeated_cancel_request_is_idempotent ... ok +test machine::tests::reset_recovery_still_retries_a_worker_reported_transport_loss ... ok +test machine::parallel_tests::workspace_ancestor_and_descendant_paths_conflict_but_siblings_do_not ... ok +test machine::tests::deterministic_lease_override_and_default_are_journaled ... ok +test machine::tests::worker_reported_failure_without_detail_still_records_a_verification ... ok +test machine::tests::reset_recovery_dispatches_the_original_pinned_revision ... ok +test machine::parallel_tests::parallel_lanes_do_not_cross_the_dependency_barrier_early ... ok +test retry::tests::jitter_is_repeatable_and_bounded ... ok +test machine::tests::classified_transport_retry_is_bounded_and_preserves_pin_and_idempotency ... ok +test memory::tests::caps_compare_exact_decimals_and_each_token_dimension ... ok +test machine::tests::verification_failure_schedules_a_durable_retry ... ok +test machine::tests::failed_deterministic_completion_preserves_exit_code_and_stderr ... ok +test machine::parallel_tests::external_ancestor_and_descendant_paths_conflict_but_siblings_do_not ... ok +test machine::parallel_tests::crash_resume_preserves_each_parallel_lease_exactly_once ... ok +test spec::tests::a_misspelled_step_level_key_is_a_parse_error ... ok +test schema::tests::in_document_uri_references_resolve_to_the_node_they_name ... ok +test schema::tests::refusal_names_the_cycle_it_found ... ok +test spec::tests::a_misspelled_verification_gate_key_is_a_parse_error_not_a_dropped_gate ... ok +test spec::tests::spec_version_is_semver_and_gated ... ok +test spec::tests::cycles_are_rejected ... ok +test machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote ... ok +test spec::tests::unknown_root_and_nested_fields_are_rejected ... ok +test machine::parallel_tests::disjoint_agent_lanes_merge_pins_in_either_completion_order ... ok +test spec::tests::preflight_data_is_fail_closed ... ok +test spec::tests::zero_agent_flow_is_valid ... ok +test state::budget::tests::adds_costs_exactly_beyond_machine_decimal_precision ... ok +test state::budget::tests::overflow_and_malformed_cost_leave_total_unchanged ... ok +test state::tests::budget_decimal_strings_add_without_floats ... ok +test spec::tests::external_surface_paths_must_have_one_canonical_spelling ... ok +test machine::tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching ... ok +test spec::tests::workspace_mounts_and_worktrees_must_have_one_canonical_spelling ... ok +test verify::tests::deterministic_output_requires_successful_exit_and_content ... ok +test state::tests::journal_replays_data_gate_verdict_without_rerunning_completed_code ... ok +test verify::tests::an_unbounded_schema_in_a_journal_fails_its_gate_instead_of_aborting ... ok +test state::tests::a_completion_that_omits_a_surface_does_not_drop_it_from_the_pin_chain ... ok +test state::tests::end_pin_chain_is_enforced_and_a_broken_chain_is_a_hard_error ... ok +test schema::tests::references_the_bound_leaves_opaque_are_refused_by_the_engine ... ok +test spec::tests::the_full_ladder_parses_in_the_one_dialect ... ok +test verify::tests::json_schema_is_a_control_gate ... ok +test schema::tests::a_property_named_ref_is_not_a_reference ... ok +test schema::tests::shared_declarations_and_boolean_schemas_are_validated ... ok +test spec::tests::sdk_boundary_accepts_a_valid_10_000_step_reverse_chain ... ok +test spec::tests::sdk_boundary_rejects_a_10_000_step_cycle_with_a_typed_error ... ok +test schema::tests::every_accepted_corpus_schema_is_accepted ... ok +test schema::tests::every_refused_corpus_schema_compiles_but_is_refused_by_the_bound ... ok +test schema::tests::deeply_nested_schemas_do_not_overflow_the_checker ... ok + +test result: ok. 72 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.72s + + Running tests/memoization.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/memoization-79f552e4cb727501) + +running 4 tests +test distinct_large_kernel_integers_do_not_alias_through_float_rounding ... ok +test match_reuses_output_with_provenance_and_zero_cost_without_dispatch ... ok +test changed_spec_or_input_dispatches_and_legacy_or_failed_records_miss ... ok +test canonical_corpus_agrees_with_typescript_and_key_permutations ... ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Running tests/spec_parity.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/spec_parity-a9966affd5aca87f) + +running 10 tests +test the_kernel_parses_the_event_triggered_spec_and_stamps_the_same_hash ... ok +test placement_requirements_have_identical_canonical_bytes_and_hash ... ok +test the_kernel_round_trips_declared_agent_transports_and_rejects_unknown_values ... ok +test memory_declaration_acceptance_matches_the_sdk_corpus ... ok +test the_kernel_parses_the_deterministic_rung_and_stamps_the_same_hash ... ok +test the_kernel_parses_the_rung_c_agent_spec_and_stamps_the_same_hash ... ok +test step_memory_has_identical_canonical_bytes_and_hash ... ok +test placement_declaration_acceptance_matches_the_sdk_corpus ... ok +test the_kernel_parses_the_sdk_compiled_spec_and_stamps_the_same_hash ... ok +test the_kernel_parses_the_rung_b_spec_and_stamps_the_same_hash ... ok + +test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s + + Running unittests src/lib.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/relayflowd_journal-0286edd157fee7d6) + +running 32 tests +test channel::tests::stale_attempts_and_raw_forged_acknowledgements_cannot_change_offsets ... ok +test registry::tests::a_same_boot_claim_with_no_run_yet_is_a_duplicate_not_wreckage ... ok +test registry::tests::a_previous_boots_claim_with_no_run_is_repaired ... ok +test subscriptions::tests::sweep_marks_row_stale_when_silence_exceeds_budget ... ok +test registry::tests::registry_is_a_rebuildable_run_locator ... ok +test registry::tests::run_admission_reuses_registered_run_and_rejects_spec_drift ... ok +test registry::tests::prior_boot_unregistered_run_admission_is_repaired ... ok +test registry::tests::a_registered_run_dedupes_across_boots ... ok +test subscriptions::tests::sweep_ignores_subscriptions_whose_silence_is_still_within_budget ... ok +test registry::tests::releasing_is_scoped_to_the_claiming_run ... ok +test subscriptions::tests::detect_without_latch_stays_available_for_the_next_sweep ... ok +test subscriptions::tests::last_run_for_subscription_returns_none_before_first_arrival ... ok +test subscriptions::tests::sweep_does_not_re_emit_the_same_stale_row_on_a_later_tick ... ok +test registry::tests::releasing_a_claim_lets_the_same_boot_retry ... ok +test subscriptions::tests::prune_sweep_claims_deletes_only_rows_older_than_cutoff ... ok +test subscriptions::tests::sweep_election_gives_the_first_caller_the_result_and_second_gets_empty ... ok +test subscriptions::tests::latch_is_a_no_op_if_a_fresh_event_arrived_between_detect_and_latch ... ok +test subscriptions::tests::last_run_for_subscription_returns_the_lex_greatest_ulid_regardless_of_insertion ... ok +test registry::tests::concurrent_new_boot_retries_have_one_recovery_owner ... ok +test channel::tests::channels_cross_segment_boundaries_and_terminal_runs_reject_mutations ... ok +test subscriptions::tests::upsert_after_stale_re_arms_and_next_silence_can_re_emit ... ok +test registry::tests::concurrent_same_boot_run_admissions_have_one_owner ... ok +test registry::tests::a_pre_migration_registry_gains_boot_id_and_its_claims_are_repairable ... ok +test subscriptions::tests::upsert_is_idempotent_across_bumps_and_preserves_event_type_updates ... ok +test tests::failed_commit_is_returned_not_swallowed ... ok +test tests::an_unconfirmed_election_is_reclaimed_by_the_next_attempt_not_treated_as_done ... ok +test tests::rollover_is_atomic_scaffolding_for_epoch_resume ... ok +test tests::terminal_run_refuses_every_later_entry_atomically ... ok +test tests::append_is_durable_and_monotonic_after_reopen ... ok +test tests::effects_are_deduplicated_at_the_journal_boundary ... ok +test channel::tests::failed_channel_writes_never_expose_delivery_or_advance_acknowledged_offset ... ok +test channel::tests::independent_connections_serialize_send_receive_and_acknowledgement ... ok + +test result: ok. 32 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.26s + + Doc-tests relayflowd + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Doc-tests relayflowd_core + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + + Doc-tests relayflowd_journal + +running 0 tests + +test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s + +exit_code=0 diff --git a/kernel/evidence/501/green-sdk-bundle-pristine.txt b/kernel/evidence/501/green-sdk-bundle-pristine.txt new file mode 100644 index 000000000..564ffe51b --- /dev/null +++ b/kernel/evidence/501/green-sdk-bundle-pristine.txt @@ -0,0 +1,22 @@ +# The full-suite run (green-sdk.txt) had node_modules overridden with the locally packed surface (--no-save), which resolved @agent-relay/cli-surface to 12.4.0 against the lockfile pin 12.2.4; bundle.test.ts refuses to build from a non-pristine tree. Re-run that file from a pristine `npm ci`: +$ cd packages/sdk && npm ci --ignore-scripts && npx vitest run tests/bundle.test.ts +added 193 packages in 985ms + + RUN v2.1.9 /tmp/flows-fleet-501/packages/sdk + + ✓ tests/bundle.test.ts (23 tests) 5279ms + ✓ immutable bundles > verifies with --verify in any position and answers --json with one object 528ms + ✓ immutable bundles > builds and verifies the canonical YAML fixture through the compiled CLI 711ms + ✓ immutable bundles > emits the ephemeral warning on CLI stderr and uses the default output directory 414ms + ✓ immutable bundles > builds a standalone TS fixture twice with identical executable hashes 1422ms + + Test Files 1 passed (1) + Tests 23 passed (23) + Start at 02:51:18 + Duration 6.00s (transform 289ms, setup 0ms, collect 526ms, tests 5.28s, environment 0ms, prepare 89ms) + +exit_code=0 + +# authored-node-runtime.test.ts pins bun --version to exactly 1.4.0; this machine has: +$ bun --version +1.4.2 diff --git a/kernel/evidence/501/green-sdk.txt b/kernel/evidence/501/green-sdk.txt new file mode 100644 index 000000000..5eaf05e9a --- /dev/null +++ b/kernel/evidence/501/green-sdk.txt @@ -0,0 +1,453 @@ +$ cd packages/sdk && RELAYFLOWD_BIN=/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/relayflowd npx vitest run + + RUN v2.1.9 /tmp/flows-fleet-501/packages/sdk + + ✓ tests/run-state.test.ts (21 tests) 9ms + ✓ tests/agent-transcript.test.ts (29 tests) 310ms + ✓ tests/journal-client.test.ts (15 tests) 112ms +stdout | tests/live-kernel.test.ts +LIVE_KERNEL relayflowd=/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/relayflowd +LIVE_KERNEL flows=/tmp/flows-fleet-501/packages/sdk/dist/cli.js + + ❯ tests/authored-node-runtime.test.ts (14 tests | 14 skipped) 17ms + ✓ tests/tick-source.test.ts (33 tests) 23ms + ✓ tests/daemon-lifecycle.test.ts (42 tests) 63ms + ✓ tests/relay-cli-surface.test.ts (66 tests) 65ms + ✓ tests/cloud-read.test.ts (39 tests) 106ms + ✓ tests/observer-link.test.ts (39 tests) 192ms + ✓ tests/validate.test.ts (68 tests) 108ms + ✓ tests/preflight.test.ts (57 tests) 185ms + ✓ tests/close-pr-flow.test.ts (28 tests) 663ms + ✓ close-pr journaled repair loop > executes the deterministic commit and force-push steps against a local Git remote, including a no-op repair 347ms + ✓ tests/authored-root.test.ts (12 tests) 238ms + ✓ tests/cloud-run.test.ts (58 tests) 607ms + ✓ tests/authored-flow.test.ts (26 tests) 914ms + ✓ tests/verb-field-lint.test.ts (96 tests) 745ms + ✓ tests/cloud-deploy.test.ts (40 tests) 903ms + ✓ tests/cli-status.test.ts (26 tests) 1118ms + ✓ flows status > resolves the run with no arguments from inside a worker-spawned agent 980ms + ✓ tests/step-failure-diagnostic.test.ts (22 tests) 1361ms + ✓ step failure diagnostic > surfaces command exit, stderr and replay hint through the CLI (json=false) 841ms + ✓ step failure diagnostic > surfaces command exit, stderr and replay hint through the CLI (json=true) 504ms + ✓ tests/authored-flow-lifecycle-executor.test.ts (27 tests) 726ms + ✓ tests/authored-human.test.ts (13 tests) 253ms + ✓ tests/gate-contract.test.ts (20 tests) 285ms + ✓ tests/cli-hn-monitor.test.ts (16 tests) 106ms + ✓ tests/authored-node-result.test.ts (38 tests) 20ms + ✓ tests/backlog-picker.test.ts (14 tests) 80ms + ✓ tests/cloud-sync.test.ts (40 tests) 2422ms + ✓ tests/cli-replay.test.ts (37 tests) 1462ms + ✓ flows replay > --json is byte-identical across two CLI invocations (diff) 1227ms + ✓ tests/cloud-connect.test.ts (24 tests) 2887ms + ✓ hosted verbs connect before they submit > flows run --cloud submits once the prompt connected the integration 2111ms +(node:2335415) Warning: Transcript tail for run-9/analyze attempt 1 (stdout) could not be written; the step continues without it: EACCES: permission denied, mkdir '/tmp/transcript-tail-8P264q/runs/run-9/steps' +(Use `node --trace-warnings ...` to show where the warning was created) + ✓ tests/transcript-tail.test.ts (11 tests) 935ms + ✓ direct agent spawn > tees stdout and stderr into tail files that name the dispatch 372ms + ✓ tests/agent-relay-transport.test.ts (16 tests) 2315ms + ✓ Relay completion at the journal boundary > does not complete at readiness and journals exact output, receipt, and priced accounting 1016ms + ✓ Relay completion at the journal boundary > aborts polling on rejected renewal and never writes a stale completion 1006ms + ✓ tests/backlog-picker-flow.test.ts (6 tests) 420ms + ✓ tests/authored-agent-artifacts.test.ts (4 tests) 514ms + ✓ tests/authored-flow-slack.test.ts (7 tests) 2279ms + ✓ authored Slack helper effects > replays after SIGKILL before confirm with the same token and one successful completion 828ms + ✓ authored Slack helper effects > replays after SIGKILL before complete with the same token and one successful completion 816ms + ✓ authored Slack helper effects > writes two files for two calls and supports dm, reply, and react 314ms + ✓ tests/pr-review-post.test.ts (21 tests) 3192ms + ✓ tests/preflight-permissions-unenforced.test.ts (17 tests) 198ms + ✓ tests/tick-runner.test.ts (22 tests) 3535ms + ✓ CLI argument parsing refuses coercion rather than accepting it > refuses --interval-ms fractional as an invocation error 530ms + ✓ CLI argument parsing refuses coercion rather than accepting it > refuses --interval-ms exponent notation as an invocation error 577ms + ✓ CLI argument parsing refuses coercion rather than accepting it > refuses --interval-ms hex as an invocation error 655ms + ✓ CLI argument parsing refuses coercion rather than accepting it > refuses --interval-ms trailing text as an invocation error 543ms + ✓ CLI argument parsing refuses coercion rather than accepting it > refuses --interval-ms empty as an invocation error 625ms + ✓ CLI argument parsing refuses coercion rather than accepting it > accepts an exact integer and proceeds past parsing 534ms + ✓ tests/flow-requirements.test.ts (13 tests) 380ms + ✓ tests/worker-transcript.test.ts (5 tests) 791ms + ✓ tests/webhook.test.ts (9 tests) 460ms + ✓ tests/authored-step-failed.test.ts (10 tests) 62ms + ✓ tests/authored-flow-operation.test.ts (23 tests) 520ms + ✓ completes the gate in linear time over a body with 30000 ordinary awaits 373ms + ✓ tests/authored-step-index.test.ts (12 tests) 21ms + ✓ tests/authored-run-failure-evidence.test.ts (8 tests) 941ms + ✓ the child index after the process that wrote it is gone > still names every child, with its own run id, after a daemon restart 580ms + ✓ tests/budget-preflight.test.ts (25 tests) 41ms + ✓ tests/artifact-gates.test.ts (6 tests) 222ms + ✓ tests/spec-parity.test.ts (33 tests) 667ms + ✓ tests/stuck-run-triage.test.ts (22 tests) 3297ms + ✓ stuck-run-triage shell text > collects tails with no GNU timeout on PATH, as on a stock macOS 3176ms + ✓ tests/work-package-consumer.test.ts (13 tests) 204ms + ✓ tests/stop-process-group.test.ts (6 tests) 6042ms + ✓ every stop reaches the process group, not just the direct child > exits the run after an execution-timeout stop 748ms + ✓ every stop reaches the process group, not just the direct child > exits the run after a protocol terminate stop 338ms + ✓ every stop reaches the process group, not just the direct child > kills a SIGTERM-deaf grandchild after a protocol terminate stop 1563ms + ✓ every stop reaches the process group, not just the direct child > kills a SIGTERM-deaf grandchild after an execution-timeout stop 1917ms + ✓ every stop reaches the process group, not just the direct child > holds the loop open long enough for the escalation to run 1127ms + ✓ every stop reaches the process group, not just the direct child > terminate() forces a group that outlives SIGTERM 348ms + ✓ tests/provider-trigger-contract.test.ts (7 tests) 296ms + ✓ tests/agent-transcript-live.test.ts (4 tests) 5013ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured agent failure details and its completed root index 1267ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > preserves structured llm failure details and its completed root index 1416ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > journals the digest in trajectory_tail on a successful agent step and writes the file it points at 1123ms + ✓ the transcript digest through the built CLI, a real daemon and the local agent > on a failed agent step, names the failure and the transcript in the terminal diagnostic, redacted 1205ms + ✓ tests/authored-helpers.test.ts (6 tests) 4145ms + ✓ runs every available provider through the real kernel and resumes completed effects without a second write 2161ms + ✓ replays after SIGKILL before confirm with the same token and one successful completion 839ms + ✓ replays after SIGKILL before complete with the same token and one successful completion 758ms + ✓ tests/webhook-hardening.test.ts (11 tests) 127ms + ✓ tests/human-to.test.ts (8 tests) 23ms + ✓ tests/helpers-fanout.test.ts (96 tests) 208ms + ✓ tests/budget-unmetered-live.test.ts (3 tests) 1695ms + ✓ unmetered budget spend through the live kernel > runs an unpriced step under a dollar budget without tripping it, journaling unknown dollars 804ms + ✓ unmetered budget spend through the live kernel > still counts an unpriced step toward a token budget 469ms + ✓ unmetered budget spend through the live kernel > accrues a priced step and stops the run when it crosses the dollar budget 421ms + ✓ tests/worker-lease.test.ts (7 tests) 16ms + ✓ tests/plugin-loader.test.ts (9 tests) 263ms + ✓ tests/agent-artifacts-live.test.ts (5 tests) 6326ms + ✓ agent artifacts and gates through the built CLI, a real daemon and the local agent > journals the files the agent wrote, and both artifact gates pass on that journal 1442ms + ✓ agent artifacts and gates through the built CLI, a real daemon and the local agent > fails the run when the artifact_exists gate names a file the agent did not write 1474ms + ✓ agent artifacts and gates through the built CLI, a real daemon and the local agent > fails the run with the author reason when a predicate gate returns false, journaling the verdict 1004ms + ✓ review follow-ups > applies a predicate gate on a helper step too, and journals its verdict 1222ms + ✓ review follow-ups > records predicate verdicts on the root run so a resume reuses them instead of re-running the closure 1181ms + ✓ tests/redact.test.ts (35 tests) 14ms + ✓ tests/yaml-helpers.test.ts (33 tests) 166ms + ✓ tests/generate-triggers.test.ts (7 tests) 1520ms + ✓ discovers new adapters, preserves exact event names, and prefers adapter-local mappings 523ms + ✓ tests/communication.test.ts (10 tests) 29ms + ✓ tests/budget-attribution.test.ts (5 tests) 16ms + ✓ tests/cloud-schedule.test.ts (17 tests) 6607ms + ✓ schedule lowering > marks a non-grid cron as Cloud-only rather than approximating it, with a silence budget from its own cadence 4385ms + ✓ flows check prints declared schedules > shows the lowering for a fixed interval and the Cloud-only note for a real cron 1885ms + ✓ tests/cli.test.ts (65 tests) 8318ms + ✓ flows check CLI > binds a checked relative wrapper to the flow directory for worker execution 343ms + ✓ flows check CLI > refuses a typo model before probing or contacting relayflowd 618ms + ✓ flows check CLI > maps every input refusal path to its declared kind without raw exceptions 513ms + ✓ flows run/resume CLI over the journal protocol > exits 1 and emits the declared completionReason for a failed run 590ms + ✓ flows run/resume CLI over the journal protocol > exits 3 and names the parked llm step 659ms + ✓ flows run/resume CLI over the journal protocol > reports a needs_human agent step as parked for human recovery 490ms + ✓ flows run/resume CLI over the journal protocol > classifies a typed hello refusal as a protocol error, not an unreachable daemon 834ms + ✓ flows run/resume CLI over the journal protocol > follows a dispatched worker step instead of reporting a protocol error 701ms + ✓ flows run/resume CLI over the journal protocol > bounds a worker wait by its lease and reports what it is waiting for 537ms + ✓ flows run/resume CLI over the journal protocol > maps only run_not_found resumes to exit 2 1609ms + ✓ tests/relayflowd-path.test.ts (10 tests) 8ms + ✓ tests/mcp.test.ts (30 tests) 9060ms + ✓ MCP preflight and transports > flows check refuses an undeclared server with exit 2 and no daemon 718ms + ✓ MCP preflight and transports > flows check reports a refusing server and leaves no PID 766ms + ✓ MCP preflight and transports > kills a SIGTERM-resistant silent child after a parent-owned handshake deadline 1339ms + ✓ MCP preflight and transports > reaps a SIGTERM-resistant descendant with inherit stdio before cleanup finishes 1162ms + ✓ MCP preflight and transports > reaps a SIGTERM-resistant descendant with ignore stdio before cleanup finishes 2084ms + ✓ MCP preflight and transports > reports malformed connection configuration as config_invalid 726ms + ✓ authored MCP effects against the real kernel > journals one MCP receipt per call with args, result, stable logical key, and a confirmed effect 303ms + ✓ authored MCP effects against the real kernel > reports a dropped tool connection as a failed CLI run 947ms + ✓ tests/model-selection.test.ts (10 tests) 33ms + ✓ tests/authored-agent-permissions.test.ts (27 tests) 1366ms + ✓ tests/mcp-lifecycle.test.ts (4 tests) 20ms + ✓ tests/effect-channel.test.ts (5 tests) 412ms + ✓ tests/typed-output.test.ts (14 tests) 545ms + ✓ tests/local-dev-ux.test.ts (8 tests) 18ms + ✓ tests/authored-plugin-effect.test.ts (6 tests) 155ms + ✓ tests/deterministic-llm.test.ts (5 tests) 125ms + ✓ tests/authored-declined.test.ts (13 tests) 115ms + ✓ tests/resume-failure.test.ts (2 tests) 7ms + ✓ tests/relay-cli-surface-live.test.ts (3 tests) 489ms + ✓ tests/communication-review.test.ts (5 tests) 342ms + ✓ tests/input-binding.test.ts (12 tests) 375ms + ✓ tests/scope-compiler.test.ts (25 tests) 25ms + ✓ tests/yaml-helper-effect.test.ts (4 tests) 97ms + ✓ tests/f-memory.test.ts (7 tests) 1618ms + ✓ reads the seeded local SQLite database with cloud unused and fallback disabled 330ms + ✓ tests/dependency-validation.test.ts (6 tests) 1124ms + ✓ dependency validation > accepts a valid 10,000-step reverse chain through every direct public boundary 604ms + ✓ dependency validation > bounds the author-facing path for a 10,000-step cycle 455ms + ✓ tests/scope-preflight.test.ts (6 tests) 11ms + ✓ tests/direct-input.test.ts (6 tests) 10080ms + ✓ direct .flow.ts input through the built CLI and live runtime > returns exit 3 for an authored human handoff and persists its outcome 959ms + ✓ direct .flow.ts input through the built CLI and live runtime > returns exit 1 for an authored step_failed verdict and persists its outcome 844ms + ✓ direct .flow.ts input through the built CLI and live runtime > executes inline and file JSON input through relayflowd 3673ms + ✓ direct .flow.ts input through the built CLI and live runtime > refuses missing and malformed input before contacting relayflowd 3158ms + ✓ direct .flow.ts input through the built CLI and live runtime > does not run the authored body before daemon availability 655ms + ✓ direct .flow.ts input through the built CLI and live runtime > refuses oversized file input before contacting relayflowd 789ms + ✓ tests/hn-poller.test.ts (6 tests) 17ms + ❯ tests/bundle.test.ts (23 tests | 1 failed) 10136ms + ✓ immutable bundles > returns exit 2 naming a byte-flipped payload and refuses to reuse corruption 708ms + ✓ immutable bundles > verifies with --verify in any position and answers --json with one object 1123ms + ✓ immutable bundles > refuses --out with --verify rather than ignoring the destination 519ms + ✓ immutable bundles > builds and verifies the canonical YAML fixture through the compiled CLI 1613ms + ✓ immutable bundles > emits the ephemeral warning on CLI stderr and uses the default output directory 1101ms + ✓ immutable bundles > refuses build-provable CLI resolution errors without environment probes 547ms + × immutable bundles > builds a standalone TS fixture twice with identical executable hashes 681ms + → expected 'REFUSED [bundle_invalid] package-lock…' to be '' // Object.is equality + ✓ immutable bundles > refuses to label installed dependency drift with lockfile pins 618ms + ✓ immutable bundles > refuses invalid CLI arguments %j 541ms + ✓ immutable bundles > refuses invalid CLI arguments "--out" 688ms + ✓ immutable bundles > refuses invalid CLI arguments "--verify" 600ms + ✓ immutable bundles > refuses invalid CLI arguments "--verify" 637ms + ✓ immutable bundles > refuses invalid CLI arguments "--out" 591ms + ✓ tests/dir-watcher-poller.test.ts (6 tests) 7ms + ✓ tests/model-pricing.test.ts (10 tests) 15ms + ✓ tests/authored-step-failed-exit.test.ts (3 tests) 15ms + ✓ tests/direct-run-failure.test.ts (8 tests) 17ms + ✓ tests/build-gate.test.ts (3 tests) 1413ms + ✓ flows build gates on flows check green (#318) > refuses a flow with an unresolvable named-agent CLI and leaves no artifacts 471ms + ✓ flows build gates on flows check green (#318) > --json emits one CheckReport object on stdout on refusal, exits 2, no artifacts 462ms + ✓ flows build gates on flows check green (#318) > builds the bundle on success (regression: gate must not block valid flows) 479ms + ✓ tests/provider-trigger-executor.test.ts (4 tests) 262ms + ✓ tests/json-schema-bound.test.ts (71 tests) 3690ms + ✓ JSON Schema termination bound > walks a deep schema with an explicit stack rather than recursion 2717ms + ✓ tests/wrapper-artifacts-cwd.test.ts (2 tests) 93ms + ✓ tests/hello-deterministic.test.ts (5 tests) 25ms + ✓ tests/cli-adapter.test.ts (4 tests) 11ms + ✓ tests/communication-worker.test.ts (15 tests) 1523ms + ✓ tests/daemon-lifecycle-live.test.ts (9 tests) 11501ms + ✓ flows run against a data dir with no daemon (§6 test 7) > cold start spawns exactly one daemon, the run succeeds, and the daemon outlives the CLI 904ms + ✓ flows run against a data dir with no daemon (§6 test 7) > polls, bounded, for a daemon that holds the lock before it binds 1603ms + ✓ flows run against a data dir with no daemon (§6 test 7) > attaches to a serving daemon that has not published a connection file 1457ms + ✓ flows run against a data dir with no daemon (§6 test 7) > a second run attaches to the daemon the first one started, spawning nothing 1591ms + ✓ flows run against a data dir with no daemon (§6 test 7) > detects a stale connection file left by a hard kill and starts a fresh daemon 1906ms + ✓ concurrent invocations against one empty data dir (§6 test 15) > ends with exactly one daemon owning the socket, and both runs succeed 1279ms + ✓ refusals from a spawn that cannot produce a daemon > names relayflowd_not_found rather than falling through to PATH 701ms + ✓ refusals from a spawn that cannot produce a daemon > names daemon_start_failed and quotes the daemon log when startup dies 759ms + ✓ refusals from a spawn that cannot produce a daemon > refuses a daemon speaking another protocol version instead of binding over it 1299ms + ✓ tests/work-package-validator.test.ts (7 tests) 39ms + ✓ tests/human-live.test.ts (3 tests) 8355ms + ✓ f.human against a real daemon > parks with the question, refuses wrong answers, records one, and resumes to success 5058ms + ✓ f.human against a real daemon > a "no" is a value the body branches on: declined, exit 0, no effect 1776ms + ✓ f.human against a real daemon > refuses to answer a run the daemon does not know 1520ms + ✓ tests/transcript-exclusion-timeout.test.ts (1 test) 343ms + ✓ a transcript close that outruns its deadline > still keeps the transcript out of the agent's artifacts 340ms + ✓ tests/yaml-helper-live.test.ts (1 test) 1286ms + ✓ runs compiled YAML helpers through the built CLI and kernel effect journal 1285ms + ✓ tests/transcript-tail-close.test.ts (2 tests) 1097ms + ✓ a stalled transcript-tail close > does not hold the spawn open past its bounded window 565ms + ✓ a stalled tail close beside a transcript that finished > still journals the transcript pointer 531ms + ✓ tests/agent-relay-hardening.test.ts (12 tests) 32ms + ✓ tests/authored-use-loader.test.ts (5 tests) 467ms + ✓ tests/flow-executor-chain.test.ts (14 tests) 13863ms + ✓ flow executor LLM and output-binding chain > runs f.llm -> f.agent -> f.run with schema-verified journal output and the exact allowed model 904ms + ✓ flow executor LLM and output-binding chain > runs a dollar-budgeted authored Claude agent with the same default used by preflight 469ms + ✓ flow executor LLM and output-binding chain > fails invalid LLM output before the next step: not JSON 397ms + ✓ flow executor LLM and output-binding chain > fails invalid LLM output before the next step: {"message":7} 428ms + ✓ flow executor LLM and output-binding chain > preserves JSON values without promoting them to process wrappers: null 392ms + ✓ flow executor LLM and output-binding chain > preserves JSON values without promoting them to process wrappers: [1,2] 435ms + ✓ flow executor LLM and output-binding chain > preserves JSON values without promoting them to process wrappers: "hello" 459ms + ✓ flow executor LLM and output-binding chain > runs the exact authored flagship f.llm -> f.agent -> f.run path through the durable CLI root 2737ms + ✓ flow executor LLM and output-binding chain > resumes an interrupted durable authored root without replaying completed flagship effects 4323ms + ✓ flow executor LLM and output-binding chain > passes a declarative verified value through an agent into a deterministic artifact 735ms + ✓ flow executor LLM and output-binding chain > journals a missing optional field as a failure before the consuming command executes 423ms + ✓ flow executor LLM and output-binding chain > flows run consumes YAML bindings and resume reuses the original journal output 1902ms + ✓ tests/plugin-add.test.ts (7 tests) 1833ms + ✓ installs a real offline npm fixture and includes declarations 367ms + ✓ typechecks the augmented verb and rejects unknown namespaces 1443ms + ✓ tests/agent-artifacts.test.ts (6 tests) 17ms + ✓ tests/communication-mixed-resume.test.ts (1 test) 241ms + ✓ tests/communication-environment-preflight.test.ts (6 tests) 5ms + ✓ tests/cli-answer.test.ts (15 tests) 19ms + ✓ tests/communication-preflight.test.ts (13 tests) 65ms + ↓ tests/real-cli-adapters.test.ts (3 tests | 3 skipped) + ✓ tests/journal-client-completion.test.ts (4 tests) 108ms + ✓ tests/bin.test.ts (7 tests) 3843ms + ✓ built flows binary > refuses through a symlink to the built artifact 613ms + ✓ built flows binary > refuses through a symlinked directory component 632ms + ✓ built flows binary > classifies a signal-terminated auth probe as probe_failed 717ms + ✓ built flows binary > classifies an unavailable PATH resolver as probe_failed 824ms + ✓ built flows binary > does not describe a present non-executable CLI as missing 501ms + ✓ built flows binary > runs one auth probe for three steps sharing a flow CLI 545ms + ✓ tests/parse-json-output.test.ts (7 tests) 6ms + ✓ tests/authored-surface-authority.test.ts (2 tests) 18ms + ✓ tests/adapters/claude.test.ts (7 tests) 10ms + ✓ tests/pty-sidechannel.test.ts (12 tests) 8963ms + ✓ view attach preserves worker completion and marks only drive 837ms + ✓ passthrough attach preserves worker completion and marks only drive 1295ms + ✓ none attach preserves worker completion and marks only drive 1329ms + ✓ none subscriber lets an unattended CLI read EOF 739ms + ✓ view subscriber lets an unattended CLI read EOF 690ms + ✓ passthrough subscriber lets an unattended CLI read EOF 695ms + ✓ incomplete subscriber lets an unattended CLI read EOF 749ms + ✓ unattended Codex receives closed stdin before startup instead of entering its additional-input lifecycle 868ms + ✓ rejects drive after EOF without marking human intervention 935ms + ✓ delivers all drive bytes in order across child stdin backpressure 578ms + ✓ tests/adapters/codex.test.ts (7 tests) 9ms + ✓ tests/communication-history.test.ts (1 test) 4ms + ✓ tests/adapters/registry.test.ts (4 tests) 5ms + ✓ tests/worker-cli-cwd.test.ts (2 tests) 193ms + ✓ tests/webhook-live.test.ts (6 tests) 11176ms + ✓ executes and deduplicates 'app_mention' only for its provider and matching payload 1612ms + ✓ executes and deduplicates 'reaction_added' only for its provider and matching payload 1678ms + ✓ executes and deduplicates 'pull_request' only for its provider and matching payload 1792ms + ✓ flows serve-webhook writes JSON before the daemon starts, then journals and archives exactly once 1727ms + ✓ replays a dropped file after SIGKILL before spawn 744ms + ✓ resumes the same journal after SIGKILL after spawn and before acknowledgement 3620ms + ✓ tests/yaml-local-agent-live.test.ts (7 tests) 5759ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked step CLI and model and journals done 888ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked named CLI and model and journals done 980ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked flow CLI and model and journals done 640ms + ✓ YAML --local-agent through the built CLI and real daemon > runs with the checked project CLI and model and journals done 739ms + ✓ YAML --local-agent through the built CLI and real daemon > still parks without --local-agent 777ms + ✓ YAML --local-agent through the built CLI and real daemon > reports the agent process failure 954ms + ✓ YAML --local-agent through the built CLI and real daemon > preserves declared workspace surfaces that the local worker cannot pin 780ms + ✓ tests/slack-writeback.test.ts (1 test) 262ms + ✓ tests/budget-authored-live.test.ts (2 tests) 222ms + ✓ tests/memoization.test.ts (57 tests) 695ms + ✓ refuses invalid reuse invocation "run" 586ms + ✓ tests/bundle-preflight.test.ts (4 tests) 1353ms + ✓ bundle execution preflight > ignores surrounding cache configuration on a verified cache hit 731ms + ✓ bundle execution preflight > uses the built alias for a nameless flow even in a digest-only cache directory 577ms + ✓ tests/authored-declined-live.test.ts (1 test) 2251ms + ✓ runs an input guard and resumes its completed declined root without repeated effects 2247ms + ✓ tests/slack-block-kit.test.ts (5 tests) 25ms + ✓ tests/placement.test.ts (54 tests) 29ms + ✓ tests/authored-admission.test.ts (2 tests) 6ms + ✓ tests/communication-tools.test.ts (1 test) 103ms + ✓ tests/authored-declined-report.test.ts (6 tests) 12ms + ✓ tests/worker-platform.test.ts (1 test) 9ms + ✓ tests/communication-lazy.test.ts (1 test) 7ms + ✓ tests/communication-refusal.test.ts (1 test) 27ms + ✓ tests/check-command-cwd.test.ts (1 test) 17ms + ✓ tests/memory.test.ts (18 tests) 10ms + ✓ tests/deploy.test.ts (11 tests) 7363ms + ✓ flows deploy file buckets > publishes the full signed layout byte-for-byte and redeploys as a noop 1144ms + ✓ flows deploy file buckets > answers --json with one object per outcome 1118ms + ✓ flows deploy file buckets > reports a refusal as JSON under --json 579ms + ✓ flows deploy file buckets > refuses a missing local bundle before creating the bucket 500ms + ✓ flows deploy file buckets > refuses an unreachable bucket before copying 454ms + ✓ flows deploy file buckets > refuses an unwritable bucket 484ms + ✓ flows deploy file buckets > refuses local tampering of spec.canonical.json 897ms + ✓ flows deploy file buckets > refuses local tampering of identity.json 596ms + ✓ flows deploy file buckets > refuses asset bundles instead of using daemon-relative files 544ms + ✓ flows deploy file buckets > never labels a corrupt existing deployment as a noop 998ms + ✓ tests/classify-outcome.test.ts (2 tests) 2172ms + ✓ classifyOutcome > gives up and reports when a running run never becomes classifiable 2016ms + ✓ tests/worker-cli-abort.test.ts (2 tests) 2413ms + ✓ stops claude and its process group when lease ownership is lost 1198ms + ✓ stops wrapper.mjs and its process group when lease ownership is lost 1213ms + ✓ tests/run-digest-live.test.ts (1 test) 630ms + ✓ executes a deployed digest on the real kernel after deleting the authoring tree 629ms + ✓ tests/cli-progress-wait.test.ts (2 tests) 1105ms + ✓ run starts the wait clock on its first observed lease 617ms + ✓ resume starts the wait clock on its first observed lease 486ms + ✓ tests/run-from-digest.test.ts (6 tests) 5573ms + ✓ flows run digest input > submits the sealed canonical spec through the normal journal path without checkout 547ms + ✓ flows run digest input > uses a verified cache hit even after the bucket is removed 881ms + ✓ flows run digest input > resolves deploy.bucket from flows.json and honors explicit override 1729ms + ✓ flows run digest input > refuses an unconfigured bucket 902ms + ✓ flows run digest input > refuses tampered spec.canonical.json before creating run data 784ms + ✓ flows run digest input > refuses tampered identity.json before creating run data 728ms + ✓ tests/run-digest.test.ts (4 tests) 1505ms + ✓ digest run configuration refusals > reports config_invalid before fetching or starting a run for {invalid json 414ms + ✓ digest run configuration refusals > reports config_invalid before fetching or starting a run for {"deploy":{}} 377ms + ✓ digest run configuration refusals > reports config_invalid before fetching or starting a run for {"deploy":{"bucket":123}} 351ms + ✓ digest run configuration refusals > reports config_invalid before fetching or starting a run for {"deploy":{"bucket":""}} 362ms + ✓ tests/bundle-transport.test.ts (20 tests) 2096ms + ✓ digest references > accepts and deploys the build output for hello 413ms + ✓ digest references > accepts and deploys the build output for Hello 414ms + ✓ digest references > accepts and deploys the build output for hello.world 384ms + ✓ digest references > accepts and deploys the build output for hello_world 382ms + ✓ tests/cli-watch.test.ts (10 tests) 17421ms + ✓ flows check --watch > rechecks syntax errors, clears once, and returns the last refusal on Ctrl-C 1647ms + ✓ flows check --watch > streams JSON lines without ANSI, recovers after atomic saves, and exits zero after repair 2574ms + ✓ flows check --watch > coalesces 20 concurrent saves into at most two rechecks 2463ms + ✓ flows check --watch > watches transitive relative use imports, cycles, and nearest config changes 3426ms + ✓ flows check --watch > refreshes the import graph and notices missing imports being created 2584ms + ✓ flows check --watch > reloads authored TypeScript instead of reusing the first imported definition 1134ms + ✓ flows check --watch > detects a nearer config appearing and falls back after it is deleted 1397ms + ✓ flows check --watch > keeps watching after the target is deleted and recreated 1413ms + ✓ flows check --watch > queues changes during a slow check without overlapping checks 778ms + ✓ tests/worker-cli.test.ts (22 tests) 27359ms + ✓ direct transport lifecycle evidence > classifies only the exact Codex stdin lifecycle signature as retryable 320ms + ✓ direct transport lifecycle evidence > records a signal close separately from an ordinary nonzero exit 1472ms + ✓ direct transport lifecycle evidence > records a spawn error code without treating a missing executable as transient 640ms + ✓ direct transport lifecycle evidence > journals classified lifecycle evidence and reports crashed instead of generic worker_error 748ms + ✓ step discovery environment > names the run, step, attempt and an absolute data dir for a direct agent spawn 799ms + ✓ step discovery environment > exports none of the four without a data dir, even when the worker inherited them 510ms + ✓ wrapper discovery environment > sets the four names from the dispatch and still refuses ambient values and other secrets 493ms + ✓ wrapper discovery environment > exports none of the four to a wrapper without a data dir, even when the worker inherited them 595ms + ✓ custom wrapper execution identity > passes an explicit safe environment at identification and execution 555ms + ✓ custom wrapper execution identity > refuses a wrapper symlink retarget before delivering private values 557ms + ✓ custom wrapper execution identity > bounds wrapper execution after acknowledgement 353ms + ✓ custom wrapper execution identity > bounds captured wrapper output 496ms + ✓ custom wrapper execution identity > refuses a duplicate execute protocol frame 427ms + ✓ custom wrapper execution bounds are reader-owned > resolves when a conforming wrapper leaks a stdio pipe to a background helper 1993ms + ✓ custom wrapper execution bounds are reader-owned > resolves when the leaked helper inherits stderr only 1871ms + ✓ custom wrapper execution bounds are reader-owned > resolves when a wrapper leaks a stdio pipe and exits before identifying 3406ms + ✓ custom wrapper execution bounds are reader-owned > journals a completionReason at the default bound when a wrapper leaks a stdio pipe 11384ms + ✓ tests/worker-cli-result-exit.test.ts (5 tests) 32878ms + ✓ a Claude agent step completes on its result, not only on process exit > settles a hung, successful run within the grace and stops its whole tree 31632ms + ✓ a Claude agent step completes on its result, not only on process exit > maps an error result on a hung run to a failed exit 31631ms + ✓ a Claude agent step completes on its result, not only on process exit > leaves a hang before any result to the existing stops 32019ms + ✓ an agent tree does not outlive the process that spawned it > kills the agent group when the run process is terminated by SIGTERM 780ms + ✓ tests/local-agent-live.test.ts (5 tests) 39499ms + ✓ built CLI local agent against a real daemon > dispatches through the wrapper and keeps --json stdout report-shaped 1237ms + ✓ built CLI local agent against a real daemon > runs beyond the initial 30-second lease without a second invocation 36173ms + ✓ built CLI local agent against a real daemon > renders actual agent completion in text output 523ms + ✓ built CLI local agent against a real daemon > returns a failed run when the agent process fails 803ms + ✓ built CLI local agent against a real daemon > refuses a workspace it cannot pin before invoking the agent 762ms +stdout | tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +LIVE_ANALYZER ready: claude -p --model claude-haiku-4-5-20251001 round-trip OK + +stdout | tests/live-kernel.test.ts > built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI +LIVE_ANALYZER analysis: {"reasoning":"This story is directly relevant to AI agents and automation, describing an agent system capable of autonomously performing software engineering tasks such as opening and reviewing pull requests, which represents a core use case in agent-based automation.","relevance_score":9,"story_title":"Show HN: an agent that opens and reviews its own pull requests [wake-nonce-7f3a91c4]"} + +stdout | tests/live-kernel.test.ts > surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once +LIVE_KERNEL kill -9 pid=2349976 run=01M2Z3JEZVY46W10Q5ATZXN2G1 while step=two state=Running + + ✓ tests/live-kernel.test.ts (31 tests) 62658ms + ✓ built flows CLI against live relayflowd > twenty-six-step reuses 25 durable completions after editing the failed final step 3019ms + ✓ built flows CLI against live relayflowd > runs rung (a), parks rung (b), and keeps JSON report-shaped 5323ms + ✓ built flows CLI against live relayflowd > allows a deterministic run to exceed the bounded request timeout 32560ms + ✓ built flows CLI against live relayflowd > follows a live worker dispatch through flows run 1001ms + ✓ built flows CLI against live relayflowd > can always get a parked run to a late-attaching worker 5566ms + ✓ built flows CLI against live relayflowd > reports a real manual-recovery NeedsHuman state as parked 778ms + ✓ built flows CLI against live relayflowd > hn-monitor analyze-story reaches done through the real Claude analyzer CLI 8430ms + ✓ built flows CLI against live relayflowd > preflights before journaling and names an unreachable socket 1694ms + ✓ built flows CLI against live relayflowd > starts exactly one daemon when two runs race for one empty data dir 911ms + ✓ surface resume after a real daemon kill > resumes a three-step run with each successful completion exactly once 922ms + ✓ tests/step-lease.test.ts (36 tests) 66504ms + ✓ f.run leases against the live kernel > enforces 10000 ms for 'sleep 5; printf ok' 5132ms + ✓ f.run leases against the live kernel > enforces 40000 ms for 'sleep 31; printf ok' 31088ms + ✓ f.run leases against the live kernel > enforces 30000 ms for 'sleep 31; printf ok' 30078ms + +⎯⎯⎯⎯⎯⎯ Failed Suites 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/authored-node-runtime.test.ts [ tests/authored-node-runtime.test.ts ] +AssertionError: expected '1.4.2' to be '1.4.0' // Object.is equality + +Expected: "1.4.0" +Received: "1.4.2" + + ❯ tests/authored-node-runtime.test.ts:18:77 + 16| + 17| beforeAll(() => { + 18| expect(spawnSync(bun, ['--version'], { encoding: 'utf8' }).stdout.tr… + | ^ + 19| expect(existsSync(daemon), 'build the current kernel or set RELAYFLO… + 20| stage = mkdtempSync(join(tmpdir(), 'authored-standalone-build-')); + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[1/2]⎯ + +⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯ + + FAIL tests/bundle.test.ts > immutable bundles > builds a standalone TS fixture twice with identical executable hashes +AssertionError: expected 'REFUSED [bundle_invalid] package-lock…' to be '' // Object.is equality + +- Expected ++ Received + ++ REFUSED [bundle_invalid] package-lock.json: node_modules/@agent-relay/cli-surface does not match its pinned version; run npm ci before building ++ + + ❯ tests/bundle.test.ts:233:27 + 231| it('builds a standalone TS fixture twice with identical executable h… + 232| const result = invoke(['--out', await temp(), 'packages/sdk/tests/… + 233| expect(result.stderr).toBe(''); expect(result.status).toBe(0); + | ^ + 234| const bundle = result.stdout.trim(); + 235| const second = invoke(['--out', await temp(), 'packages/sdk/tests/… + +⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯[2/2]⎯ + + Test Files 2 failed | 154 passed | 1 skipped (157) + Tests 1 failed | 2410 passed | 17 skipped (2428) + Start at 02:49:13 + Duration 76.21s (transform 6.22s, setup 0ms, collect 84.85s, tests 464.50s, environment 33ms, prepare 11.75s) + +exit_code=1 diff --git a/kernel/evidence/501/green-targeted.txt b/kernel/evidence/501/green-targeted.txt new file mode 100644 index 000000000..7e69e547b --- /dev/null +++ b/kernel/evidence/501/green-targeted.txt @@ -0,0 +1,54 @@ +# Production files restored byte-for-byte (sha256sum -c against pre-mutation copies: OK). Same commands as red.txt: +$ cd kernel && cargo test -p relayflowd-core --lib -- machine::tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote machine::tests::reset_recovery_still_retries_a_worker_reported_transport_loss + Compiling relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.60s + Running unittests src/lib.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/relayflowd_core-82d87717b3255c51) + +running 3 tests +test machine::tests::reset_recovery_still_retries_a_worker_reported_transport_loss ... ok +test machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote ... ok +test machine::tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 69 filtered out; finished in 0.00s + +exit_code=0 + +$ cd kernel && cargo test -p relayflowd --test manual_recovery + Compiling relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) + Compiling relayflowd-journal v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-journal) + Compiling relayflowd v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd) + Finished `test` profile [unoptimized + debuginfo] target(s) in 1.00s + Running tests/manual_recovery.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/manual_recovery-6793123a1244aad2) + +running 2 tests +test a_park_torn_between_its_two_appends_is_repaired_on_resume ... ok +test manual_park_of_a_worker_reported_loss_survives_reopen_and_answers_to_a_human ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s + +exit_code=0 + +$ cd kernel && cargo test -p relayflowd --test crash_resume manual_recovery + Compiling relayflowd v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd) + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.40s + Running tests/crash_resume.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/crash_resume-d2d102ec3e89b705) + +running 1 test +test manual_recovery::manual_recovery_parks_a_worker_reported_loss_across_daemon_restart_until_a_human_answers ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 40 filtered out; finished in 1.46s + +exit_code=0 + +$ cd kernel && cargo test -p relayflowd-core --lib -- machine::tests::all_backing_off_steps_return_timers entry::attempt_started_tests + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.02s + Running unittests src/lib.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/relayflowd_core-82d87717b3255c51) + +running 3 tests +test entry::attempt_started_tests::a_pre_field_attempt_started_still_reads ... ok +test entry::attempt_started_tests::max_transport_retries_is_always_journaled ... ok +test machine::tests::all_backing_off_steps_return_timers ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 69 filtered out; finished in 0.00s + +exit_code=0 diff --git a/kernel/evidence/501/red-repair-only.txt b/kernel/evidence/501/red-repair-only.txt new file mode 100644 index 000000000..c2774318e --- /dev/null +++ b/kernel/evidence/501/red-repair-only.txt @@ -0,0 +1,64 @@ +# MUTATION (repair only): recovery.rs `if false && *wait_id == park_placeholder_wait_id(..)`; machine.rs park branch intact. Diff applied: +51c51 +< if *wait_id == park_placeholder_wait_id(&spec.id, runtime.attempts) => +--- +> if false && *wait_id == park_placeholder_wait_id(&spec.id, runtime.attempts) => + +$ cd kernel && cargo test -p relayflowd-core --lib -- machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote + Compiling relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.67s + Running unittests src/lib.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/relayflowd_core-82d87717b3255c51) + +running 1 test +test machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote ... FAILED + +failures: + +---- machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote stdout ---- + +thread 'machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote' (2313987) panicked at relayflowd-core/src/machine/tests.rs:843:9: +assertion `left == right` failed: worker-reported: recovery journals exactly the missing wait: [] + left: 0 + right: 1 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + + +failures: + machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote + +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 71 filtered out; finished in 0.00s + +error: test failed, to rerun pass `-p relayflowd-core --lib` +exit_code=101 + +$ cd kernel && cargo test -p relayflowd --test manual_recovery a_park_torn_between_its_two_appends_is_repaired_on_resume + Compiling relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) + Compiling relayflowd-journal v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-journal) + Compiling relayflowd v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd) + Finished `test` profile [unoptimized + debuginfo] target(s) in 1.17s + Running tests/manual_recovery.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/manual_recovery-6793123a1244aad2) + +running 1 test +test a_park_torn_between_its_two_appends_is_repaired_on_resume ... FAILED + +failures: + +---- a_park_torn_between_its_two_appends_is_repaired_on_resume stdout ---- + +thread 'a_park_torn_between_its_two_appends_is_repaired_on_resume' (2314845) panicked at relayflowd/tests/manual_recovery.rs:66:13: +injected crash between the park completion and its wait.human +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + +thread 'a_park_torn_between_its_two_appends_is_repaired_on_resume' (2314845) panicked at relayflowd/tests/manual_recovery.rs:144:5: +assertion `left == right` failed: exactly one wait.human: [] + left: 0 + right: 1 + + +failures: + a_park_torn_between_its_two_appends_is_repaired_on_resume + +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 1 filtered out; finished in 0.01s + +error: test failed, to rerun pass `-p relayflowd --test manual_recovery` +exit_code=101 diff --git a/kernel/evidence/501/red.txt b/kernel/evidence/501/red.txt new file mode 100644 index 000000000..6228765bd --- /dev/null +++ b/kernel/evidence/501/red.txt @@ -0,0 +1,107 @@ +# MUTATION: machine.rs `let manual_park = false && transport_failure ...` (worker-reported park disabled) and recovery.rs `if false && *wait_id == park_placeholder_wait_id(..)` (torn-park repair disabled). Diff applied: +397c397 +< let manual_park = transport_failure +--- +> let manual_park = false && transport_failure +51c51 +< if *wait_id == park_placeholder_wait_id(&spec.id, runtime.attempts) => +--- +> if false && *wait_id == park_placeholder_wait_id(&spec.id, runtime.attempts) => + +$ cd kernel && cargo test -p relayflowd-core --lib -- machine::tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote machine::tests::reset_recovery_still_retries_a_worker_reported_transport_loss + Compiling relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) + Finished `test` profile [unoptimized + debuginfo] target(s) in 1.26s + Running unittests src/lib.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/relayflowd_core-82d87717b3255c51) + +running 3 tests +test machine::tests::reset_recovery_still_retries_a_worker_reported_transport_loss ... ok +test machine::tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching ... FAILED +test machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote ... FAILED + +failures: + +---- machine::tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching stdout ---- + +thread 'machine::tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching' (2310933) panicked at relayflowd-core/src/machine/tests.rs:701:9: +assertion `left == right` failed: Crashed: a park is a completion plus a wait.human and nothing else (no retry timer): [Append(JournalEntry { seq: 0, segment_id: 0, entry_type: StepCompleted, run_id: "run", step_id: Some("agent"), attempt: Some(1), at_ms: 20, payload: Object {"budget": Object {"dollars": String("0"), "tokens_in": Number(0), "tokens_out": Number(0)}, "completed_by": String("worker"), "completionReason": String("crashed"), "disposition": String("retry"), "effects": Array [], "end_pins": Object {"streams": Array [], "workspace": Array [Object {"revision_id": String("rev-dirty"), "surface": String("repo")}]}, "next_attempt_at_ms": Number(20), "output": Null, "spend": Object {"dollars": Number(0), "tokens_input": Number(0), "tokens_output": Number(0), "wallclock_ms": Number(0)}, "trajectory_tail": Object {"transport": Object {"cause": String("signal_close")}}, "verification": Object {"detail": String("direct transport closed by signal"), "gate": String("execution"), "verdict": String("fail")}} }), Append(JournalEntry { seq: 0, segment_id: 0, entry_type: SleepUntil, run_id: "run", step_id: Some("agent"), attempt: Some(1), at_ms: 20, payload: Object {"reason": String("retry_backoff"), "wait_id": String("000000000MNSWFKX4NGBY4K6EJ"), "wake_at_ms": Number(20)} }), ArmTimer { at_ms: 20 }] + left: 3 + right: 2 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + +---- machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote stdout ---- + +thread 'machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote' (2310934) panicked at relayflowd-core/src/machine/tests.rs:830:9: +assertion `left == right` failed: worker-reported: the torn prefix folds to the placeholder + left: Backoff { attempt: 1, wake_at_ms: 20 } + right: NeedsHuman { wait_id: "park-agent-1" } + + +failures: + machine::tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching + machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote + +test result: FAILED. 1 passed; 2 failed; 0 ignored; 0 measured; 69 filtered out; finished in 0.00s + +error: test failed, to rerun pass `-p relayflowd-core --lib` +exit_code=101 + +$ cd kernel && cargo test -p relayflowd --test manual_recovery + Compiling relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) + Compiling relayflowd-journal v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-journal) + Compiling relayflowd v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd) + Finished `test` profile [unoptimized + debuginfo] target(s) in 1.13s + Running tests/manual_recovery.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/manual_recovery-6793123a1244aad2) + +running 2 tests +test a_park_torn_between_its_two_appends_is_repaired_on_resume ... FAILED +test manual_park_of_a_worker_reported_loss_survives_reopen_and_answers_to_a_human ... FAILED + +failures: + +---- a_park_torn_between_its_two_appends_is_repaired_on_resume stdout ---- + +thread 'a_park_torn_between_its_two_appends_is_repaired_on_resume' (2311731) panicked at relayflowd/tests/manual_recovery.rs:265:5: +the injected crash must fire after the park append +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + +---- manual_park_of_a_worker_reported_loss_survives_reopen_and_answers_to_a_human stdout ---- + +thread 'manual_park_of_a_worker_reported_loss_survives_reopen_and_answers_to_a_human' (2311732) panicked at relayflowd/tests/manual_recovery.rs:189:9: +assertion `left == right` failed: Crashed with max_transport_retries=1: a manual step must not be redispatched after a reported loss + left: 2 + right: 1 + + +failures: + a_park_torn_between_its_two_appends_is_repaired_on_resume + manual_park_of_a_worker_reported_loss_survives_reopen_and_answers_to_a_human + +test result: FAILED. 0 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + +error: test failed, to rerun pass `-p relayflowd --test manual_recovery` +exit_code=101 + +$ cd kernel && cargo test -p relayflowd --test crash_resume manual_recovery + Compiling relayflowd v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd) + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.38s + Running tests/crash_resume.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/crash_resume-d2d102ec3e89b705) + +running 1 test +test manual_recovery::manual_recovery_parks_a_worker_reported_loss_across_daemon_restart_until_a_human_answers ... FAILED + +failures: + +---- manual_recovery::manual_recovery_parks_a_worker_reported_loss_across_daemon_restart_until_a_human_answers stdout ---- + +thread 'manual_recovery::manual_recovery_parks_a_worker_reported_loss_across_daemon_restart_until_a_human_answers' (2311978) panicked at relayflowd/tests/crash_resume/manual_recovery.rs:74:5: +crashed with max_transport_retries=1: a parked manual step must not be redispatched +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + + +failures: + manual_recovery::manual_recovery_parks_a_worker_reported_loss_across_daemon_restart_until_a_human_answers + +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 40 filtered out; finished in 0.03s + +error: test failed, to rerun pass `-p relayflowd --test crash_resume` +exit_code=101 diff --git a/kernel/evidence/501/sdk-typecheck-build.txt b/kernel/evidence/501/sdk-typecheck-build.txt new file mode 100644 index 000000000..d972cc25f --- /dev/null +++ b/kernel/evidence/501/sdk-typecheck-build.txt @@ -0,0 +1,19 @@ +# Setup (documented in .github/workflows/publish.yml and scripts/surface-package-gate.sh): the SDK typechecks against the published @relayflows/surface unless the locally built surface is packed and installed with --no-save. +$ cd packages/surface && npm ci --ignore-scripts && ./node_modules/.bin/tsc && cd ../.. && node scripts/pack-release.mjs surface + -> PACK_OK @relayflows/surface@2.0.22 (dist/publish/relayflows-surface-2.0.22.tgz) +$ cd packages/sdk && npm ci && npm install --no-save --package-lock=false --ignore-scripts ../../dist/publish/relayflows-surface-2.0.22.tgz && test ! -L node_modules/@relayflows/surface + +$ cd packages/sdk && npm run typecheck && npm run typecheck:tests && npm run build + +> @relayflows/sdk@2.0.22 typecheck +> tsc --noEmit && tsc -p tsconfig.type-tests.json + + +> @relayflows/sdk@2.0.22 typecheck:tests +> tsc -p tsconfig.tests.json + + +> @relayflows/sdk@2.0.22 build +> tsc && node scripts/make-cli-executable.mjs + +exit_code=0 diff --git a/kernel/relayflowd-core/src/entry.rs b/kernel/relayflowd-core/src/entry.rs index bc08608ce..3eb3399bc 100644 --- a/kernel/relayflowd-core/src/entry.rs +++ b/kernel/relayflowd-core/src/entry.rs @@ -204,14 +204,14 @@ pub struct AttemptStartedPayload { pub recovery_mode: Option, pub pins: Pins, pub max_iterations: u32, - #[serde(default, skip_serializing_if = "is_zero_u32")] + /// Echoed from the spec beside `max_iterations`, and always written: + /// an explicit zero is the value a reader most needs to see (it is why a + /// lost process was NOT retried), so no value is omitted. `default` only + /// reads journals written before the field existed. + #[serde(default)] pub max_transport_retries: u32, } -fn is_zero_u32(value: &u32) -> bool { - *value == 0 -} - /// Runtime pins journaled per attempt (RFC Appendix A rules 2 and 6): the /// revision id of each declared workspace surface and the offset of each /// declared stream. Pins are journal facts, never spec fields — the spec only @@ -335,6 +335,54 @@ mod completion_reason_tests { } } +#[cfg(test)] +mod attempt_started_tests { + use super::{AttemptStartedPayload, Pins, StepType}; + + fn payload(max_transport_retries: u32) -> AttemptStartedPayload { + AttemptStartedPayload { + step_type: StepType::Agent, + idempotency_key: "key".to_owned(), + lease_id: "lease".to_owned(), + lease_deadline_ms: 1, + executor: "unassigned".to_owned(), + recovery_mode: None, + pins: Pins::default(), + max_iterations: 1, + max_transport_retries, + } + } + + /// The transport budget is journaled at every value, zero included: the + /// spec's canonical form may omit its default, the journal never does. + /// An earlier version skipped zero, which erased exactly the value that + /// explains why a lost process was not retried. + #[test] + fn max_transport_retries_is_always_journaled() { + for value in [0, 1, 4] { + let json = serde_json::to_value(payload(value)).unwrap(); + assert_eq!( + json["max_transport_retries"], + serde_json::json!(value), + "the journal must carry the budget verbatim" + ); + let back: AttemptStartedPayload = serde_json::from_value(json).unwrap(); + assert_eq!(back.max_transport_retries, value); + } + } + + /// Journals written before the field existed still fold. + #[test] + fn a_pre_field_attempt_started_still_reads() { + let mut json = serde_json::to_value(payload(1)).unwrap(); + json.as_object_mut() + .unwrap() + .remove("max_transport_retries"); + let back: AttemptStartedPayload = serde_json::from_value(json).unwrap(); + assert_eq!(back.max_transport_retries, 0); + } +} + #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum Disposition { diff --git a/kernel/relayflowd-core/src/lib.rs b/kernel/relayflowd-core/src/lib.rs index a7358a13d..c1cfd68e3 100644 --- a/kernel/relayflowd-core/src/lib.rs +++ b/kernel/relayflowd-core/src/lib.rs @@ -31,7 +31,7 @@ pub use machine::{ pub use memory::{MemoryInjectedPayload, MemoryScope, MemorySpec}; pub use placement::{ExecutionMode, PlacementRequirements, RoutingDecision}; pub use spec::*; -pub use state::{RunState, StateError, StepRuntime, StepState}; +pub use state::{RunState, StateError, StepRuntime, StepState, park_placeholder_wait_id}; pub const JOURNAL_VERSION: u32 = 1; pub const PROTOCOL_VERSION: u32 = 0; diff --git a/kernel/relayflowd-core/src/machine.rs b/kernel/relayflowd-core/src/machine.rs index 533809cf8..c36d37b04 100644 --- a/kernel/relayflowd-core/src/machine.rs +++ b/kernel/relayflowd-core/src/machine.rs @@ -333,11 +333,18 @@ fn start_actions(state: &RunState, step: &StepSpec, attempt: u32, now_ms: i64) - /// completed here ran to a result, so it is the `semantic_executions + 1`-th /// semantic execution; `max_iterations` bounds that count, never the raw /// attempt number — a crashed attempt must not consume iteration allowance. +/// +/// `start_pins` are the attempt's journaled starting pins +/// (`StepRuntime::last_start_pins`). They are read only when a `manual` agent +/// step reports its own transport loss: Appendix A rule 4 parks that step with +/// a diff of the *pinned* revision vs. current state, and the pinned revision +/// is the kernel's record, never the worker's claim. pub fn completion_actions( run_id: &str, step: &StepSpec, attempt: u32, semantic_executions: u32, + start_pins: Option<&Pins>, result: AttemptResult, now_ms: i64, ) -> Vec { @@ -381,6 +388,20 @@ pub fn completion_actions( result.failure_reason, Some(CompletionReason::Crashed | CompletionReason::LeaseExpired) ); + // Appendix A rule 4: a dead attempt under `manual` parks as `needs_human` + // with a diff, whichever way the kernel learned of the death. The + // abandoned-lease path (`abandonment_actions`) always honoured that; a + // worker that reported its own crash through `step.complete` used to be + // redispatched under the transport budget instead, so the same dead + // attempt parked or continued depending on who noticed it first. + let manual_park = transport_failure + && matches!( + step.kind, + StepKind::Agent { + recovery_mode: RecoveryMode::Manual, + .. + } + ); // Preserve `result.output` for successful completions, and for FAILED // deterministic completions specifically — deterministic attempts journal // `{exit_code, stdout_tail, stderr_tail}` so the CLI can render the @@ -395,6 +416,15 @@ pub fn completion_actions( result.output, None, ) + } else if manual_park { + ( + result + .failure_reason + .expect("a transport failure carries its reason"), + Disposition::Park, + Value::Null, + None, + ) } else if (semantic_failure && may_retry_semantic) || (transport_failure && may_retry_transport) { let key = idempotency_key(run_id, &step.id); @@ -450,7 +480,11 @@ pub fn completion_actions( }, ); let mut actions = vec![Action::Append(completed)]; - if let Some(wake_at_ms) = next_attempt_at_ms { + if disposition == Disposition::Park { + actions.push(Action::Append(recovery::manual_park_wait( + run_id, &step.id, attempt, reason, now_ms, start_pins, + ))); + } else if let Some(wake_at_ms) = next_attempt_at_ms { actions.push(Action::Append(JournalEntry::new( EntryType::SleepUntil, run_id, diff --git a/kernel/relayflowd-core/src/machine/parallel_tests.rs b/kernel/relayflowd-core/src/machine/parallel_tests.rs index 7b41c3339..4ea83863c 100644 --- a/kernel/relayflowd-core/src/machine/parallel_tests.rs +++ b/kernel/relayflowd-core/src/machine/parallel_tests.rs @@ -95,7 +95,7 @@ fn agent_success( }], streams: Vec::new(), }); - completion_actions("run", step, 1, 0, result, now_ms) + completion_actions("run", step, 1, 0, None, result, now_ms) .into_iter() .find_map(|action| match action { Action::Append(entry) if entry.entry_type == EntryType::StepCompleted => Some(entry), @@ -142,6 +142,7 @@ fn parallel_lanes_do_not_cross_the_dependency_barrier_early() { step, 1, 0, + None, AttemptResult::successful(json!({"answer": answer}), "worker"), 20, ) @@ -398,6 +399,7 @@ fn failed_run_drains_open_siblings_before_terminal_entry() { &spec.steps[0], 1, 0, + None, failed, 20, ))); @@ -413,6 +415,7 @@ fn failed_run_drains_open_siblings_before_terminal_entry() { &spec.steps[2], 1, 0, + None, AttemptResult::successful(json!({"answer": "a"}), "worker"), 21, ) diff --git a/kernel/relayflowd-core/src/machine/recovery.rs b/kernel/relayflowd-core/src/machine/recovery.rs index 3786d0a24..355bb149c 100644 --- a/kernel/relayflowd-core/src/machine/recovery.rs +++ b/kernel/relayflowd-core/src/machine/recovery.rs @@ -11,7 +11,7 @@ use crate::{ StepCompletedPayload, WaitHumanPayload, }, spec::{RecoveryMode, StepKind}, - state::{RunState, StepState}, + state::{RunState, StepState, park_placeholder_wait_id}, }; pub fn recovery_actions(state: &RunState, now_ms: i64) -> Vec { @@ -36,13 +36,39 @@ pub fn recovery_actions_filtered( let mut actions = Vec::new(); for spec in &state.spec.steps { let runtime = &state.steps[&spec.id]; - let StepState::Running { - attempt, - lease_deadline_ms, - .. - } = runtime.state - else { - continue; + let (attempt, lease_deadline_ms) = match &runtime.state { + StepState::Running { + attempt, + lease_deadline_ms, + .. + } => (*attempt, *lease_deadline_ms), + // A `manual` park is two appends: `step.completed` (`park`), then + // the `wait.human` a human answers. Dying between them leaves the + // step parked on the placeholder id with no wait to answer — a + // permanent park nobody can end. Journal the wait now; it is + // rebuilt from the same journaled facts the first writer used. + StepState::NeedsHuman { wait_id } + if matches!( + spec.kind, + StepKind::Agent { + recovery_mode: RecoveryMode::Manual, + .. + } + ) && *wait_id == park_placeholder_wait_id(&spec.id, runtime.attempts) => + { + actions.push(Action::Append(manual_park_wait( + &state.run_id, + &spec.id, + runtime.attempts, + runtime + .last_completion_reason + .unwrap_or(CompletionReason::Crashed), + now_ms, + runtime.last_start_pins.as_ref(), + ))); + continue; + } + _ => continue, }; if lease_is_active(&spec.id, attempt) { continue; @@ -140,24 +166,13 @@ pub fn abandonment_actions( }, ))]; if manual { - actions.push(Action::Append(JournalEntry::new( - EntryType::WaitHuman, - state.run_id.clone(), - Some(step_id.to_owned()), - Some(attempt), + actions.push(Action::Append(manual_park_wait( + &state.run_id, + step_id, + attempt, + reason, now_ms, - WaitHumanPayload { - wait_id: deterministic_ulid(&state.run_id, step_id, attempt, now_ms, "manual"), - prompt: format!( - "agent step {step_id} of run {} ended {reason:?} with a dirty workspace; \ - a human must inspect it before another attempt", - state.run_id - ), - requested_of: "run-owner".to_owned(), - options: Some(vec!["retry".to_owned(), "cancel".to_owned()]), - timeout_at_ms: None, - diff_ref: dirty_diff_ref(runtime.last_start_pins.as_ref()), - }, + runtime.last_start_pins.as_ref(), ))); } else if let Some(wake_at_ms) = next_attempt_at_ms { actions.push(Action::Append(JournalEntry::new( @@ -176,6 +191,39 @@ pub fn abandonment_actions( actions } +/// The `wait.human` entry that parks a `manual` agent step after a dead +/// attempt. One constructor for both ways the kernel learns of the death — +/// an abandoned lease (`abandonment_actions`) and a worker that reported its +/// own transport loss through `step.complete` (`completion_actions`) — so the +/// two paths cannot drift into parking with different prompts or diffs. +pub(super) fn manual_park_wait( + run_id: &str, + step_id: &str, + attempt: u32, + reason: CompletionReason, + now_ms: i64, + start_pins: Option<&Pins>, +) -> JournalEntry { + JournalEntry::new( + EntryType::WaitHuman, + run_id.to_owned(), + Some(step_id.to_owned()), + Some(attempt), + now_ms, + WaitHumanPayload { + wait_id: deterministic_ulid(run_id, step_id, attempt, now_ms, "manual"), + prompt: format!( + "agent step {step_id} of run {run_id} ended {reason:?} with a dirty workspace; \ + a human must inspect it before another attempt" + ), + requested_of: "run-owner".to_owned(), + options: Some(vec!["retry".to_owned(), "cancel".to_owned()]), + timeout_at_ms: None, + diff_ref: dirty_diff_ref(start_pins), + }, + ) +} + /// Appendix A rule 4: the `manual` park hands the human a diff of the pinned /// revision vs. current state. The reference names each pinned surface and the /// revision the attempt started from — the only revision the kernel knows. diff --git a/kernel/relayflowd-core/src/machine/tests.rs b/kernel/relayflowd-core/src/machine/tests.rs index 61bbfe96a..8db94883c 100644 --- a/kernel/relayflowd-core/src/machine/tests.rs +++ b/kernel/relayflowd-core/src/machine/tests.rs @@ -40,6 +40,7 @@ fn verification_failure_schedules_a_durable_retry() { &spec.steps[0], 1, 0, + None, AttemptResult::successful(json!({"exit_code": 0, "stdout_tail": "wrong"}), "kernel"), 1_000, ); @@ -81,6 +82,7 @@ fn failed_deterministic_completion_preserves_exit_code_and_stderr() { &retryable_step, 1, 0, + None, AttemptResult::successful(retry_output.clone(), "kernel"), 1_000, ); @@ -108,6 +110,7 @@ fn failed_deterministic_completion_preserves_exit_code_and_stderr() { &terminal_step, 1, 0, + None, AttemptResult::successful(terminal_output.clone(), "kernel"), 2_000, ); @@ -150,7 +153,7 @@ fn every_failed_run_terminates_with_declared_completion_reasons() { result.failure_reason = Some(reason); result.failure_detail = Some("declared test failure".to_owned()); let Action::Append(completed) = - completion_actions("run", &step, 1, 0, result, 10).remove(0) + completion_actions("run", &step, 1, 0, None, result, 10).remove(0) else { panic!("failed attempt must append a typed completion"); }; @@ -178,6 +181,7 @@ fn successful_memo_is_never_scheduled_again() { &spec.steps[0], 1, 0, + None, AttemptResult::successful(json!({"exit_code": 0, "stdout_tail": "hello"}), "kernel"), 1_000, ) @@ -319,6 +323,7 @@ fn crashed_attempt_does_not_consume_an_iteration() { &spec.steps[0], 2, state.steps["hello"].semantic_executions, + None, AttemptResult::successful(json!({"exit_code": 0, "stdout_tail": "wrong"}), "kernel"), 2_000, ); @@ -363,13 +368,19 @@ fn all_backing_off_steps_return_timers() { end_pins: None, effects: Vec::new(), trajectory_tail: None, - failure_reason: None, - failure_detail: None, + // A classified transport loss: retryable under the default transport + // budget without consuming a semantic iteration. `worker_error` is + // terminal since transport and semantic budgets were split, so it can + // no longer stand in for "a failure that backs off"; the case has to + // be a reason that still schedules a retry, or the assertion below + // stops exercising failure backoff at all. + failure_reason: Some(CompletionReason::Crashed), + failure_detail: Some("direct transport closed without a status".to_owned()), }; let mut entries = Vec::new(); for step in &spec.steps { entries.extend( - completion_actions("run", step, 1, 0, result.clone(), 1_000) + completion_actions("run", step, 1, 0, None, result.clone(), 1_000) .into_iter() .filter_map(|action| match action { Action::Append(entry) => Some(entry), @@ -478,6 +489,7 @@ fn classified_transport_retry_is_bounded_and_preserves_pin_and_idempotency() { &spec.steps[0], 1, running.steps["agent"].semantic_executions, + None, crashed.clone(), 20, ); @@ -530,6 +542,7 @@ fn classified_transport_retry_is_bounded_and_preserves_pin_and_idempotency() { &spec.steps[0], 2, ready.steps["agent"].semantic_executions, + None, crashed, 40, ); @@ -550,7 +563,7 @@ fn ordinary_worker_error_is_not_retried_by_either_budget() { let mut failed = AttemptResult::successful(Value::Null, "worker"); failed.failure_reason = Some(CompletionReason::WorkerError); failed.failure_detail = Some("CLI rejected the task".to_owned()); - let actions = completion_actions("run", &spec.steps[0], 1, 0, failed, 20); + let actions = completion_actions("run", &spec.steps[0], 1, 0, None, failed, 20); assert_eq!( actions.len(), 1, @@ -588,6 +601,7 @@ fn inspect_recovery_injects_the_dirty_pin_completion_reason_and_tail() { &spec.steps[0], 1, running.steps["agent"].semantic_executions, + None, result, 20, ); @@ -653,6 +667,215 @@ fn manual_recovery_parks_needs_human_and_never_redispatches() { assert!(next_actions(&parked, 30).is_empty()); } +/// Regression (#501 review): `RecoveryMode::Manual` was honoured only when the +/// KERNEL noticed a dead lease (`abandonment_actions`). A worker that reported +/// its own transport loss through `step.complete` went through +/// `completion_actions`, which retried every budget-eligible `crashed` or +/// `lease_expired` without reading the recovery mode — so the same dead +/// attempt parked or redispatched depending on who noticed it first. +/// Appendix A rule 4 makes `manual` a park, whichever path records the death. +#[test] +fn manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching() { + for reason in [CompletionReason::Crashed, CompletionReason::LeaseExpired] { + let spec = agent_spec("manual"); + let started = started_agent(&spec, workspace_pins("rev-clean")); + let running = RunState::fold("run", spec.clone(), std::slice::from_ref(&started)).unwrap(); + let runtime = &running.steps["agent"]; + let mut result = AttemptResult::successful(Value::Null, "worker"); + result.failure_reason = Some(reason); + result.failure_detail = Some("direct transport closed by signal".to_owned()); + result.trajectory_tail = Some(json!({"transport": {"cause": "signal_close"}})); + // The worker's own claim about where the workspace ended up. The diff + // the human is handed must be anchored on the journaled START pin, not + // on this. + result.end_pins = Some(workspace_pins("rev-dirty")); + let actions = completion_actions( + "run", + &spec.steps[0], + 1, + runtime.semantic_executions, + runtime.last_start_pins.as_ref(), + result, + 20, + ); + assert_eq!( + actions.len(), + 2, + "{reason:?}: a park is a completion plus a wait.human and nothing else \ + (no retry timer): {actions:?}" + ); + let completed: StepCompletedPayload = + serde_json::from_value(actions[0].clone().into_append().payload).unwrap(); + assert_eq!(completed.completion_reason, reason); + assert_eq!(completed.disposition, Disposition::Park); + assert_eq!(completed.next_attempt_at_ms, None); + // The worker's evidence still travels with the completion; parking is + // not a reason to discard the account of what went wrong. + assert_eq!( + completed.trajectory_tail, + Some(json!({"transport": {"cause": "signal_close"}})) + ); + assert_eq!( + completed.verification.as_ref().map(|record| record.verdict), + Some(crate::VerificationVerdict::Fail) + ); + let wait = actions[1].clone().into_append(); + assert_eq!(wait.entry_type, EntryType::WaitHuman); + assert_eq!(wait.step_id.as_deref(), Some("agent")); + assert_eq!(wait.attempt, Some(1)); + let human: crate::entry::WaitHumanPayload = + serde_json::from_value(wait.payload.clone()).unwrap(); + assert_eq!(human.diff_ref.as_deref(), Some("repo@rev-clean..current")); + assert_eq!( + human.options, + Some(vec!["retry".to_owned(), "cancel".to_owned()]) + ); + assert!( + human.prompt.contains("agent") && human.prompt.contains("run"), + "the prompt must name the step and run it parked: {}", + human.prompt + ); + + let mut entries = vec![started]; + entries.extend(actions.into_iter().map(Action::into_append)); + let parked = RunState::fold("run", spec, &entries).unwrap(); + assert!( + matches!(parked.steps["agent"].state, StepState::NeedsHuman { .. }), + "{reason:?}: the step must be parked on a human, got {:?}", + parked.steps["agent"].state + ); + assert_eq!( + parked.steps["agent"].semantic_executions, 0, + "a transport loss must not consume a semantic iteration" + ); + assert!( + next_actions(&parked, 30).is_empty(), + "{reason:?}: a parked manual step must never be redispatched" + ); + } +} + +/// The park above is specific to `manual`. The same worker-reported crash +/// under `reset` still takes the bounded transport retry, so the two modes +/// stay distinguishable at the completion. +#[test] +fn reset_recovery_still_retries_a_worker_reported_transport_loss() { + let spec = agent_spec("reset"); + let started = started_agent(&spec, workspace_pins("rev-clean")); + let running = RunState::fold("run", spec.clone(), std::slice::from_ref(&started)).unwrap(); + let runtime = &running.steps["agent"]; + let mut result = AttemptResult::successful(Value::Null, "worker"); + result.failure_reason = Some(CompletionReason::Crashed); + result.failure_detail = Some("direct transport closed by signal".to_owned()); + let actions = completion_actions( + "run", + &spec.steps[0], + 1, + runtime.semantic_executions, + runtime.last_start_pins.as_ref(), + result, + 20, + ); + let completed: StepCompletedPayload = + serde_json::from_value(actions[0].clone().into_append().payload).unwrap(); + assert_eq!(completed.disposition, Disposition::Retry); + assert!( + actions + .iter() + .all(|action| !matches!(action, Action::Append(entry) if entry.entry_type == EntryType::WaitHuman)), + "reset must not park: {actions:?}" + ); +} + +/// A `manual` park is two appends — `step.completed` (`park`), then the +/// `wait.human` a human answers — and each append is its own transaction. A +/// process death between them used to leave the step folded to the placeholder +/// wait id with nothing answerable: a permanent park. Recovery must journal +/// the missing wait exactly once, for BOTH producers of a park. +#[test] +fn recovery_journals_the_wait_human_a_torn_manual_park_never_wrote() { + let spec = agent_spec("manual"); + let started = started_agent(&spec, workspace_pins("rev-clean")); + let running = RunState::fold("run", spec.clone(), std::slice::from_ref(&started)).unwrap(); + let runtime = &running.steps["agent"]; + + let mut reported = AttemptResult::successful(Value::Null, "worker"); + reported.failure_reason = Some(CompletionReason::LeaseExpired); + reported.failure_detail = Some("direct transport closed by signal".to_owned()); + let worker_reported = completion_actions( + "run", + &spec.steps[0], + 1, + runtime.semantic_executions, + runtime.last_start_pins.as_ref(), + reported, + 20, + ) + .remove(0) + .into_append(); + let abandoned = abandonment_actions(&running, "agent", 1, CompletionReason::Crashed, 20) + .remove(0) + .into_append(); + + for (producer, park, reason) in [ + ( + "worker-reported", + worker_reported, + CompletionReason::LeaseExpired, + ), + ("abandoned lease", abandoned, CompletionReason::Crashed), + ] { + // Only the park landed; the wait.human did not. + let torn = RunState::fold("run", spec.clone(), &[started.clone(), park.clone()]).unwrap(); + assert_eq!( + torn.steps["agent"].state, + StepState::NeedsHuman { + wait_id: crate::park_placeholder_wait_id("agent", 1) + }, + "{producer}: the torn prefix folds to the placeholder" + ); + assert!( + next_actions(&torn, 30).is_empty(), + "{producer}: a torn park must not dispatch" + ); + + let repaired = recovery_actions(&torn, 30); + assert_eq!( + repaired.len(), + 1, + "{producer}: recovery journals exactly the missing wait: {repaired:?}" + ); + let wait = repaired[0].clone().into_append(); + assert_eq!(wait.entry_type, EntryType::WaitHuman); + assert_eq!(wait.step_id.as_deref(), Some("agent")); + assert_eq!(wait.attempt, Some(1)); + let human: crate::entry::WaitHumanPayload = + serde_json::from_value(wait.payload.clone()).unwrap(); + assert_eq!(human.diff_ref.as_deref(), Some("repo@rev-clean..current")); + assert!( + human.prompt.contains(&format!("{reason:?}")), + "{producer}: the rebuilt prompt names the journaled reason: {}", + human.prompt + ); + + // Healed: the real wait id replaces the placeholder, and recovery has + // nothing further to add — the repair is idempotent across resumes. + let healed = RunState::fold("run", spec.clone(), &[started.clone(), park, wait]).unwrap(); + assert_eq!( + healed.steps["agent"].state, + StepState::NeedsHuman { + wait_id: human.wait_id + }, + "{producer}: the journaled wait names the parked step" + ); + assert!( + recovery_actions(&healed, 40).is_empty(), + "{producer}: a healed park must not be repaired twice" + ); + assert!(next_actions(&healed, 40).is_empty()); + } +} + trait AppendAction { fn into_append(self) -> JournalEntry; } @@ -703,7 +926,7 @@ fn worker_reported_failure_without_detail_still_records_a_verification() { // detail with it. failure_detail: None, }; - let entries: Vec<_> = completion_actions("run", &spec.steps[0], 1, 0, result, 1_000) + let entries: Vec<_> = completion_actions("run", &spec.steps[0], 1, 0, None, result, 1_000) .into_iter() .filter_map(|action| match action { Action::Append(entry) => Some(entry), diff --git a/kernel/relayflowd-core/src/state.rs b/kernel/relayflowd-core/src/state.rs index 0575b8cb4..af1061e60 100644 --- a/kernel/relayflowd-core/src/state.rs +++ b/kernel/relayflowd-core/src/state.rs @@ -83,6 +83,15 @@ pub struct RunState { pub routing: BTreeMap, } +/// The wait id a parked step carries between its `step.completed` with +/// `disposition: park` and the `wait.human` that names the wait a human can +/// answer. The two are separate journal appends, so a process death between +/// them leaves the step folded to this placeholder with nothing answerable; +/// recovery recognises it and journals the missing wait (`recovery_actions`). +pub fn park_placeholder_wait_id(step_id: &str, attempt: u32) -> String { + format!("park-{step_id}-{attempt}") +} + impl RunState { pub fn fold( run_id: impl Into, @@ -325,7 +334,7 @@ impl RunState { wake_at_ms: payload.next_attempt_at_ms.unwrap_or(entry.at_ms), }, Disposition::Park => StepState::NeedsHuman { - wait_id: format!("park-{step_id}-{attempt}"), + wait_id: park_placeholder_wait_id(&step_id, attempt), }, }; if payload.disposition == Disposition::StepDone diff --git a/kernel/relayflowd-core/src/state/tests.rs b/kernel/relayflowd-core/src/state/tests.rs index 9d7fc53c1..6b260c49c 100644 --- a/kernel/relayflowd-core/src/state/tests.rs +++ b/kernel/relayflowd-core/src/state/tests.rs @@ -129,7 +129,7 @@ fn end_pin_chain_is_enforced_and_a_broken_chain_is_a_hard_error() { let mut result = crate::AttemptResult::successful(json!({"edited": true}), "worker"); result.end_pins = Some(end_pin.clone()); let crate::Action::Append(first_completed) = - crate::completion_actions("run", &spec.steps[0], 1, 0, result, 2).remove(0) + crate::completion_actions("run", &spec.steps[0], 1, 0, None, result, 2).remove(0) else { panic!("first agent completion") }; @@ -214,7 +214,7 @@ fn a_completion_that_omits_a_surface_does_not_drop_it_from_the_pin_chain() { let mut result = crate::AttemptResult::successful(json!({"done": true}), "worker"); result.end_pins = Some(end_pins); let crate::Action::Append(entry) = - crate::completion_actions("run", &spec.steps[index], 1, 0, result, 2).remove(0) + crate::completion_actions("run", &spec.steps[index], 1, 0, None, result, 2).remove(0) else { panic!("the agent step completes") }; diff --git a/kernel/relayflowd-core/tests/memoization.rs b/kernel/relayflowd-core/tests/memoization.rs index b1cfda4e5..338a32461 100644 --- a/kernel/relayflowd-core/tests/memoization.rs +++ b/kernel/relayflowd-core/tests/memoization.rs @@ -50,6 +50,7 @@ fn source(state: &RunState) -> relayflowd_core::JournalEntry { step, 1, 0, + None, AttemptResult::successful(json!("answer"), "worker"), 1, ) diff --git a/kernel/relayflowd/src/engine/drive.rs b/kernel/relayflowd/src/engine/drive.rs index 2005b9e05..9f782d532 100644 --- a/kernel/relayflowd/src/engine/drive.rs +++ b/kernel/relayflowd/src/engine/drive.rs @@ -158,12 +158,13 @@ impl Engine { } } } - let semantic_executions = state.steps[&step.id].semantic_executions; + let runtime = &state.steps[&step.id]; for action in completion_actions( journal.run_id(), &step, attempt, - semantic_executions, + runtime.semantic_executions, + runtime.last_start_pins.as_ref(), result, self.clock.now_ms(), ) { @@ -381,11 +382,13 @@ impl Engine { result.failure_detail = Some(format!( "attempt was not dispatched: the attached worker does not hold its starting pins ({detail})" )); + let runtime = &state.steps[&step.id]; for action in completion_actions( journal.run_id(), step, attempt, - state.steps[&step.id].semantic_executions, + runtime.semantic_executions, + runtime.last_start_pins.as_ref(), result, self.clock.now_ms(), ) { diff --git a/kernel/relayflowd/src/engine/input.rs b/kernel/relayflowd/src/engine/input.rs index d64603fed..b46a92642 100644 --- a/kernel/relayflowd/src/engine/input.rs +++ b/kernel/relayflowd/src/engine/input.rs @@ -50,11 +50,13 @@ impl Engine { "input {name:?}: source {:?} has no successful output value at {:?}", binding.step, binding.path )); + let runtime = &state.steps[&step.id]; for action in completion_actions( journal.run_id(), step, attempt, - state.steps[&step.id].semantic_executions, + runtime.semantic_executions, + runtime.last_start_pins.as_ref(), failure, self.clock.now_ms(), ) { diff --git a/kernel/relayflowd/src/engine/memory.rs b/kernel/relayflowd/src/engine/memory.rs index 460e453a7..2f1de3fb1 100644 --- a/kernel/relayflowd/src/engine/memory.rs +++ b/kernel/relayflowd/src/engine/memory.rs @@ -66,11 +66,13 @@ impl Engine { let mut result = AttemptResult::successful(serde_json::Value::Null, "kernel"); result.failure_reason = Some(reason); result.failure_detail = Some(detail); + let runtime = &state.steps[&step.id]; for action in completion_actions( journal.run_id(), step, attempt, - state.steps[&step.id].semantic_executions, + runtime.semantic_executions, + runtime.last_start_pins.as_ref(), result, self.clock.now_ms(), ) { diff --git a/kernel/relayflowd/src/engine/remote.rs b/kernel/relayflowd/src/engine/remote.rs index 1d98763c6..8870c5a01 100644 --- a/kernel/relayflowd/src/engine/remote.rs +++ b/kernel/relayflowd/src/engine/remote.rs @@ -244,6 +244,7 @@ impl Engine { &step, attempt, runtime.semantic_executions, + runtime.last_start_pins.as_ref(), result, self.clock.now_ms(), ) { diff --git a/kernel/relayflowd/src/exec_det.rs b/kernel/relayflowd/src/exec_det.rs index 3a6ef16b6..a6d447486 100644 --- a/kernel/relayflowd/src/exec_det.rs +++ b/kernel/relayflowd/src/exec_det.rs @@ -205,7 +205,7 @@ mod tests { "command": "printf 'shakedown intentional failure' >&2; exit 7" })) .unwrap(); - let actions = completion_actions("run", &step, 1, 0, execute(&step), 0); + let actions = completion_actions("run", &step, 1, 0, None, execute(&step), 0); let Action::Append(completed) = &actions[0] else { panic!("expected completion") }; diff --git a/kernel/relayflowd/tests/budget_gate.rs b/kernel/relayflowd/tests/budget_gate.rs index 8e3ce33a6..6dd111825 100644 --- a/kernel/relayflowd/tests/budget_gate.rs +++ b/kernel/relayflowd/tests/budget_gate.rs @@ -155,7 +155,7 @@ fn daily_windows_reset_and_exact_limits_do_not_refuse() { dollars_unmetered: false, }; entries.extend( - completion_actions("run", &spec.steps[0], 1, 0, result, 1) + completion_actions("run", &spec.steps[0], 1, 0, None, result, 1) .into_iter() .filter_map(|a| { if let Action::Append(e) = a { diff --git a/kernel/relayflowd/tests/crash_resume.rs b/kernel/relayflowd/tests/crash_resume.rs index 8b51e80a4..f2234a7e1 100644 --- a/kernel/relayflowd/tests/crash_resume.rs +++ b/kernel/relayflowd/tests/crash_resume.rs @@ -13,6 +13,8 @@ mod concurrency; mod llm; #[path = "crash_resume/llm_support.rs"] mod llm_support; +#[path = "crash_resume/manual_recovery.rs"] +mod manual_recovery; #[path = "crash_resume/memory.rs"] mod memory; #[path = "crash_resume/parallel_lifecycle.rs"] diff --git a/kernel/relayflowd/tests/crash_resume/manual_recovery.rs b/kernel/relayflowd/tests/crash_resume/manual_recovery.rs new file mode 100644 index 000000000..f9dc11ab7 --- /dev/null +++ b/kernel/relayflowd/tests/crash_resume/manual_recovery.rs @@ -0,0 +1,192 @@ +//! Appendix A rule 4 against the real daemon: a `manual` agent step whose +//! attached WORKER reports its own transport loss through `step.complete` +//! parks as `needs_human`, is not redispatched — before or after the daemon +//! itself is killed and resumed — and is redispatched by a human answer on the +//! pinned revision. The kernel-noticed death (`abandonment_actions`) already +//! parked; this pins the worker-reported one to the same behaviour. + +use std::time::Duration; + +use relayflowd_core::{CompletionReason, Disposition, EntryType, StepCompletedPayload}; +use serde_json::{Value, json}; + +use super::{ + llm_support::{LlmFixture, ProtocolClient, ServerGuard, spawn_resume}, + support::journal_entries, +}; + +fn manual_spec(max_transport_retries: u32) -> Value { + json!({"steps": [{ + "id": "edit", + "type": "agent", + "instruction": "edit", + "recovery_mode": "manual", + "max_iterations": 2, + "retry": { + "initial_backoff_ms": 0, + "max_backoff_ms": 0, + "multiplier": 1, + "jitter_percent": 0, + "max_transport_retries": max_transport_retries + }, + "surfaces": {"workspace": [{"surface": "repo"}]} + }]}) +} + +fn attached_agent(fixture: &LlmFixture, id: &str) -> ProtocolClient { + let mut worker = ProtocolClient::connect(&fixture.socket()); + worker + .request( + "worker.attach", + json!({ + "worker_id": id, + "step_types": ["agent"], + "pins": {"workspace": [{"surface": "repo", "revision_id": "rev-0"}]} + }), + ) + .unwrap(); + worker +} + +/// The shape the SDK worker sends when its direct transport is lost. +fn report_transport_loss(worker: &mut ProtocolClient, dispatch: &Value, reason: &str) -> Value { + worker + .request( + "step.complete", + json!({ + "run_id": dispatch["run_id"], + "step_id": dispatch["step_id"], + "attempt": dispatch["attempt"], + "idempotency_key": dispatch["idempotency_key"], + "completionReason": reason, + "output": {"exit_code": null, "stdout_tail": "", "stderr_tail": "killed"}, + "started_pins": dispatch["pins"], + "trajectory_tail": {"transport": {"cause": "signal_close"}} + }), + ) + .unwrap() +} + +/// Probe for silence: no `step.dispatch` reaches this worker within the +/// window (same shape as `parallel_lifecycle.rs`). +fn assert_no_dispatch(worker: &mut ProtocolClient, what: &str) { + worker.override_read_timeout(Some(Duration::from_millis(200))); + assert!( + worker.event("step.dispatch").is_err(), + "{what}: a parked manual step must not be redispatched" + ); + worker.override_read_timeout(None); +} + +fn run_snapshot(fixture: &LlmFixture, run_id: &Value) -> Value { + ProtocolClient::connect(&fixture.socket()) + .request("run.get", json!({"run_id": run_id})) + .unwrap() +} + +#[test] +fn manual_recovery_parks_a_worker_reported_loss_across_daemon_restart_until_a_human_answers() { + for (reason, max_transport_retries, expected) in [ + ("crashed", 1, CompletionReason::Crashed), + ("lease_expired", 1, CompletionReason::LeaseExpired), + // No transport budget: still a park, never a terminal `step_done`. + ("crashed", 0, CompletionReason::Crashed), + ] { + let case = format!("{reason} with max_transport_retries={max_transport_retries}"); + let fixture = LlmFixture::parallel(&format!("manual-{reason}-{max_transport_retries}")); + let mut server = ServerGuard::start(&fixture); + let mut worker = attached_agent(&fixture, "manual-agent"); + let started = ProtocolClient::connect(&fixture.socket()) + .request( + "run.start", + json!({"spec": manual_spec(max_transport_retries)}), + ) + .unwrap(); + let run_id = started["run_id"].clone(); + let first = worker.event("step.dispatch").unwrap(); + assert_eq!(first["attempt"], 1, "{case}"); + assert_eq!( + first["pins"]["workspace"][0]["revision_id"], "rev-0", + "{case}" + ); + + let parked = report_transport_loss(&mut worker, &first, reason); + assert_eq!(parked["status"], "parked", "{case}: {parked}"); + assert_no_dispatch(&mut worker, &case); + + let entries = journal_entries(&fixture.data_dir).unwrap(); + let starts = entries + .iter() + .filter(|entry| entry.entry_type == EntryType::StepAttemptStarted) + .count(); + assert_eq!(starts, 1, "{case}: exactly one attempt was started"); + let completed = entries + .iter() + .find(|entry| entry.entry_type == EntryType::StepCompleted) + .unwrap_or_else(|| panic!("{case}: the reported loss is journaled")); + let payload: StepCompletedPayload = + serde_json::from_value(completed.payload.clone()).unwrap(); + assert_eq!(payload.completion_reason, expected, "{case}"); + assert_eq!(payload.disposition, Disposition::Park, "{case}"); + let wait = entries + .iter() + .find(|entry| entry.entry_type == EntryType::WaitHuman) + .unwrap_or_else(|| panic!("{case}: the park journals a wait.human")); + assert_eq!(wait.step_id.as_deref(), Some("edit"), "{case}"); + assert_eq!(wait.attempt, Some(1), "{case}"); + assert_eq!(wait.payload["diff_ref"], "repo@rev-0..current", "{case}"); + let wait_id = wait.payload["wait_id"].as_str().unwrap().to_owned(); + let snapshot = run_snapshot(&fixture, &run_id); + assert_eq!(snapshot["status"], "parked", "{case}"); + assert_eq!(snapshot["steps"]["edit"]["state"], "needs_human", "{case}"); + let journaled = entries.len(); + + // Kill the daemon with the run parked, restart, resume: the park must + // hold — no redispatch to the replacement worker and no new entries. + server.kill(); + drop(worker); + let _restarted = ServerGuard::start(&fixture); + let mut replacement = attached_agent(&fixture, "manual-agent-after-restart"); + let resume = spawn_resume(&fixture, run_id.as_str().unwrap()) + .wait_with_output() + .unwrap(); + assert!(resume.status.success(), "{case}: resume failed: {resume:?}"); + let outcome: Value = serde_json::from_slice(&resume.stdout).unwrap(); + assert_eq!(outcome["status"], "parked", "{case}: {outcome}"); + assert_no_dispatch(&mut replacement, &format!("{case} after restart")); + assert_eq!( + journal_entries(&fixture.data_dir).unwrap().len(), + journaled, + "{case}: resume must not append to a cleanly parked run" + ); + assert_eq!( + run_snapshot(&fixture, &run_id)["steps"]["edit"]["state"], + "needs_human", + "{case}" + ); + + // The human's answer, and only that, ends the park; the replacement + // attempt starts on the pinned revision under the same effect key. + let answered = ProtocolClient::connect(&fixture.socket()) + .request( + "event.emit", + json!({ + "run_id": run_id, + "event_key": wait_id, + "payload": {"answer": "retry", "answeredBy": "khaliq"} + }), + ) + .unwrap(); + assert_eq!(answered["matched"], 1, "{case}: {answered}"); + let second = replacement.event("step.dispatch").unwrap(); + assert_eq!(second["attempt"], 2, "{case}"); + assert_eq!( + second["pins"]["workspace"][0]["revision_id"], "rev-0", + "{case}" + ); + assert_eq!( + second["idempotency_key"], first["idempotency_key"], + "{case}" + ); + } +} diff --git a/kernel/relayflowd/tests/manual_recovery.rs b/kernel/relayflowd/tests/manual_recovery.rs new file mode 100644 index 000000000..7444fba1e --- /dev/null +++ b/kernel/relayflowd/tests/manual_recovery.rs @@ -0,0 +1,305 @@ +//! RFC-0001 Appendix A rule 4 at the engine boundary, for the death the +//! WORKER reports: a `manual` agent step whose worker completes with +//! `crashed` or `lease_expired` parks as `needs_human` instead of taking the +//! transport retry, stays parked across a reopen + resume, and is redispatched +//! only by a human answer. The second test tears the park between its two +//! journal appends and proves resume repairs it. + +use std::{ + panic::{AssertUnwindSafe, catch_unwind}, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, Ordering}, + }, +}; + +use relayflowd::{ + Engine, OutOfBandCompletion, RunStatus, + worker::{DispatchOutcome, JournalObserver, StepDispatch, StepDispatcher}, +}; +use relayflowd_core::{ + Budget, CompletionReason, Disposition, EntryType, JournalEntry, Pins, RunSpec, + StepCompletedPayload, StepSpec, StepType, WaitHumanPayload, WorkspacePin, +}; +use serde_json::json; + +fn pinned(revision_id: &str) -> Pins { + Pins { + workspace: vec![WorkspacePin { + surface: "repo".to_owned(), + revision_id: revision_id.to_owned(), + }], + streams: vec![], + } +} + +/// In-process agent worker: records every dispatch, pins `repo@rev-0` at +/// start, and can die exactly once right after the park completion is +/// durable — between the two appends a `manual` park is made of. +#[derive(Default)] +struct AgentWorker { + dispatches: Mutex>, + crash_after_park: bool, + crashed: AtomicBool, +} + +impl AgentWorker { + fn crashing_after_park() -> Self { + Self { + crash_after_park: true, + ..Self::default() + } + } + + fn dispatches(&self) -> Vec { + self.dispatches.lock().unwrap().clone() + } +} + +impl JournalObserver for AgentWorker { + fn appended(&self, entry: &JournalEntry) { + if self.crash_after_park + && entry.entry_type == EntryType::StepCompleted + && entry.payload["disposition"] == "park" + && !self.crashed.swap(true, Ordering::SeqCst) + { + panic!("injected crash between the park completion and its wait.human"); + } + } +} + +impl StepDispatcher for AgentWorker { + fn executor(&self, _: StepType) -> Option { + Some("manual-recovery-test".to_owned()) + } + + fn available(&self, step_type: StepType) -> bool { + step_type == StepType::Agent + } + + fn starting_pins(&self, _: &StepSpec) -> anyhow::Result { + Ok(pinned("rev-0")) + } + + fn dispatch(&self, dispatch: StepDispatch) -> anyhow::Result { + self.dispatches.lock().unwrap().push(dispatch); + Ok(DispatchOutcome::Dispatched) + } +} + +fn manual_spec(max_transport_retries: u32) -> RunSpec { + RunSpec::parse(&json!({ + "name": "manual-recovery", + "steps": [{ + "id": "edit", + "type": "agent", + "instruction": "edit the workspace", + "recovery_mode": "manual", + "max_iterations": 2, + "retry": { + "initial_backoff_ms": 0, + "max_backoff_ms": 0, + "multiplier": 1, + "jitter_percent": 0, + "max_transport_retries": max_transport_retries + }, + "surfaces": {"workspace": [{"surface": "repo"}]} + }] + })) + .unwrap() +} + +/// What the SDK worker sends when its direct transport is lost: the failure +/// reason, the pins it was dispatched with, and bounded transport evidence. +fn transport_loss(dispatch: &StepDispatch, reason: CompletionReason) -> OutOfBandCompletion { + OutOfBandCompletion { + human_intervention: false, + attempt: dispatch.attempt, + idempotency_key: dispatch.idempotency_key.clone(), + completion_reason: reason, + output: json!({"exit_code": null, "stdout_tail": "", "stderr_tail": "killed"}), + budget: Budget::default(), + completed_by: "manual-recovery-test".to_owned(), + started_pins: Some(dispatch.pins.clone()), + end_pins: None, + effects: vec![], + trajectory_tail: Some(json!({"transport": {"cause": "signal_close"}})), + } +} + +fn entries(engine: &Engine, run_id: &str) -> Vec { + engine.journal_entries(run_id, 1, usize::MAX).unwrap() +} + +fn of_type(entries: &[JournalEntry], entry_type: EntryType) -> Vec<&JournalEntry> { + entries + .iter() + .filter(|entry| entry.entry_type == entry_type) + .collect() +} + +fn parked_wait(engine: &Engine, run_id: &str) -> WaitHumanPayload { + let all = entries(engine, run_id); + let waits = of_type(&all, EntryType::WaitHuman); + assert_eq!(waits.len(), 1, "exactly one wait.human: {waits:?}"); + assert_eq!(waits[0].step_id.as_deref(), Some("edit")); + assert_eq!(waits[0].attempt, Some(1)); + let wait: WaitHumanPayload = serde_json::from_value(waits[0].payload.clone()).unwrap(); + // Rule 4: the diff is anchored on the journaled START pin. + assert_eq!(wait.diff_ref.as_deref(), Some("repo@rev-0..current")); + wait +} + +fn answer_retry(engine: &Engine, run_id: &str, wait_id: &str) { + let matched = engine + .emit_event( + run_id, + wait_id, + json!({"answer": "retry", "answeredBy": "khaliq"}), + ) + .unwrap(); + assert_eq!(matched, 1, "the human answer must close the park's wait"); +} + +#[test] +fn manual_park_of_a_worker_reported_loss_survives_reopen_and_answers_to_a_human() { + for (reason, max_transport_retries) in [ + (CompletionReason::Crashed, 1), + (CompletionReason::LeaseExpired, 1), + // No transport budget at all: `manual` still parks rather than + // ending the step terminally, exactly as an abandoned lease does. + (CompletionReason::Crashed, 0), + ] { + let case = format!("{reason:?} with max_transport_retries={max_transport_retries}"); + let dir = tempfile::tempdir().unwrap(); + let worker = Arc::new(AgentWorker::default()); + let engine = Engine::with_runtime(dir.path(), worker.clone(), worker.clone()); + let started = engine + .start(manual_spec(max_transport_retries), "test", None) + .unwrap(); + let run_id = started.run_id.clone(); + let first = worker.dispatches().remove(0); + assert_eq!(first.attempt, 1); + assert_eq!(first.pins, pinned("rev-0")); + + let outcome = engine + .complete_out_of_band(&run_id, "edit", transport_loss(&first, reason)) + .unwrap(); + assert_eq!(outcome.status, RunStatus::Parked, "{case}"); + assert_eq!( + worker.dispatches().len(), + 1, + "{case}: a manual step must not be redispatched after a reported loss" + ); + + let all = entries(&engine, &run_id); + assert_eq!( + of_type(&all, EntryType::StepAttemptStarted).len(), + 1, + "{case}" + ); + let completions = of_type(&all, EntryType::StepCompleted); + assert_eq!(completions.len(), 1, "{case}: {completions:?}"); + let completed: StepCompletedPayload = + serde_json::from_value(completions[0].payload.clone()).unwrap(); + assert_eq!(completed.completion_reason, reason, "{case}"); + assert_eq!(completed.disposition, Disposition::Park, "{case}"); + assert_eq!( + completed.trajectory_tail, + Some(json!({"transport": {"cause": "signal_close"}})), + "{case}: the worker's transport evidence rides the park" + ); + let wait = parked_wait(&engine, &run_id); + assert_eq!(engine.snapshot(&run_id).unwrap().status, RunStatus::Parked); + + // A fresh engine over the same data dir is a restarted daemon. Resume + // must leave the park alone: no redispatch before a human resolves it. + drop(engine); + let reopened = Engine::with_runtime(dir.path(), worker.clone(), worker.clone()); + let resumed = reopened.resume(&run_id, None).unwrap(); + assert_eq!(resumed.status, RunStatus::Parked, "{case}"); + assert_eq!( + worker.dispatches().len(), + 1, + "{case}: resume redispatched a parked step" + ); + assert_eq!( + entries(&reopened, &run_id).len(), + all.len(), + "{case}: resume must not append to a cleanly parked run" + ); + + // Only the human's answer ends the park, and the replacement attempt + // starts on the pinned revision (Appendix A rule 4, `manual`). + answer_retry(&reopened, &run_id, &wait.wait_id); + let dispatches = worker.dispatches(); + assert_eq!(dispatches.len(), 2, "{case}: the answer must redispatch"); + assert_eq!(dispatches[1].attempt, 2, "{case}"); + assert_eq!(dispatches[1].pins, pinned("rev-0"), "{case}"); + assert_eq!( + dispatches[1].idempotency_key, first.idempotency_key, + "{case}: the replacement keeps the idempotency key" + ); + } +} + +/// A park is two appends and each append is its own transaction. Die after +/// the first and the run is parked on a placeholder nobody can answer — unless +/// resume notices and journals the wait it never wrote. +#[test] +fn a_park_torn_between_its_two_appends_is_repaired_on_resume() { + let dir = tempfile::tempdir().unwrap(); + let worker = Arc::new(AgentWorker::crashing_after_park()); + let engine = Engine::with_runtime(dir.path(), worker.clone(), worker.clone()); + let started = engine.start(manual_spec(1), "test", None).unwrap(); + let run_id = started.run_id.clone(); + let first = worker.dispatches().remove(0); + + let died = catch_unwind(AssertUnwindSafe(|| { + engine.complete_out_of_band( + &run_id, + "edit", + transport_loss(&first, CompletionReason::Crashed), + ) + })); + assert!( + died.is_err(), + "the injected crash must fire after the park append" + ); + drop(engine); + + let torn = Engine::new(dir.path()); + let prefix = entries(&torn, &run_id); + let completions = of_type(&prefix, EntryType::StepCompleted); + assert_eq!(completions.len(), 1); + assert_eq!(completions[0].payload["disposition"], "park"); + assert!( + of_type(&prefix, EntryType::WaitHuman).is_empty(), + "the crash landed before the wait.human: {prefix:?}" + ); + assert_eq!(torn.snapshot(&run_id).unwrap().status, RunStatus::Parked); + + // Resume repairs the torn park: the wait is journaled once, from the + // journaled start pins, and nothing is dispatched. + let reopened = Engine::with_runtime(dir.path(), worker.clone(), worker.clone()); + assert_eq!( + reopened.resume(&run_id, None).unwrap().status, + RunStatus::Parked + ); + let wait = parked_wait(&reopened, &run_id); + assert_eq!(worker.dispatches().len(), 1, "repair must not redispatch"); + + // Resuming again adds nothing: the repair is idempotent. + assert_eq!( + reopened.resume(&run_id, None).unwrap().status, + RunStatus::Parked + ); + assert_eq!(parked_wait(&reopened, &run_id).wait_id, wait.wait_id); + + // And the repaired wait is a real one: a human can end it. + answer_retry(&reopened, &run_id, &wait.wait_id); + let dispatches = worker.dispatches(); + assert_eq!(dispatches.len(), 2); + assert_eq!(dispatches[1].attempt, 2); + assert_eq!(dispatches[1].pins, pinned("rev-0")); +} From 0fd5cab247f816cf2b6f8be143da9615ba3fcee5 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 20 Sep 2026 03:02:12 -0700 Subject: [PATCH 4/7] =?UTF-8?q?chore(kernel):=20review=20follow-ups=20for?= =?UTF-8?q?=20#501=20=E2=80=94=20split=20recovery=20tests,=20keep=20the=20?= =?UTF-8?q?park=20sentinel=20crate-internal,=20literal=20mutation=20transc?= =?UTF-8?q?ript?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner audit of 2e784eaf asked for three things and a reviewer asked for two more verification captures; nothing in production behaviour changes. - Move the three new manual-recovery unit tests into `machine/recovery_tests.rs`; `machine/tests.rs` keeps the restored `all_backing_off_steps_return_timers` and lends its fixtures as `pub(super)`. No assertion weakened or removed. - `park_placeholder_wait_id` is `pub(crate)` again with no lib.rs re-export; it is a fold sentinel, not kernel API. - Replace the three mutation evidence files with one literal transcript (`mutation-transcript.txt`): pre-mutation sha256, the applied diffs, the red runs, `cp` restore, `sha256sum -c` OK, green runs. The README's "byte-for-byte" claim now points at the command that proves it. - Strip trailing whitespace from captured logs so `git diff --check` passes; README says so. - Add `green-sdk-authored-node-runtime.txt` (standalone suite under isolated Bun 1.4.0 + Node 22.23.2: 14/14) and `codex-live-probe.txt` (one bounded live run of codex-cli 0.154.0 through the direct unattended transport: success; also records the pre-existing `cwd` preflight/run mismatch). Co-Authored-By: Claude Opus 5 (1M context) Session-Id: 144d3b43-0019-4de3-988a-7cd9ba4fc148 --- kernel/evidence/501/README.md | 21 +- kernel/evidence/501/codex-live-probe.txt | 114 +++++++ kernel/evidence/501/green-kernel.txt | 208 ++++++------ .../501/green-sdk-authored-node-runtime.txt | 28 ++ kernel/evidence/501/green-sdk.txt | 2 +- kernel/evidence/501/green-targeted.txt | 54 ---- kernel/evidence/501/mutation-transcript.txt | 305 ++++++++++++++++++ kernel/evidence/501/red-repair-only.txt | 64 ---- kernel/evidence/501/red.txt | 107 ------ kernel/relayflowd-core/src/lib.rs | 2 +- kernel/relayflowd-core/src/machine.rs | 2 + .../src/machine/recovery_tests.rs | 221 +++++++++++++ kernel/relayflowd-core/src/machine/tests.rs | 217 +------------ kernel/relayflowd-core/src/state.rs | 2 +- 14 files changed, 798 insertions(+), 549 deletions(-) create mode 100644 kernel/evidence/501/codex-live-probe.txt create mode 100644 kernel/evidence/501/green-sdk-authored-node-runtime.txt delete mode 100644 kernel/evidence/501/green-targeted.txt create mode 100644 kernel/evidence/501/mutation-transcript.txt delete mode 100644 kernel/evidence/501/red-repair-only.txt delete mode 100644 kernel/evidence/501/red.txt create mode 100644 kernel/relayflowd-core/src/machine/recovery_tests.rs diff --git a/kernel/evidence/501/README.md b/kernel/evidence/501/README.md index 5ff118672..a93dcc7db 100644 --- a/kernel/evidence/501/README.md +++ b/kernel/evidence/501/README.md @@ -38,14 +38,20 @@ first — contradicting RFC-0001 Appendix A rule 4. | file | what | |---|---| -| `red.txt` | both production changes mutated off (`manual_park = false && …`, repair guard `false && …`): 5 regression tests fail with the original symptom — `disposition: retry`, second dispatch | -| `red-repair-only.txt` | repair alone mutated off: torn-park tests fail (0 `wait.human` where 1 expected) | -| `green-targeted.txt` | files restored byte-for-byte (`sha256sum -c` OK), same commands pass | +| `mutation-transcript.txt` | one literal transcript, every command echoed: pre-mutation `sha256sum` of `machine.rs` + `recovery.rs`; mutation A+B applied (`manual_park = false && …`, repair guard `false && …`, shown as `git diff -U0`); RED — 5 regression tests fail with the original symptom (`disposition: retry`, second dispatch); restore via `cp` and `sha256sum -c` → `OK` for both files; mutation B alone (repair off); RED — the two torn-park tests fail (0 `wait.human` where 1 expected); restore + `sha256sum -c` → `OK`; GREEN — same commands pass | | `green-kernel.txt` | `cargo test --workspace`, exit 0 | | `clippy.txt` | `cargo clippy --workspace --all-targets -- -D warnings`: exits 101 on pre-existing findings only (`schema.rs:125`, `spec.rs:77`, `memoization.rs:102` as in the PR body, plus pre-existing test-target findings); `clippy-all-targets-warn.txt` lists every warning location — none on lines this change added | | `sdk-typecheck-build.txt` | surface built + packed + installed `--no-save` into sdk (documented flow), then `npm run typecheck && npm run typecheck:tests && npm run build`, exit 0 | | `green-sdk.txt` | `RELAYFLOWD_BIN= npx vitest run`: 154 files / 2410 tests pass; 2 environmental failures explained below | | `green-sdk-bundle-pristine.txt` | `tests/bundle.test.ts` re-run from a pristine `npm ci`: 23/23 pass, exit 0 | +| `green-sdk-authored-node-runtime.txt` | the standalone suite under an isolated `mise install bun@1.4.0` (global config untouched) + Node 22.23.2 via `mise exec`, `FLOWS_BUILD_BUN` / `FLOWS_AUTHORED_NODE` absolute: 14/14 pass, exit 0 | +| `codex-live-probe.txt` | one bounded live run of the installed `codex-cli 0.154.0` through the direct unattended transport (`flows check` + `flows run --local-agent`, output-only instruction, disposable cwd verified unchanged): `success`, exit 0, verified output, journal facts incl. the transport evidence. First attempt refused by the kernel on `cwd` — pre-existing preflight/run mismatch, see below | + + +Pre-existing defect observed while probing (not fixed here, out of scope): +`flows check` accepts a step-level `cwd:` (compiled into the kernel spec since +#358) but `relayflowd` rejects the spec at `run.start` with +`invalid_spec: unknown field "cwd"` — a Covenant 2 preflight/run mismatch. The two failures in `green-sdk.txt` are environmental, not from this change: @@ -60,10 +66,13 @@ The two failures in `green-sdk.txt` are environmental, not from this change: Regression tests added: -- `relayflowd-core/src/machine/tests.rs`: +- `relayflowd-core/src/machine/recovery_tests.rs` (focused module; owner asked + for it split out of the general `tests.rs`): `manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching` (crashed + lease_expired), `reset_recovery_still_retries_a_worker_reported_transport_loss`, `recovery_journals_the_wait_human_a_torn_manual_park_never_wrote` (both producers) +- `relayflowd-core/src/machine/tests.rs`: `all_backing_off_steps_return_timers` + precondition restored (stays in the general file) - `relayflowd-core/src/entry.rs`: `max_transport_retries_is_always_journaled`, `a_pre_field_attempt_started_still_reads` - `relayflowd/tests/manual_recovery.rs` (in-process engine, mock worker): @@ -78,3 +87,7 @@ Regression tests added: `rustfmt --check` drift is unchanged from the PR head (20 files, none touched by this change beyond formatting the lines it added). + +Captured logs are verbatim except that trailing whitespace on captured lines +was stripped (`sed 's/[ \t]*$//'`) so `git diff --check` passes; no other +byte was edited. diff --git a/kernel/evidence/501/codex-live-probe.txt b/kernel/evidence/501/codex-live-probe.txt new file mode 100644 index 000000000..2ed374ac8 --- /dev/null +++ b/kernel/evidence/501/codex-live-probe.txt @@ -0,0 +1,114 @@ +# Live probe of the installed Codex CLI through the direct unattended transport (stdin-ignored spawn from #501). +$ which codex && codex --version +/home/khaliqgant/.local/share/mise/installs/codex/latest/bin/codex +codex-cli 0.154.0 +$ cat /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/codex-probe/codex-probe.flow.yaml +version: '0.1.0' +name: codex-stdin-probe +description: >- + Bounded live probe of the installed Codex CLI through the direct + unattended transport: one output-only agent step in a disposable cwd. +steps: + - id: say-ok + type: agent + cli: codex + cwd: /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/codex-probe/cwd + maxIterations: 1 + transportRetries: 0 + instruction: >- + Do not create, modify or delete any file and do not run any command. + Reply with exactly one line of text: RELAYFLOWS_CODEX_STDIN_OK + verification: + type: output_contains + value: RELAYFLOWS_CODEX_STDIN_OK + +$ cd packages/sdk && RELAYFLOWD_BIN=/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/relayflowd node dist/cli.js check /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/codex-probe/codex-probe.flow.yaml +WARNING [editor_schema_missing] For editor validation, add this first line: # yaml-language-server: $schema=https://schema.relayflows.dev/v0.1/flows.schema.json +GATE step "say-ok" output_contains from data (kernel, journal-replayable) +RESOLVED step "say-ok" cli "codex" from step +REQUIRES codex (step "say-ok") +CHECK PASSED /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/codex-probe/codex-probe.flow.yaml +exit_code=0 + +$ cd packages/sdk && RELAYFLOWD_BIN=/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/relayflowd timeout 600 node dist/cli.js run --local-agent --no-observer-link --data-dir /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/codex-probe/data /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/codex-probe/codex-probe.flow.yaml +WARNING [editor_schema_missing] For editor validation, add this first line: # yaml-language-server: $schema=https://schema.relayflows.dev/v0.1/flows.schema.json +FAILED [protocol_error] relayflowd could not complete the run request: invalid_spec: unknown field "cwd" at steps[0] — refusing to guess (fail closed) +exit_code=1 + +# Attempt 2: `cwd:` removed from the flow (the kernel spec rejects it although flows check accepts it — pre-existing, see result file); the run is launched FROM the disposable directory so the CLI inherits it as cwd. +$ cat /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/codex-probe/codex-probe.flow.yaml +version: '0.1.0' +name: codex-stdin-probe +description: >- + Bounded live probe of the installed Codex CLI through the direct + unattended transport: one output-only agent step in a disposable cwd. +steps: + - id: say-ok + type: agent + cli: codex + maxIterations: 1 + transportRetries: 0 + instruction: >- + Do not create, modify or delete any file and do not run any command. + Reply with exactly one line of text: RELAYFLOWS_CODEX_STDIN_OK + verification: + type: output_contains + value: RELAYFLOWS_CODEX_STDIN_OK +$ cd /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/codex-probe/cwd && pwd && ls -la +/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/codex-probe/cwd +total 0 +drwxr-xr-x 2 khaliqgant khaliqgant 40 Sep 20 02:59 . +drwxr-xr-x 4 khaliqgant khaliqgant 100 Sep 20 03:00 .. +$ RELAYFLOWD_BIN=/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/relayflowd timeout 600 node /tmp/flows-fleet-501/packages/sdk/dist/cli.js run --local-agent --no-observer-link --data-dir /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/codex-probe/data /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/codex-probe/codex-probe.flow.yaml +WAITING [worker_lease] Run "01M2Z44G4WB8BPC6JSMXT3NGZB" step "say-ok" (agent) is running under a worker lease until 1789898438097. +↻ say-ok (agent) [agent: running] 0.00s +WARNING [editor_schema_missing] For editor validation, add this first line: # yaml-language-server: $schema=https://schema.relayflows.dev/v0.1/flows.schema.json +RUN 01M2Z44G4WB8BPC6JSMXT3NGZB completed (1 step) completionReason: success +exit_code=0 +$ ls -la /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/codex-probe/cwd # must be unchanged +total 0 +drwxr-xr-x 2 khaliqgant khaliqgant 40 Sep 20 02:59 . +drwxr-xr-x 4 khaliqgant khaliqgant 100 Sep 20 03:00 .. + +# Journal facts for the run +$ node /tmp/flows-fleet-501/packages/sdk/dist/cli.js logs --json --data-dir /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/codex-probe/data 01M2Z44G4WB8BPC6JSMXT3NGZB | jq '.[] | select(.entry_type=="step.attempt.started" or .entry_type=="step.completed") | {entry_type, attempt, payload: (.payload | {step_type, executor, max_iterations, max_transport_retries, completionReason, disposition, verification, output: (.output | if type=="object" then {exit_code, stdout_tail} else . end), trajectory_tail: (.trajectory_tail | if type=="object" then {transport} else . end), spend})}' +jq: parse error: Invalid numeric literal at line 1, column 8 +exit_code=5 + +# Journal facts (read directly from /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/codex-probe/data/runs/01M2Z44G4WB8BPC6JSMXT3NGZB.sqlite3 with python3 sqlite3; 'flows logs' takes no --data-dir): +step.attempt.started attempt 1 {"step_type": "agent", "executor": "local-agent-98587efa-bcbb-4c24-8869-26d03ac86eb2", "max_iterations": 1, "max_transport_retries": 0, "recovery_mode": "reset"} +step.completed attempt 1 { + "completionReason": "success", + "disposition": "step_done", + "verification": { + "detail": "all gates passed", + "gate": "output_contains", + "verdict": "pass" + }, + "output": { + "exit_code": 0, + "stdout_tail": "RELAYFLOWS_CODEX_STDIN_OK" + }, + "transport": { + "cause": "exited", + "exit_code": 0, + "phase": "close", + "retryable": false, + "signal": null, + "stderr_tail": "Reading additional input from stdin...\n" + }, + "transcript_model": null, + "spend": { + "dollars": 0, + "dollars_unmetered": true, + "tokens_input": 13963, + "tokens_output": 14, + "wallclock_ms": 3282 + } +} + +# Model: no model declared on the step and the codex adapter has no default (undefined below), so `codex exec` ran with NO --model flag: the model is whatever the installed Codex CLI's own configuration selects. The kernel journal does not record it for the direct transport. +$ node -e "import('./dist/cli-adapter.js').then(m=>console.log(m.resolveCliModel('codex')))" +undefined + +# Reading: exit 0, verified output, no retry. stderr_tail still shows the Codex 'Reading additional input from stdin...' line on codex-cli 0.154.0: with stdin ignored (EOF) the read returns at once and the process completes, so the lifecycle no longer strands it; the line itself is not gone. transport.retryable=false because exit was 0 (the retryable signature requires the nonzero exit + that stderr). diff --git a/kernel/evidence/501/green-kernel.txt b/kernel/evidence/501/green-kernel.txt index cca0036b7..989d5d6be 100644 --- a/kernel/evidence/501/green-kernel.txt +++ b/kernel/evidence/501/green-kernel.txt @@ -2,34 +2,34 @@ $ cd kernel && cargo test --workspace Compiling relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) Compiling relayflowd-journal v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-journal) Compiling relayflowd v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd) - Finished `test` profile [unoptimized + debuginfo] target(s) in 2.02s + Finished `test` profile [unoptimized + debuginfo] target(s) in 1.58s Running unittests src/lib.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/relayflowd-f043db0bb3534a16) running 53 tests test engine::remote::worker_failure_detail_tests::a_null_or_blank_output_yields_no_detail ... ok test engine::remote::worker_failure_detail_tests::a_string_output_is_carried_verbatim_and_trimmed ... ok -test engine::remote::worker_failure_detail_tests::a_non_string_output_is_rendered_rather_than_dropped ... ok test engine::remote::worker_failure_detail_tests::an_output_at_the_boundary_is_not_truncated ... ok test engine::boot_identity_tests::every_engine_in_this_process_shares_one_boot_id ... ok -test engine::remote::worker_failure_detail_tests::truncation_does_not_split_a_multi_byte_char ... ok +test engine::remote::worker_failure_detail_tests::a_non_string_output_is_rendered_rather_than_dropped ... ok +test server::client::tests::resume_waits_while_the_heartbeat_renewed_lease_is_live ... ok test server::liveness::tests::sweep_id_buckets_by_the_interval ... ok test server::channels::tests::unknown_verb_never_falls_through_to_receive ... ok -test server::client::tests::resume_waits_while_the_heartbeat_renewed_lease_is_live ... ok test engine::remote::worker_failure_detail_tests::render_is_bounded_before_allocation_for_hostile_string ... ok -test server::tests::agent::contract::an_oversized_trajectory_tail_is_refused_at_step_complete ... ok -test server::tests::agent::eligibility::required_streams_must_be_held_before_worker_registration ... ok -test engine::remote::worker_failure_detail_tests::render_is_bounded_before_allocation_for_hostile_json ... ok -test exec_det::tests::timeout_has_an_explicit_completion_reason ... ok -test exec_det::tests::captures_deterministic_output ... ok +test engine::remote::worker_failure_detail_tests::truncation_does_not_split_a_multi_byte_char ... ok test server::tests::agent::contract::an_agent_worker_attaching_without_pins_is_refused_at_attach ... ok +test server::tests::agent::eligibility::required_streams_must_be_held_before_worker_registration ... ok test exec_det::tests::failed_command_evidence_survives_completion ... ok +test exec_det::tests::captures_deterministic_output ... ok +test engine::remote::worker_failure_detail_tests::render_is_bounded_before_allocation_for_hostile_json ... ok +test server::tests::agent::contract::an_oversized_trajectory_tail_is_refused_at_step_complete ... ok test server::tests::hello_enforces_protocol_version ... ok test exec_det::tests::lease_override_bounds_execution_and_preserves_command_timeout ... ok -test server::tests::run_resume_asks_the_registry_instead_of_treating_an_orphan_file_as_a_run ... ok +test exec_det::tests::timeout_has_an_explicit_completion_reason ... ok test engine::wake::claim_guard_tests::a_disarmed_guard_leaves_the_claim_alone ... ok test server::liveness::tests::sweep_pass_healthy_subscription_is_a_noop ... ok -test server::tests::run_start_fails_closed_on_an_unknown_verification_key ... ok test server::liveness::tests::sweep_pass_latches_after_journaling_and_next_bucket_is_empty ... ok +test server::tests::run_start_fails_closed_on_an_unknown_verification_key ... ok +test server::tests::run_resume_asks_the_registry_instead_of_treating_an_orphan_file_as_a_run ... ok test engine::wake::claim_guard_tests::a_panic_between_claim_and_register_still_releases ... ok test engine::wake::claim_guard_tests::a_guard_only_releases_its_own_run ... ok test socket_path::tests::deep_data_dir_produces_short_socket_path ... ok @@ -39,18 +39,18 @@ test socket_path::tests::same_data_dir_yields_same_socket ... ok test engine::wake::claim_guard_tests::an_armed_guard_releases_the_claim_when_dropped ... ok test server::tests::run_start_refuses_invalid_admission_keys ... ok test server::tests::run_resume_refuses_a_journal_that_never_recorded_its_run ... ok -test server::tests::agent::contract::an_agent_worker_missing_a_declared_surface_parks_the_run_instead_of_erroring ... ok test server::tests::agent::contract::agent_without_a_compatible_worker_parks_without_starting ... ok +test server::tests::agent::contract::an_agent_worker_missing_a_declared_surface_parks_the_run_instead_of_erroring ... ok test server::tests::agent::eligibility::required_streams_keep_ordinary_steps_off_conversation_workers ... ok test engine::boot_identity_tests::prior_boot_registered_undriven_admission_is_recovered_by_start_retry ... ok test server::tests::run_start_admission_key_recovers_the_same_run_and_refuses_spec_drift ... ok +test server::tests::run_resume_refuses_a_valid_journal_that_belongs_to_another_run ... ok +test server::tests::agent::contract::an_llm_completion_claiming_an_effect_fails_closed_with_the_reason_journaled ... ok test server::tests::run_resume_adopts_a_real_journal_whose_registry_row_is_missing ... ok -test engine::boot_identity_tests::failure_after_workspace_binding_releases_admission_without_exposing_effects ... ok test server::tests::agent::pins::reset_worker_reporting_a_revision_other_than_its_pin_fails_closed_as_worker_error ... ok -test server::tests::agent::contract::an_llm_completion_claiming_an_effect_fails_closed_with_the_reason_journaled ... ok -test server::tests::run_resume_refuses_a_valid_journal_that_belongs_to_another_run ... ok -test server::tests::agent::contract::a_replacement_worker_that_never_reported_the_pinned_surface_is_not_dispatched_to ... ok +test engine::boot_identity_tests::failure_after_workspace_binding_releases_admission_without_exposing_effects ... ok test server::tests::agent::contract::a_transcript_digest_at_its_budget_rides_trajectory_tail_verbatim ... ok +test server::tests::agent::contract::a_replacement_worker_that_never_reported_the_pinned_surface_is_not_dispatched_to ... ok test server::tests::a_failed_disconnect_journal_append_is_retained_and_retried_not_dropped ... ok test server::tests::agent::contract::human_intervention_is_durable_and_resume_requires_explicit_override ... ok test server::tests::stopped_heartbeats_past_the_deadline_journal_lease_expired_and_release_the_step ... ok @@ -75,9 +75,9 @@ test metering_flag_is_additive_on_the_wire ... ok test prior_spend_metering_flag_is_additive_and_fails_closed_for_older_kernels ... ok test daily_windows_reset_and_exact_limits_do_not_refuse ... ok test carried_metered_dollars_still_stop_the_continuing_run ... ok -test unmetered_tokens_still_cross_a_token_ceiling ... ok -test unmetered_usage_may_not_claim_priced_dollars ... ok test carried_prior_spend_keeps_unknown_dollar_cost_unmetered ... ok +test unmetered_usage_may_not_claim_priced_dollars ... ok +test unmetered_tokens_still_cross_a_token_ceiling ... ok test crossing_completion_is_durable_and_next_step_is_refused ... ok test unmetered_spend_is_journaled_as_unknown_and_never_crosses_a_dollar_ceiling ... ok test deterministic_spend_and_wallclock_limit_gate_parallel_batch_starts ... ok @@ -89,36 +89,36 @@ test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; fin running 41 tests test agent::resume_without_a_worker_parks_immediately_instead_of_timing_out ... ok test concurrency::run_start_dispatches_every_independent_lane_before_any_completion ... ok -test channels::channels_reject_foreign_workers_stale_attempts_and_invalid_acknowledgements ... ok test agent::rung_c_sigkill_after_final_effect_replays_results_without_redispatch ... ok +test concurrency::cancel_closes_the_lease_and_rejects_a_late_completion ... ok +test channels::channels_reject_foreign_workers_stale_attempts_and_invalid_acknowledgements ... ok test agent::rung_c_sigkill_between_agent_completion_and_final_effect_memoizes_the_agent ... ok -test llm::serve_plumbs_watch_events_and_replayable_stream_verbs ... ok -test concurrency::live_resume_leaves_an_active_lease_running ... ok test concurrency::concurrent_resumes_lease_exactly_one_attempt ... ok -test concurrency::server_restart_recovers_every_parallel_lease_without_duplicate_success ... ok -test concurrency::cancel_closes_the_lease_and_rejects_a_late_completion ... ok test agent::rung_c_crash_between_effect_election_and_the_provider_call_performs_it_exactly_once ... ok +test llm::serve_plumbs_watch_events_and_replayable_stream_verbs ... ok +test concurrency::server_restart_recovers_every_parallel_lease_without_duplicate_success ... ok test concurrency::cancel_and_completion_race_has_one_terminal_fact ... ok -test llm::failing_llm_verification_schedules_a_durable_retry_and_succeeds ... ok -test agent::rung_c_reset_sigkill_mid_edit_restores_pins_dedupes_effect_and_explains_attempts ... ok +test concurrency::live_resume_leaves_an_active_lease_running ... ok test pin_projection::rejected_completion_cannot_forge_pins_or_trigger_a_blind_retry ... ok +test agent::rung_c_reset_sigkill_mid_edit_restores_pins_dedupes_effect_and_explains_attempts ... ok test memory::memory_sigkill_after_injection_replays_pack_and_charges_it_once ... ok test llm::llm_verification_exhaustion_is_a_declared_failure_kind ... ok +test llm::completed_llm_output_is_memoized_when_serve_dies_during_the_next_step ... ok test sigkill_mid_step_replaces_and_explains_the_dead_attempt ... ok test protocol_admission::every_mutating_run_verb_refuses_terminal_before_changing_state ... ok +test llm::failing_llm_verification_schedules_a_durable_retry_and_succeeds ... ok test sigkill_after_cancel_request_resumes_to_one_canceled_fact ... ok +test agent::rung_c_sigkill_boundaries_resume_only_unfinished_steps_via_real_cli ... ok +test surface_identity::aliases_are_rejected_and_external_ancestors_serialize_over_real_sockets ... ok test llm::sigkill_after_the_final_rung_b_effect_resumes_without_redispatching_llm ... ok +test sigkill_under_serve_resumes_the_socket_started_run ... ok test llm::worker_killed_while_holding_a_lease_is_explained_and_released_on_cli_resume ... ok -test llm::completed_llm_output_is_memoized_when_serve_dies_during_the_next_step ... ok test worker_capacity::two_workers_receive_a_deterministic_fair_capacity_bounded_batch ... ok -test sigkill_under_serve_resumes_the_socket_started_run ... ok -test surface_identity::aliases_are_rejected_and_external_ancestors_serialize_over_real_sockets ... ok -test placement::declared_placement_keeps_one_source_tree_across_resume ... ok -test agent::rung_c_sigkill_boundaries_resume_only_unfinished_steps_via_real_cli ... ok test workspace_identity::workspace_aliases_are_refused_and_canonical_subtrees_serialize_over_real_sockets ... ok +test placement::declared_placement_keeps_one_source_tree_across_resume ... ok test placement::sigkill_before_first_step_preserves_the_submitted_workspace ... ok -test placement::sigkill_mid_step_keeps_the_route_and_source_tree ... ok test llm::sigkill_under_serve_mid_llm_releases_the_lease_and_finishes_via_cli_resume ... ok +test placement::sigkill_mid_step_keeps_the_route_and_source_tree ... ok test parallel_lifecycle::terminal_failure_drains_or_explains_every_live_sibling ... ok test worker_capacity::default_capacity_one_reopens_only_after_durable_completion_or_crash ... ok test sigkill_sweep_covers_every_hello_step_boundary ... ok @@ -129,15 +129,15 @@ test channels::channels_sigkill_resume_redelivers_unacked_messages_with_exactly_ test manual_recovery::manual_recovery_parks_a_worker_reported_loss_across_daemon_restart_until_a_human_answers ... ok test parallel_lifecycle::renewed_parallel_leases_survive_the_original_grant_and_remain_distinct ... ok -test result: ok. 41 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 36.42s +test result: ok. 41 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 36.41s Running tests/daemon_lifecycle.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/daemon_lifecycle-b705da9761b2a254) running 6 tests -test deep_data_dir_still_binds ... ok test sigkill_leaves_a_stale_file_with_a_dead_pid ... ok -test clean_shutdown_removes_advertisement_and_socket ... ok test connection_file_is_published_only_after_the_socket_is_live ... ok +test deep_data_dir_still_binds ... ok +test clean_shutdown_removes_advertisement_and_socket ... ok test a_second_serve_on_a_served_data_dir_refuses_and_the_first_keeps_serving ... ok test a_sigkilled_daemons_successor_starts_cleanly ... ok @@ -146,8 +146,8 @@ test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; fini Running tests/event_wake.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/event_wake-b98778c1ad4de872) running 3 tests -test matching_event_wakes_once_with_fresh_context ... ok test two_racing_deliveries_of_one_event_produce_exactly_one_run ... ok +test matching_event_wakes_once_with_fresh_context ... ok test a_resumed_run_dispatches_the_original_wake_context ... ok test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s @@ -157,7 +157,7 @@ test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; fini running 1 test test hn_story_event_wakes_monitor_once_with_story_context ... ok -test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s Running tests/input_binding.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/input_binding-4f132e6302508de8) @@ -165,7 +165,7 @@ running 2 tests test binding_schema_is_additive_and_fails_closed ... ok test sigkill_before_consumer_resolves_original_journal_output_without_reexecuting_source ... ok -test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.06s +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.07s Running tests/invalid_schema_preflight.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/invalid_schema_preflight-9c08869ea58d283f) @@ -182,7 +182,7 @@ running 2 tests test a_park_torn_between_its_two_appends_is_repaired_on_resume ... ok test manual_park_of_a_worker_reported_loss_survives_reopen_and_answers_to_a_human ... ok -test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.05s Running tests/memoization.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/memoization-e5537edfeea8b814) @@ -191,16 +191,16 @@ test refuses_missing_wrong_flow_and_unreadable_journal_before_creating_run ... o test reused_prefix_survives_restart_without_source_and_never_mutates_prior ... ok test actual_changed_input_invalidates_consumer_even_with_identical_consumer_spec ... ok -test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.04s +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.05s Running tests/memory.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/memory-fe5dbc6738ee15ff) running 5 tests test rejected_journal_fact_releases_reservation_and_never_dispatches ... ok test llm_dispatch_receives_same_pack_after_resume_without_provider ... ok -test over_budget_and_provider_errors_fail_without_dispatch_or_charge ... ok -test replay_and_resume_need_no_provider_and_script_receives_recorded_pack ... ok test semantic_retry_reuses_memory_without_a_second_charge ... ok +test replay_and_resume_need_no_provider_and_script_receives_recorded_pack ... ok +test over_budget_and_provider_errors_fail_without_dispatch_or_charge ... ok test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.02s @@ -219,7 +219,7 @@ test backpressured_or_mismatched_lane_does_not_drop_a_later_dispatch ... ok test pause_before_second_independent_step_holds_the_driver_boundary ... ok test crash_boundaries_resume_the_real_driver_with_one_effect_per_lane ... ok -test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.05s Running tests/placement_pins.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/placement_pins-839e136752b4e5af) @@ -245,14 +245,14 @@ running 2 tests test duplicate_routes_have_a_distinct_diagnostic_and_leave_the_original_fact_intact ... ok test malformed_routes_name_the_same_field_at_append_replay_and_epoch_replay ... ok -test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s Running tests/spec_review_routing.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/spec_review_routing-032df51688a72750) running 4 tests test attempt_scoped_route_is_rejected_at_append_and_replay ... ok -test malformed_epoch_routes_are_rejected_before_commit ... ok test epoch_cannot_drop_or_replace_a_durable_route ... ok +test malformed_epoch_routes_are_rejected_before_commit ... ok test workspace_pin_peels_tags_and_refuses_non_commit_objects ... ok test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s @@ -280,78 +280,78 @@ test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; fini running 72 tests test clock::tests::simulated_clock_is_explicitly_advanced ... ok test entry::completion_reason_tests::every_journal_label_matches_serialized ... ok -test entry::completion_reason_tests::all_covers_every_serialized_label ... ok test entry::attempt_started_tests::a_pre_field_attempt_started_still_reads ... ok -test channel::tests::send_retry_is_stable_and_conflicting_content_is_rejected ... ok +test entry::completion_reason_tests::all_covers_every_serialized_label ... ok test entry::attempt_started_tests::max_transport_retries_is_always_journaled ... ok -test channel::tests::malformed_payloads_and_invalid_new_channel_appends_leave_state_unchanged ... ok test journal::tests::memory_journal_assigns_sequences_and_rolls_epochs ... ok +test channel::tests::send_retry_is_stable_and_conflicting_content_is_rejected ... ok +test channel::tests::malformed_payloads_and_invalid_new_channel_appends_leave_state_unchanged ... ok test channel::tests::forged_deliveries_and_acknowledgements_fail_closed ... ok test channel::tests::delivery_replay_and_independent_acknowledged_offsets ... ok -test machine::parallel_tests::every_declared_mutable_surface_participates_in_conflict_selection ... ok -test machine::tests::every_reason_label_matches_its_serialized_form ... ok -test machine::parallel_tests::failed_run_drains_open_siblings_before_terminal_entry ... ok test machine::parallel_tests::machine_starts_every_runnable_step_in_authored_order ... ok +test machine::parallel_tests::every_declared_mutable_surface_participates_in_conflict_selection ... ok test machine::tests::deterministic_lease_rejects_invalid_and_foreign_fields ... ok -test machine::tests::crashed_attempt_does_not_consume_an_iteration ... ok -test machine::parallel_tests::overlapping_agent_surfaces_are_serialized_in_authored_order ... ok -test machine::tests::cancel_request_closes_the_active_lease_before_the_terminal_fact ... ok +test machine::tests::deterministic_lease_override_and_default_are_journaled ... ok +test machine::tests::every_reason_label_matches_its_serialized_form ... ok +test machine::recovery_tests::reset_recovery_still_retries_a_worker_reported_transport_loss ... ok test machine::tests::all_backing_off_steps_return_timers ... ok -test machine::tests::machine_starts_runnable_step_with_stable_effect_key ... ok -test machine::tests::every_failed_run_terminates_with_declared_completion_reasons ... ok -test machine::tests::ordinary_worker_error_is_not_retried_by_either_budget ... ok +test machine::parallel_tests::workspace_ancestor_and_descendant_paths_conflict_but_siblings_do_not ... ok +test machine::tests::cancel_request_closes_the_active_lease_before_the_terminal_fact ... ok +test machine::parallel_tests::external_ancestor_and_descendant_paths_conflict_but_siblings_do_not ... ok +test machine::tests::repeated_cancel_request_is_idempotent ... ok test machine::tests::durable_cancel_request_outranks_crash_recovery ... ok -test machine::tests::manual_recovery_parks_needs_human_and_never_redispatches ... ok -test machine::tests::inspect_recovery_injects_the_dirty_pin_completion_reason_and_tail ... ok +test machine::tests::crashed_attempt_does_not_consume_an_iteration ... ok +test machine::tests::classified_transport_retry_is_bounded_and_preserves_pin_and_idempotency ... ok +test machine::parallel_tests::parallel_lanes_do_not_cross_the_dependency_barrier_early ... ok test machine::tests::successful_memo_is_never_scheduled_again ... ok -test machine::tests::repeated_cancel_request_is_idempotent ... ok -test machine::tests::reset_recovery_still_retries_a_worker_reported_transport_loss ... ok -test machine::parallel_tests::workspace_ancestor_and_descendant_paths_conflict_but_siblings_do_not ... ok -test machine::tests::deterministic_lease_override_and_default_are_journaled ... ok +test machine::parallel_tests::failed_run_drains_open_siblings_before_terminal_entry ... ok test machine::tests::worker_reported_failure_without_detail_still_records_a_verification ... ok -test machine::tests::reset_recovery_dispatches_the_original_pinned_revision ... ok -test machine::parallel_tests::parallel_lanes_do_not_cross_the_dependency_barrier_early ... ok -test retry::tests::jitter_is_repeatable_and_bounded ... ok -test machine::tests::classified_transport_retry_is_bounded_and_preserves_pin_and_idempotency ... ok +test machine::tests::inspect_recovery_injects_the_dirty_pin_completion_reason_and_tail ... ok +test machine::parallel_tests::overlapping_agent_surfaces_are_serialized_in_authored_order ... ok test memory::tests::caps_compare_exact_decimals_and_each_token_dimension ... ok test machine::tests::verification_failure_schedules_a_durable_retry ... ok -test machine::tests::failed_deterministic_completion_preserves_exit_code_and_stderr ... ok -test machine::parallel_tests::external_ancestor_and_descendant_paths_conflict_but_siblings_do_not ... ok test machine::parallel_tests::crash_resume_preserves_each_parallel_lease_exactly_once ... ok +test machine::tests::ordinary_worker_error_is_not_retried_by_either_budget ... ok +test machine::tests::machine_starts_runnable_step_with_stable_effect_key ... ok +test machine::tests::manual_recovery_parks_needs_human_and_never_redispatches ... ok +test machine::tests::reset_recovery_dispatches_the_original_pinned_revision ... ok +test machine::recovery_tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching ... ok +test machine::parallel_tests::disjoint_agent_lanes_merge_pins_in_either_completion_order ... ok +test machine::recovery_tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote ... ok test spec::tests::a_misspelled_step_level_key_is_a_parse_error ... ok test schema::tests::in_document_uri_references_resolve_to_the_node_they_name ... ok +test machine::tests::every_failed_run_terminates_with_declared_completion_reasons ... ok test schema::tests::refusal_names_the_cycle_it_found ... ok test spec::tests::a_misspelled_verification_gate_key_is_a_parse_error_not_a_dropped_gate ... ok -test spec::tests::spec_version_is_semver_and_gated ... ok +test machine::tests::failed_deterministic_completion_preserves_exit_code_and_stderr ... ok +test retry::tests::jitter_is_repeatable_and_bounded ... ok test spec::tests::cycles_are_rejected ... ok -test machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote ... ok +test spec::tests::spec_version_is_semver_and_gated ... ok test spec::tests::unknown_root_and_nested_fields_are_rejected ... ok -test machine::parallel_tests::disjoint_agent_lanes_merge_pins_in_either_completion_order ... ok -test spec::tests::preflight_data_is_fail_closed ... ok +test state::budget::tests::overflow_and_malformed_cost_leave_total_unchanged ... ok test spec::tests::zero_agent_flow_is_valid ... ok test state::budget::tests::adds_costs_exactly_beyond_machine_decimal_precision ... ok -test state::budget::tests::overflow_and_malformed_cost_leave_total_unchanged ... ok +test spec::tests::preflight_data_is_fail_closed ... ok test state::tests::budget_decimal_strings_add_without_floats ... ok +test verify::tests::deterministic_output_requires_successful_exit_and_content ... ok +test verify::tests::an_unbounded_schema_in_a_journal_fails_its_gate_instead_of_aborting ... ok test spec::tests::external_surface_paths_must_have_one_canonical_spelling ... ok -test machine::tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching ... ok test spec::tests::workspace_mounts_and_worktrees_must_have_one_canonical_spelling ... ok -test verify::tests::deterministic_output_requires_successful_exit_and_content ... ok test state::tests::journal_replays_data_gate_verdict_without_rerunning_completed_code ... ok -test verify::tests::an_unbounded_schema_in_a_journal_fails_its_gate_instead_of_aborting ... ok -test state::tests::a_completion_that_omits_a_surface_does_not_drop_it_from_the_pin_chain ... ok test state::tests::end_pin_chain_is_enforced_and_a_broken_chain_is_a_hard_error ... ok +test state::tests::a_completion_that_omits_a_surface_does_not_drop_it_from_the_pin_chain ... ok test schema::tests::references_the_bound_leaves_opaque_are_refused_by_the_engine ... ok test spec::tests::the_full_ladder_parses_in_the_one_dialect ... ok test verify::tests::json_schema_is_a_control_gate ... ok test schema::tests::a_property_named_ref_is_not_a_reference ... ok test schema::tests::shared_declarations_and_boolean_schemas_are_validated ... ok -test spec::tests::sdk_boundary_accepts_a_valid_10_000_step_reverse_chain ... ok +test schema::tests::every_refused_corpus_schema_compiles_but_is_refused_by_the_bound ... ok test spec::tests::sdk_boundary_rejects_a_10_000_step_cycle_with_a_typed_error ... ok test schema::tests::every_accepted_corpus_schema_is_accepted ... ok -test schema::tests::every_refused_corpus_schema_compiles_but_is_refused_by_the_bound ... ok +test spec::tests::sdk_boundary_accepts_a_valid_10_000_step_reverse_chain ... ok test schema::tests::deeply_nested_schemas_do_not_overflow_the_checker ... ok -test result: ok. 72 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.72s +test result: ok. 72 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.51s Running tests/memoization.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/memoization-79f552e4cb727501) @@ -366,12 +366,12 @@ test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; fini Running tests/spec_parity.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/spec_parity-a9966affd5aca87f) running 10 tests -test the_kernel_parses_the_event_triggered_spec_and_stamps_the_same_hash ... ok -test placement_requirements_have_identical_canonical_bytes_and_hash ... ok test the_kernel_round_trips_declared_agent_transports_and_rejects_unknown_values ... ok +test placement_requirements_have_identical_canonical_bytes_and_hash ... ok +test the_kernel_parses_the_event_triggered_spec_and_stamps_the_same_hash ... ok test memory_declaration_acceptance_matches_the_sdk_corpus ... ok -test the_kernel_parses_the_deterministic_rung_and_stamps_the_same_hash ... ok test the_kernel_parses_the_rung_c_agent_spec_and_stamps_the_same_hash ... ok +test the_kernel_parses_the_deterministic_rung_and_stamps_the_same_hash ... ok test step_memory_has_identical_canonical_bytes_and_hash ... ok test placement_declaration_acceptance_matches_the_sdk_corpus ... ok test the_kernel_parses_the_sdk_compiled_spec_and_stamps_the_same_hash ... ok @@ -382,36 +382,36 @@ test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; fin Running unittests src/lib.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/relayflowd_journal-0286edd157fee7d6) running 32 tests -test channel::tests::stale_attempts_and_raw_forged_acknowledgements_cannot_change_offsets ... ok -test registry::tests::a_same_boot_claim_with_no_run_yet_is_a_duplicate_not_wreckage ... ok +test subscriptions::tests::last_run_for_subscription_returns_none_before_first_arrival ... ok +test registry::tests::releasing_a_claim_lets_the_same_boot_retry ... ok +test registry::tests::prior_boot_unregistered_run_admission_is_repaired ... ok +test subscriptions::tests::sweep_does_not_re_emit_the_same_stale_row_on_a_later_tick ... ok test registry::tests::a_previous_boots_claim_with_no_run_is_repaired ... ok -test subscriptions::tests::sweep_marks_row_stale_when_silence_exceeds_budget ... ok +test registry::tests::a_same_boot_claim_with_no_run_yet_is_a_duplicate_not_wreckage ... ok +test channel::tests::stale_attempts_and_raw_forged_acknowledgements_cannot_change_offsets ... ok +test registry::tests::concurrent_same_boot_run_admissions_have_one_owner ... ok +test subscriptions::tests::detect_without_latch_stays_available_for_the_next_sweep ... ok +test subscriptions::tests::upsert_after_stale_re_arms_and_next_silence_can_re_emit ... ok test registry::tests::registry_is_a_rebuildable_run_locator ... ok -test registry::tests::run_admission_reuses_registered_run_and_rejects_spec_drift ... ok -test registry::tests::prior_boot_unregistered_run_admission_is_repaired ... ok -test registry::tests::a_registered_run_dedupes_across_boots ... ok +test subscriptions::tests::sweep_marks_row_stale_when_silence_exceeds_budget ... ok test subscriptions::tests::sweep_ignores_subscriptions_whose_silence_is_still_within_budget ... ok test registry::tests::releasing_is_scoped_to_the_claiming_run ... ok -test subscriptions::tests::detect_without_latch_stays_available_for_the_next_sweep ... ok -test subscriptions::tests::last_run_for_subscription_returns_none_before_first_arrival ... ok -test subscriptions::tests::sweep_does_not_re_emit_the_same_stale_row_on_a_later_tick ... ok -test registry::tests::releasing_a_claim_lets_the_same_boot_retry ... ok -test subscriptions::tests::prune_sweep_claims_deletes_only_rows_older_than_cutoff ... ok -test subscriptions::tests::sweep_election_gives_the_first_caller_the_result_and_second_gets_empty ... ok -test subscriptions::tests::latch_is_a_no_op_if_a_fresh_event_arrived_between_detect_and_latch ... ok test subscriptions::tests::last_run_for_subscription_returns_the_lex_greatest_ulid_regardless_of_insertion ... ok -test registry::tests::concurrent_new_boot_retries_have_one_recovery_owner ... ok -test channel::tests::channels_cross_segment_boundaries_and_terminal_runs_reject_mutations ... ok -test subscriptions::tests::upsert_after_stale_re_arms_and_next_silence_can_re_emit ... ok -test registry::tests::concurrent_same_boot_run_admissions_have_one_owner ... ok -test registry::tests::a_pre_migration_registry_gains_boot_id_and_its_claims_are_repairable ... ok +test subscriptions::tests::prune_sweep_claims_deletes_only_rows_older_than_cutoff ... ok +test registry::tests::a_registered_run_dedupes_across_boots ... ok +test registry::tests::run_admission_reuses_registered_run_and_rejects_spec_drift ... ok test subscriptions::tests::upsert_is_idempotent_across_bumps_and_preserves_event_type_updates ... ok test tests::failed_commit_is_returned_not_swallowed ... ok +test subscriptions::tests::latch_is_a_no_op_if_a_fresh_event_arrived_between_detect_and_latch ... ok +test subscriptions::tests::sweep_election_gives_the_first_caller_the_result_and_second_gets_empty ... ok +test tests::effects_are_deduplicated_at_the_journal_boundary ... ok test tests::an_unconfirmed_election_is_reclaimed_by_the_next_attempt_not_treated_as_done ... ok -test tests::rollover_is_atomic_scaffolding_for_epoch_resume ... ok -test tests::terminal_run_refuses_every_later_entry_atomically ... ok +test channel::tests::channels_cross_segment_boundaries_and_terminal_runs_reject_mutations ... ok test tests::append_is_durable_and_monotonic_after_reopen ... ok -test tests::effects_are_deduplicated_at_the_journal_boundary ... ok +test tests::terminal_run_refuses_every_later_entry_atomically ... ok +test registry::tests::a_pre_migration_registry_gains_boot_id_and_its_claims_are_repairable ... ok +test tests::rollover_is_atomic_scaffolding_for_epoch_resume ... ok +test registry::tests::concurrent_new_boot_retries_have_one_recovery_owner ... ok test channel::tests::failed_channel_writes_never_expose_delivery_or_advance_acknowledged_offset ... ok test channel::tests::independent_connections_serialize_send_receive_and_acknowledgement ... ok diff --git a/kernel/evidence/501/green-sdk-authored-node-runtime.txt b/kernel/evidence/501/green-sdk-authored-node-runtime.txt new file mode 100644 index 000000000..379afe243 --- /dev/null +++ b/kernel/evidence/501/green-sdk-authored-node-runtime.txt @@ -0,0 +1,28 @@ +# Isolated toolchain: `mise install bun@1.4.0` (global config untouched: `bun --version` on PATH is still 1.4.2). Node 22.23.2 from mise. +$ mise where bun@1.4.0 -> /home/khaliqgant/.local/share/mise/installs/bun/1.4.0 +$ /home/khaliqgant/.local/share/mise/installs/bun/1.4.0/bin/bun --version -> 1.4.0 +$ /home/khaliqgant/.local/share/mise/installs/node/22.23.2/bin/node --version -> v22.23.2 +$ cd packages/sdk && FLOWS_BUILD_BUN=/home/khaliqgant/.local/share/mise/installs/bun/1.4.0/bin/bun FLOWS_AUTHORED_NODE=/home/khaliqgant/.local/share/mise/installs/node/22.23.2/bin/node RELAYFLOWD_BIN=/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/relayflowd mise exec node@22.23.2 -- npx vitest run tests/authored-node-runtime.test.ts + + RUN v2.1.9 /tmp/flows-fleet-501/packages/sdk + + ✓ tests/authored-node-runtime.test.ts (14 tests) 25718ms + ✓ Bun 1.4.0 standalone → native Node authored lifecycle > suppresses the loader warning while preserving authored experimental warnings 1049ms + ✓ Bun 1.4.0 standalone → native Node authored lifecycle > awaits agent plus three run steps and resumes without repeating effects 1713ms + ✓ Bun 1.4.0 standalone → native Node authored lifecycle > accepts a predicate-gated flow: the `.gate` child is journaled, verified, and not counted as an authored step 1516ms + ✓ Bun 1.4.0 standalone → native Node authored lifecycle > parks an f.human across the IPC boundary, answers it, and resumes the Node body with the answer 3809ms + ✓ Bun 1.4.0 standalone → native Node authored lifecycle > stops on parent SIGKILL and replays completed children before success 3321ms + ✓ Bun 1.4.0 standalone → native Node authored lifecycle > stops on parent SIGTERM and replays completed children before success 3050ms + ✓ Bun 1.4.0 standalone → native Node authored lifecycle > stops on parent blocked-SIGKILL and replays completed children before success 3300ms + ✓ Bun 1.4.0 standalone → native Node authored lifecycle > stops on parent SIGKILL and replays completed children before declined 3074ms + ✓ Bun 1.4.0 standalone → native Node authored lifecycle > refuses unawaited rather than reporting terminal success 1091ms + ✓ Bun 1.4.0 standalone → native Node authored lifecycle > refuses manual then rather than reporting terminal success 1158ms + ✓ Bun 1.4.0 standalone → native Node authored lifecycle > loads captured graph bytes before preserving the unsupported-use refusal 893ms + ✓ Bun 1.4.0 standalone → native Node authored lifecycle > rejects a forged result frame without durable completion 980ms + + Test Files 1 passed (1) + Tests 14 passed (14) + Start at 02:56:04 + Duration 26.84s (transform 485ms, setup 0ms, collect 866ms, tests 25.72s, environment 0ms, prepare 40ms) + +exit_code=0 diff --git a/kernel/evidence/501/green-sdk.txt b/kernel/evidence/501/green-sdk.txt index 5eaf05e9a..e8008fd2b 100644 --- a/kernel/evidence/501/green-sdk.txt +++ b/kernel/evidence/501/green-sdk.txt @@ -415,7 +415,7 @@ Expected: "1.4.0" Received: "1.4.2" ❯ tests/authored-node-runtime.test.ts:18:77 - 16| + 16| 17| beforeAll(() => { 18| expect(spawnSync(bun, ['--version'], { encoding: 'utf8' }).stdout.tr… | ^ diff --git a/kernel/evidence/501/green-targeted.txt b/kernel/evidence/501/green-targeted.txt deleted file mode 100644 index 7e69e547b..000000000 --- a/kernel/evidence/501/green-targeted.txt +++ /dev/null @@ -1,54 +0,0 @@ -# Production files restored byte-for-byte (sha256sum -c against pre-mutation copies: OK). Same commands as red.txt: -$ cd kernel && cargo test -p relayflowd-core --lib -- machine::tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote machine::tests::reset_recovery_still_retries_a_worker_reported_transport_loss - Compiling relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) - Finished `test` profile [unoptimized + debuginfo] target(s) in 0.60s - Running unittests src/lib.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/relayflowd_core-82d87717b3255c51) - -running 3 tests -test machine::tests::reset_recovery_still_retries_a_worker_reported_transport_loss ... ok -test machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote ... ok -test machine::tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching ... ok - -test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 69 filtered out; finished in 0.00s - -exit_code=0 - -$ cd kernel && cargo test -p relayflowd --test manual_recovery - Compiling relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) - Compiling relayflowd-journal v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-journal) - Compiling relayflowd v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd) - Finished `test` profile [unoptimized + debuginfo] target(s) in 1.00s - Running tests/manual_recovery.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/manual_recovery-6793123a1244aad2) - -running 2 tests -test a_park_torn_between_its_two_appends_is_repaired_on_resume ... ok -test manual_park_of_a_worker_reported_loss_survives_reopen_and_answers_to_a_human ... ok - -test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.03s - -exit_code=0 - -$ cd kernel && cargo test -p relayflowd --test crash_resume manual_recovery - Compiling relayflowd v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd) - Finished `test` profile [unoptimized + debuginfo] target(s) in 0.40s - Running tests/crash_resume.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/crash_resume-d2d102ec3e89b705) - -running 1 test -test manual_recovery::manual_recovery_parks_a_worker_reported_loss_across_daemon_restart_until_a_human_answers ... ok - -test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 40 filtered out; finished in 1.46s - -exit_code=0 - -$ cd kernel && cargo test -p relayflowd-core --lib -- machine::tests::all_backing_off_steps_return_timers entry::attempt_started_tests - Finished `test` profile [unoptimized + debuginfo] target(s) in 0.02s - Running unittests src/lib.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/relayflowd_core-82d87717b3255c51) - -running 3 tests -test entry::attempt_started_tests::a_pre_field_attempt_started_still_reads ... ok -test entry::attempt_started_tests::max_transport_retries_is_always_journaled ... ok -test machine::tests::all_backing_off_steps_return_timers ... ok - -test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 69 filtered out; finished in 0.00s - -exit_code=0 diff --git a/kernel/evidence/501/mutation-transcript.txt b/kernel/evidence/501/mutation-transcript.txt new file mode 100644 index 000000000..c4a214615 --- /dev/null +++ b/kernel/evidence/501/mutation-transcript.txt @@ -0,0 +1,305 @@ +### 1. Pre-mutation checksums + +$ sha256sum relayflowd-core/src/machine.rs relayflowd-core/src/machine/recovery.rs +66335467e788ee9f9a6c666dedb4bb0133e5610477feb5e8c213c5366b95d1b7 relayflowd-core/src/machine.rs +0496ed9a034d4898221a34dbe79c6d599479d2e1a3f03121b6b1c8febb498adc relayflowd-core/src/machine/recovery.rs +exit_code=0 + +### 2. Mutation A+B: worker-reported manual park disabled (machine.rs) AND torn-park repair disabled (recovery.rs) + +$ sed -i s/ let manual_park = transport_failure$/ let manual_park = false \&\& transport_failure/ relayflowd-core/src/machine.rs +exit_code=0 + +$ sed -i s/ ) \&\& \*wait_id == park_placeholder_wait_id(&spec.id, runtime.attempts) =>/ ) \&\& false \&\& *wait_id == park_placeholder_wait_id(\&spec.id, runtime.attempts) =>/ relayflowd-core/src/machine/recovery.rs +exit_code=0 + +$ git diff --stat -- relayflowd-core/src/machine.rs relayflowd-core/src/machine/recovery.rs + kernel/relayflowd-core/src/machine.rs | 4 +++- + kernel/relayflowd-core/src/machine/recovery.rs | 2 +- + 2 files changed, 4 insertions(+), 2 deletions(-) +exit_code=0 + +$ git diff -U0 -- relayflowd-core/src/machine.rs relayflowd-core/src/machine/recovery.rs +diff --git i/kernel/relayflowd-core/src/machine.rs w/kernel/relayflowd-core/src/machine.rs +index c36d37b0..91548b4c 100644 +--- i/kernel/relayflowd-core/src/machine.rs ++++ w/kernel/relayflowd-core/src/machine.rs +@@ -397 +397 @@ pub fn completion_actions( +- let manual_park = transport_failure ++ let manual_park = false && transport_failure +@@ -569,0 +570,2 @@ mod parallel_tests; ++mod recovery_tests; ++#[cfg(test)] +diff --git i/kernel/relayflowd-core/src/machine/recovery.rs w/kernel/relayflowd-core/src/machine/recovery.rs +index 355bb149..e92c3d17 100644 +--- i/kernel/relayflowd-core/src/machine/recovery.rs ++++ w/kernel/relayflowd-core/src/machine/recovery.rs +@@ -57 +57 @@ pub fn recovery_actions_filtered( +- ) && *wait_id == park_placeholder_wait_id(&spec.id, runtime.attempts) => ++ ) && false && *wait_id == park_placeholder_wait_id(&spec.id, runtime.attempts) => +exit_code=0 + +### 3. RED under mutation A+B + +$ sh -c cargo test -p relayflowd-core --lib -- machine::recovery_tests + Compiling relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.65s + Running unittests src/lib.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/relayflowd_core-82d87717b3255c51) + +running 3 tests +test machine::recovery_tests::reset_recovery_still_retries_a_worker_reported_transport_loss ... ok +test machine::recovery_tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote ... FAILED +test machine::recovery_tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching ... FAILED + +failures: + +---- machine::recovery_tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote stdout ---- + +thread 'machine::recovery_tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote' (2406525) panicked at relayflowd-core/src/machine/recovery_tests.rs:174:9: +assertion `left == right` failed: worker-reported: the torn prefix folds to the placeholder + left: Backoff { attempt: 1, wake_at_ms: 20 } + right: NeedsHuman { wait_id: "park-agent-1" } +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + +---- machine::recovery_tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching stdout ---- + +thread 'machine::recovery_tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching' (2406524) panicked at relayflowd-core/src/machine/recovery_tests.rs:45:9: +assertion `left == right` failed: Crashed: a park is a completion plus a wait.human and nothing else (no retry timer): [Append(JournalEntry { seq: 0, segment_id: 0, entry_type: StepCompleted, run_id: "run", step_id: Some("agent"), attempt: Some(1), at_ms: 20, payload: Object {"budget": Object {"dollars": String("0"), "tokens_in": Number(0), "tokens_out": Number(0)}, "completed_by": String("worker"), "completionReason": String("crashed"), "disposition": String("retry"), "effects": Array [], "end_pins": Object {"streams": Array [], "workspace": Array [Object {"revision_id": String("rev-dirty"), "surface": String("repo")}]}, "next_attempt_at_ms": Number(20), "output": Null, "spend": Object {"dollars": Number(0), "tokens_input": Number(0), "tokens_output": Number(0), "wallclock_ms": Number(0)}, "trajectory_tail": Object {"transport": Object {"cause": String("signal_close")}}, "verification": Object {"detail": String("direct transport closed by signal"), "gate": String("execution"), "verdict": String("fail")}} }), Append(JournalEntry { seq: 0, segment_id: 0, entry_type: SleepUntil, run_id: "run", step_id: Some("agent"), attempt: Some(1), at_ms: 20, payload: Object {"reason": String("retry_backoff"), "wait_id": String("000000000MNSWFKX4NGBY4K6EJ"), "wake_at_ms": Number(20)} }), ArmTimer { at_ms: 20 }] + left: 3 + right: 2 + + +failures: + machine::recovery_tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching + machine::recovery_tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote + +test result: FAILED. 1 passed; 2 failed; 0 ignored; 0 measured; 69 filtered out; finished in 0.00s + +error: test failed, to rerun pass `-p relayflowd-core --lib` +exit_code=101 + +$ sh -c cargo test -p relayflowd --test manual_recovery + Compiling relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) + Compiling relayflowd-journal v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-journal) + Compiling relayflowd v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd) + Finished `test` profile [unoptimized + debuginfo] target(s) in 1.16s + Running tests/manual_recovery.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/manual_recovery-6793123a1244aad2) + +running 2 tests +test manual_park_of_a_worker_reported_loss_survives_reopen_and_answers_to_a_human ... FAILED +test a_park_torn_between_its_two_appends_is_repaired_on_resume ... FAILED + +failures: + +---- manual_park_of_a_worker_reported_loss_survives_reopen_and_answers_to_a_human stdout ---- + +thread 'manual_park_of_a_worker_reported_loss_survives_reopen_and_answers_to_a_human' (2407345) panicked at relayflowd/tests/manual_recovery.rs:189:9: +assertion `left == right` failed: Crashed with max_transport_retries=1: a manual step must not be redispatched after a reported loss + left: 2 + right: 1 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + +---- a_park_torn_between_its_two_appends_is_repaired_on_resume stdout ---- + +thread 'a_park_torn_between_its_two_appends_is_repaired_on_resume' (2407344) panicked at relayflowd/tests/manual_recovery.rs:265:5: +the injected crash must fire after the park append + + +failures: + a_park_torn_between_its_two_appends_is_repaired_on_resume + manual_park_of_a_worker_reported_loss_survives_reopen_and_answers_to_a_human + +test result: FAILED. 0 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s + +error: test failed, to rerun pass `-p relayflowd --test manual_recovery` +exit_code=101 + +$ sh -c cargo test -p relayflowd --test crash_resume manual_recovery + Compiling relayflowd v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd) + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.39s + Running tests/crash_resume.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/crash_resume-d2d102ec3e89b705) + +running 1 test +test manual_recovery::manual_recovery_parks_a_worker_reported_loss_across_daemon_restart_until_a_human_answers ... FAILED + +failures: + +---- manual_recovery::manual_recovery_parks_a_worker_reported_loss_across_daemon_restart_until_a_human_answers stdout ---- + +thread 'manual_recovery::manual_recovery_parks_a_worker_reported_loss_across_daemon_restart_until_a_human_answers' (2407591) panicked at relayflowd/tests/crash_resume/manual_recovery.rs:74:5: +crashed with max_transport_retries=1: a parked manual step must not be redispatched +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + + +failures: + manual_recovery::manual_recovery_parks_a_worker_reported_loss_across_daemon_restart_until_a_human_answers + +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 40 filtered out; finished in 0.03s + +error: test failed, to rerun pass `-p relayflowd --test crash_resume` +exit_code=101 + +### 4. Restore byte-for-byte and verify + +$ cp /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/mutation-keep/machine.rs.keep relayflowd-core/src/machine.rs +exit_code=0 + +$ cp /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/mutation-keep/recovery.rs.keep relayflowd-core/src/machine/recovery.rs +exit_code=0 + +$ sha256sum -c /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/mutation-keep/keep.sha +relayflowd-core/src/machine.rs: OK +relayflowd-core/src/machine/recovery.rs: OK +exit_code=0 + +$ git diff --stat -- relayflowd-core/src/machine.rs relayflowd-core/src/machine/recovery.rs + kernel/relayflowd-core/src/machine.rs | 2 ++ + 1 file changed, 2 insertions(+) +exit_code=0 + +### 5. Mutation B only: torn-park repair disabled (recovery.rs); park intact + +$ sed -i s/ ) \&\& \*wait_id == park_placeholder_wait_id(&spec.id, runtime.attempts) =>/ ) \&\& false \&\& *wait_id == park_placeholder_wait_id(\&spec.id, runtime.attempts) =>/ relayflowd-core/src/machine/recovery.rs +exit_code=0 + +$ git diff -U0 -- relayflowd-core/src/machine/recovery.rs +diff --git i/kernel/relayflowd-core/src/machine/recovery.rs w/kernel/relayflowd-core/src/machine/recovery.rs +index 355bb149..e92c3d17 100644 +--- i/kernel/relayflowd-core/src/machine/recovery.rs ++++ w/kernel/relayflowd-core/src/machine/recovery.rs +@@ -57 +57 @@ pub fn recovery_actions_filtered( +- ) && *wait_id == park_placeholder_wait_id(&spec.id, runtime.attempts) => ++ ) && false && *wait_id == park_placeholder_wait_id(&spec.id, runtime.attempts) => +exit_code=0 + +### 6. RED under mutation B only (repair-specific tests) + +$ sh -c cargo test -p relayflowd-core --lib -- machine::recovery_tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote + Compiling relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.71s + Running unittests src/lib.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/relayflowd_core-82d87717b3255c51) + +running 1 test +test machine::recovery_tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote ... FAILED + +failures: + +---- machine::recovery_tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote stdout ---- + +thread 'machine::recovery_tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote' (2407915) panicked at relayflowd-core/src/machine/recovery_tests.rs:187:9: +assertion `left == right` failed: worker-reported: recovery journals exactly the missing wait: [] + left: 0 + right: 1 +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + + +failures: + machine::recovery_tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote + +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 71 filtered out; finished in 0.00s + +error: test failed, to rerun pass `-p relayflowd-core --lib` +exit_code=101 + +$ sh -c cargo test -p relayflowd --test manual_recovery a_park_torn_between_its_two_appends_is_repaired_on_resume + Compiling relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) + Compiling relayflowd-journal v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-journal) + Compiling relayflowd v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd) + Finished `test` profile [unoptimized + debuginfo] target(s) in 1.25s + Running tests/manual_recovery.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/manual_recovery-6793123a1244aad2) + +running 1 test +test a_park_torn_between_its_two_appends_is_repaired_on_resume ... FAILED + +failures: + +---- a_park_torn_between_its_two_appends_is_repaired_on_resume stdout ---- + +thread 'a_park_torn_between_its_two_appends_is_repaired_on_resume' (2408757) panicked at relayflowd/tests/manual_recovery.rs:66:13: +injected crash between the park completion and its wait.human +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace + +thread 'a_park_torn_between_its_two_appends_is_repaired_on_resume' (2408757) panicked at relayflowd/tests/manual_recovery.rs:144:5: +assertion `left == right` failed: exactly one wait.human: [] + left: 0 + right: 1 + + +failures: + a_park_torn_between_its_two_appends_is_repaired_on_resume + +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 1 filtered out; finished in 0.01s + +error: test failed, to rerun pass `-p relayflowd --test manual_recovery` +exit_code=101 + +### 7. Restore byte-for-byte and verify + +$ cp /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/mutation-keep/recovery.rs.keep relayflowd-core/src/machine/recovery.rs +exit_code=0 + +$ sha256sum -c /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/mutation-keep/keep.sha +relayflowd-core/src/machine.rs: OK +relayflowd-core/src/machine/recovery.rs: OK +exit_code=0 + +$ git diff --stat -- relayflowd-core/src/machine.rs relayflowd-core/src/machine/recovery.rs + kernel/relayflowd-core/src/machine.rs | 2 ++ + 1 file changed, 2 insertions(+) +exit_code=0 + +### 8. GREEN on restored code + +$ sh -c cargo test -p relayflowd-core --lib -- machine::recovery_tests + Compiling relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.71s + Running unittests src/lib.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/relayflowd_core-82d87717b3255c51) + +running 3 tests +test machine::recovery_tests::reset_recovery_still_retries_a_worker_reported_transport_loss ... ok +test machine::recovery_tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote ... ok +test machine::recovery_tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 69 filtered out; finished in 0.00s + +exit_code=0 + +$ sh -c cargo test -p relayflowd --test manual_recovery + Compiling relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) + Compiling relayflowd-journal v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-journal) + Compiling relayflowd v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd) + Finished `test` profile [unoptimized + debuginfo] target(s) in 1.42s + Running tests/manual_recovery.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/manual_recovery-6793123a1244aad2) + +running 2 tests +test a_park_torn_between_its_two_appends_is_repaired_on_resume ... ok +test manual_park_of_a_worker_reported_loss_survives_reopen_and_answers_to_a_human ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.05s + +exit_code=0 + +$ sh -c cargo test -p relayflowd --test crash_resume manual_recovery + Compiling relayflowd v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd) + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.40s + Running tests/crash_resume.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/crash_resume-d2d102ec3e89b705) + +running 1 test +test manual_recovery::manual_recovery_parks_a_worker_reported_loss_across_daemon_restart_until_a_human_answers ... ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 40 filtered out; finished in 1.44s + +exit_code=0 + +$ sh -c cargo test -p relayflowd-core --lib -- machine::tests::all_backing_off_steps_return_timers entry::attempt_started_tests + Finished `test` profile [unoptimized + debuginfo] target(s) in 0.03s + Running unittests src/lib.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/relayflowd_core-82d87717b3255c51) + +running 3 tests +test entry::attempt_started_tests::a_pre_field_attempt_started_still_reads ... ok +test entry::attempt_started_tests::max_transport_retries_is_always_journaled ... ok +test machine::tests::all_backing_off_steps_return_timers ... ok + +test result: ok. 3 passed; 0 failed; 0 ignored; 0 measured; 69 filtered out; finished in 0.00s + +exit_code=0 diff --git a/kernel/evidence/501/red-repair-only.txt b/kernel/evidence/501/red-repair-only.txt deleted file mode 100644 index c2774318e..000000000 --- a/kernel/evidence/501/red-repair-only.txt +++ /dev/null @@ -1,64 +0,0 @@ -# MUTATION (repair only): recovery.rs `if false && *wait_id == park_placeholder_wait_id(..)`; machine.rs park branch intact. Diff applied: -51c51 -< if *wait_id == park_placeholder_wait_id(&spec.id, runtime.attempts) => ---- -> if false && *wait_id == park_placeholder_wait_id(&spec.id, runtime.attempts) => - -$ cd kernel && cargo test -p relayflowd-core --lib -- machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote - Compiling relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) - Finished `test` profile [unoptimized + debuginfo] target(s) in 0.67s - Running unittests src/lib.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/relayflowd_core-82d87717b3255c51) - -running 1 test -test machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote ... FAILED - -failures: - ----- machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote stdout ---- - -thread 'machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote' (2313987) panicked at relayflowd-core/src/machine/tests.rs:843:9: -assertion `left == right` failed: worker-reported: recovery journals exactly the missing wait: [] - left: 0 - right: 1 -note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace - - -failures: - machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote - -test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 71 filtered out; finished in 0.00s - -error: test failed, to rerun pass `-p relayflowd-core --lib` -exit_code=101 - -$ cd kernel && cargo test -p relayflowd --test manual_recovery a_park_torn_between_its_two_appends_is_repaired_on_resume - Compiling relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) - Compiling relayflowd-journal v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-journal) - Compiling relayflowd v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd) - Finished `test` profile [unoptimized + debuginfo] target(s) in 1.17s - Running tests/manual_recovery.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/manual_recovery-6793123a1244aad2) - -running 1 test -test a_park_torn_between_its_two_appends_is_repaired_on_resume ... FAILED - -failures: - ----- a_park_torn_between_its_two_appends_is_repaired_on_resume stdout ---- - -thread 'a_park_torn_between_its_two_appends_is_repaired_on_resume' (2314845) panicked at relayflowd/tests/manual_recovery.rs:66:13: -injected crash between the park completion and its wait.human -note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace - -thread 'a_park_torn_between_its_two_appends_is_repaired_on_resume' (2314845) panicked at relayflowd/tests/manual_recovery.rs:144:5: -assertion `left == right` failed: exactly one wait.human: [] - left: 0 - right: 1 - - -failures: - a_park_torn_between_its_two_appends_is_repaired_on_resume - -test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 1 filtered out; finished in 0.01s - -error: test failed, to rerun pass `-p relayflowd --test manual_recovery` -exit_code=101 diff --git a/kernel/evidence/501/red.txt b/kernel/evidence/501/red.txt deleted file mode 100644 index 6228765bd..000000000 --- a/kernel/evidence/501/red.txt +++ /dev/null @@ -1,107 +0,0 @@ -# MUTATION: machine.rs `let manual_park = false && transport_failure ...` (worker-reported park disabled) and recovery.rs `if false && *wait_id == park_placeholder_wait_id(..)` (torn-park repair disabled). Diff applied: -397c397 -< let manual_park = transport_failure ---- -> let manual_park = false && transport_failure -51c51 -< if *wait_id == park_placeholder_wait_id(&spec.id, runtime.attempts) => ---- -> if false && *wait_id == park_placeholder_wait_id(&spec.id, runtime.attempts) => - -$ cd kernel && cargo test -p relayflowd-core --lib -- machine::tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote machine::tests::reset_recovery_still_retries_a_worker_reported_transport_loss - Compiling relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) - Finished `test` profile [unoptimized + debuginfo] target(s) in 1.26s - Running unittests src/lib.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/relayflowd_core-82d87717b3255c51) - -running 3 tests -test machine::tests::reset_recovery_still_retries_a_worker_reported_transport_loss ... ok -test machine::tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching ... FAILED -test machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote ... FAILED - -failures: - ----- machine::tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching stdout ---- - -thread 'machine::tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching' (2310933) panicked at relayflowd-core/src/machine/tests.rs:701:9: -assertion `left == right` failed: Crashed: a park is a completion plus a wait.human and nothing else (no retry timer): [Append(JournalEntry { seq: 0, segment_id: 0, entry_type: StepCompleted, run_id: "run", step_id: Some("agent"), attempt: Some(1), at_ms: 20, payload: Object {"budget": Object {"dollars": String("0"), "tokens_in": Number(0), "tokens_out": Number(0)}, "completed_by": String("worker"), "completionReason": String("crashed"), "disposition": String("retry"), "effects": Array [], "end_pins": Object {"streams": Array [], "workspace": Array [Object {"revision_id": String("rev-dirty"), "surface": String("repo")}]}, "next_attempt_at_ms": Number(20), "output": Null, "spend": Object {"dollars": Number(0), "tokens_input": Number(0), "tokens_output": Number(0), "wallclock_ms": Number(0)}, "trajectory_tail": Object {"transport": Object {"cause": String("signal_close")}}, "verification": Object {"detail": String("direct transport closed by signal"), "gate": String("execution"), "verdict": String("fail")}} }), Append(JournalEntry { seq: 0, segment_id: 0, entry_type: SleepUntil, run_id: "run", step_id: Some("agent"), attempt: Some(1), at_ms: 20, payload: Object {"reason": String("retry_backoff"), "wait_id": String("000000000MNSWFKX4NGBY4K6EJ"), "wake_at_ms": Number(20)} }), ArmTimer { at_ms: 20 }] - left: 3 - right: 2 -note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace - ----- machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote stdout ---- - -thread 'machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote' (2310934) panicked at relayflowd-core/src/machine/tests.rs:830:9: -assertion `left == right` failed: worker-reported: the torn prefix folds to the placeholder - left: Backoff { attempt: 1, wake_at_ms: 20 } - right: NeedsHuman { wait_id: "park-agent-1" } - - -failures: - machine::tests::manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching - machine::tests::recovery_journals_the_wait_human_a_torn_manual_park_never_wrote - -test result: FAILED. 1 passed; 2 failed; 0 ignored; 0 measured; 69 filtered out; finished in 0.00s - -error: test failed, to rerun pass `-p relayflowd-core --lib` -exit_code=101 - -$ cd kernel && cargo test -p relayflowd --test manual_recovery - Compiling relayflowd-core v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-core) - Compiling relayflowd-journal v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd-journal) - Compiling relayflowd v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd) - Finished `test` profile [unoptimized + debuginfo] target(s) in 1.13s - Running tests/manual_recovery.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/manual_recovery-6793123a1244aad2) - -running 2 tests -test a_park_torn_between_its_two_appends_is_repaired_on_resume ... FAILED -test manual_park_of_a_worker_reported_loss_survives_reopen_and_answers_to_a_human ... FAILED - -failures: - ----- a_park_torn_between_its_two_appends_is_repaired_on_resume stdout ---- - -thread 'a_park_torn_between_its_two_appends_is_repaired_on_resume' (2311731) panicked at relayflowd/tests/manual_recovery.rs:265:5: -the injected crash must fire after the park append -note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace - ----- manual_park_of_a_worker_reported_loss_survives_reopen_and_answers_to_a_human stdout ---- - -thread 'manual_park_of_a_worker_reported_loss_survives_reopen_and_answers_to_a_human' (2311732) panicked at relayflowd/tests/manual_recovery.rs:189:9: -assertion `left == right` failed: Crashed with max_transport_retries=1: a manual step must not be redispatched after a reported loss - left: 2 - right: 1 - - -failures: - a_park_torn_between_its_two_appends_is_repaired_on_resume - manual_park_of_a_worker_reported_loss_survives_reopen_and_answers_to_a_human - -test result: FAILED. 0 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s - -error: test failed, to rerun pass `-p relayflowd --test manual_recovery` -exit_code=101 - -$ cd kernel && cargo test -p relayflowd --test crash_resume manual_recovery - Compiling relayflowd v0.1.0 (/tmp/flows-fleet-501/kernel/relayflowd) - Finished `test` profile [unoptimized + debuginfo] target(s) in 0.38s - Running tests/crash_resume.rs (/tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/target/debug/deps/crash_resume-d2d102ec3e89b705) - -running 1 test -test manual_recovery::manual_recovery_parks_a_worker_reported_loss_across_daemon_restart_until_a_human_answers ... FAILED - -failures: - ----- manual_recovery::manual_recovery_parks_a_worker_reported_loss_across_daemon_restart_until_a_human_answers stdout ---- - -thread 'manual_recovery::manual_recovery_parks_a_worker_reported_loss_across_daemon_restart_until_a_human_answers' (2311978) panicked at relayflowd/tests/crash_resume/manual_recovery.rs:74:5: -crashed with max_transport_retries=1: a parked manual step must not be redispatched -note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace - - -failures: - manual_recovery::manual_recovery_parks_a_worker_reported_loss_across_daemon_restart_until_a_human_answers - -test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 40 filtered out; finished in 0.03s - -error: test failed, to rerun pass `-p relayflowd --test crash_resume` -exit_code=101 diff --git a/kernel/relayflowd-core/src/lib.rs b/kernel/relayflowd-core/src/lib.rs index c1cfd68e3..a7358a13d 100644 --- a/kernel/relayflowd-core/src/lib.rs +++ b/kernel/relayflowd-core/src/lib.rs @@ -31,7 +31,7 @@ pub use machine::{ pub use memory::{MemoryInjectedPayload, MemoryScope, MemorySpec}; pub use placement::{ExecutionMode, PlacementRequirements, RoutingDecision}; pub use spec::*; -pub use state::{RunState, StateError, StepRuntime, StepState, park_placeholder_wait_id}; +pub use state::{RunState, StateError, StepRuntime, StepState}; pub const JOURNAL_VERSION: u32 = 1; pub const PROTOCOL_VERSION: u32 = 0; diff --git a/kernel/relayflowd-core/src/machine.rs b/kernel/relayflowd-core/src/machine.rs index c36d37b04..b5a7d5ae7 100644 --- a/kernel/relayflowd-core/src/machine.rs +++ b/kernel/relayflowd-core/src/machine.rs @@ -567,4 +567,6 @@ mod parallel; #[cfg(test)] mod parallel_tests; #[cfg(test)] +mod recovery_tests; +#[cfg(test)] mod tests; diff --git a/kernel/relayflowd-core/src/machine/recovery_tests.rs b/kernel/relayflowd-core/src/machine/recovery_tests.rs new file mode 100644 index 000000000..1cd1bc1d5 --- /dev/null +++ b/kernel/relayflowd-core/src/machine/recovery_tests.rs @@ -0,0 +1,221 @@ +//! `manual` recovery for a dead attempt the WORKER reported (`step.complete` +//! with `crashed` / `lease_expired`), and repair of a park torn between its +//! two journal appends. Split from `tests.rs` so the general completion / +//! scheduling cases stay in one file and recovery has its own. + +use serde_json::json; + +use super::{ + tests::{AppendAction, agent_spec, started_agent, workspace_pins}, + *, +}; +use crate::state::{RunState, park_placeholder_wait_id}; + +/// Regression (#501 review): `RecoveryMode::Manual` was honoured only when the +/// KERNEL noticed a dead lease (`abandonment_actions`). A worker that reported +/// its own transport loss through `step.complete` went through +/// `completion_actions`, which retried every budget-eligible `crashed` or +/// `lease_expired` without reading the recovery mode — so the same dead +/// attempt parked or redispatched depending on who noticed it first. +/// Appendix A rule 4 makes `manual` a park, whichever path records the death. +#[test] +fn manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching() { + for reason in [CompletionReason::Crashed, CompletionReason::LeaseExpired] { + let spec = agent_spec("manual"); + let started = started_agent(&spec, workspace_pins("rev-clean")); + let running = RunState::fold("run", spec.clone(), std::slice::from_ref(&started)).unwrap(); + let runtime = &running.steps["agent"]; + let mut result = AttemptResult::successful(Value::Null, "worker"); + result.failure_reason = Some(reason); + result.failure_detail = Some("direct transport closed by signal".to_owned()); + result.trajectory_tail = Some(json!({"transport": {"cause": "signal_close"}})); + // The worker's own claim about where the workspace ended up. The diff + // the human is handed must be anchored on the journaled START pin, not + // on this. + result.end_pins = Some(workspace_pins("rev-dirty")); + let actions = completion_actions( + "run", + &spec.steps[0], + 1, + runtime.semantic_executions, + runtime.last_start_pins.as_ref(), + result, + 20, + ); + assert_eq!( + actions.len(), + 2, + "{reason:?}: a park is a completion plus a wait.human and nothing else \ + (no retry timer): {actions:?}" + ); + let completed: StepCompletedPayload = + serde_json::from_value(actions[0].clone().into_append().payload).unwrap(); + assert_eq!(completed.completion_reason, reason); + assert_eq!(completed.disposition, Disposition::Park); + assert_eq!(completed.next_attempt_at_ms, None); + // The worker's evidence still travels with the completion; parking is + // not a reason to discard the account of what went wrong. + assert_eq!( + completed.trajectory_tail, + Some(json!({"transport": {"cause": "signal_close"}})) + ); + assert_eq!( + completed.verification.as_ref().map(|record| record.verdict), + Some(crate::VerificationVerdict::Fail) + ); + let wait = actions[1].clone().into_append(); + assert_eq!(wait.entry_type, EntryType::WaitHuman); + assert_eq!(wait.step_id.as_deref(), Some("agent")); + assert_eq!(wait.attempt, Some(1)); + let human: crate::entry::WaitHumanPayload = + serde_json::from_value(wait.payload.clone()).unwrap(); + assert_eq!(human.diff_ref.as_deref(), Some("repo@rev-clean..current")); + assert_eq!( + human.options, + Some(vec!["retry".to_owned(), "cancel".to_owned()]) + ); + assert!( + human.prompt.contains("agent") && human.prompt.contains("run"), + "the prompt must name the step and run it parked: {}", + human.prompt + ); + + let mut entries = vec![started]; + entries.extend(actions.into_iter().map(Action::into_append)); + let parked = RunState::fold("run", spec, &entries).unwrap(); + assert!( + matches!(parked.steps["agent"].state, StepState::NeedsHuman { .. }), + "{reason:?}: the step must be parked on a human, got {:?}", + parked.steps["agent"].state + ); + assert_eq!( + parked.steps["agent"].semantic_executions, 0, + "a transport loss must not consume a semantic iteration" + ); + assert!( + next_actions(&parked, 30).is_empty(), + "{reason:?}: a parked manual step must never be redispatched" + ); + } +} + +/// The park above is specific to `manual`. The same worker-reported crash +/// under `reset` still takes the bounded transport retry, so the two modes +/// stay distinguishable at the completion. +#[test] +fn reset_recovery_still_retries_a_worker_reported_transport_loss() { + let spec = agent_spec("reset"); + let started = started_agent(&spec, workspace_pins("rev-clean")); + let running = RunState::fold("run", spec.clone(), std::slice::from_ref(&started)).unwrap(); + let runtime = &running.steps["agent"]; + let mut result = AttemptResult::successful(Value::Null, "worker"); + result.failure_reason = Some(CompletionReason::Crashed); + result.failure_detail = Some("direct transport closed by signal".to_owned()); + let actions = completion_actions( + "run", + &spec.steps[0], + 1, + runtime.semantic_executions, + runtime.last_start_pins.as_ref(), + result, + 20, + ); + let completed: StepCompletedPayload = + serde_json::from_value(actions[0].clone().into_append().payload).unwrap(); + assert_eq!(completed.disposition, Disposition::Retry); + assert!( + actions + .iter() + .all(|action| !matches!(action, Action::Append(entry) if entry.entry_type == EntryType::WaitHuman)), + "reset must not park: {actions:?}" + ); +} + +/// A `manual` park is two appends — `step.completed` (`park`), then the +/// `wait.human` a human answers — and each append is its own transaction. A +/// process death between them used to leave the step folded to the placeholder +/// wait id with nothing answerable: a permanent park. Recovery must journal +/// the missing wait exactly once, for BOTH producers of a park. +#[test] +fn recovery_journals_the_wait_human_a_torn_manual_park_never_wrote() { + let spec = agent_spec("manual"); + let started = started_agent(&spec, workspace_pins("rev-clean")); + let running = RunState::fold("run", spec.clone(), std::slice::from_ref(&started)).unwrap(); + let runtime = &running.steps["agent"]; + + let mut reported = AttemptResult::successful(Value::Null, "worker"); + reported.failure_reason = Some(CompletionReason::LeaseExpired); + reported.failure_detail = Some("direct transport closed by signal".to_owned()); + let worker_reported = completion_actions( + "run", + &spec.steps[0], + 1, + runtime.semantic_executions, + runtime.last_start_pins.as_ref(), + reported, + 20, + ) + .remove(0) + .into_append(); + let abandoned = abandonment_actions(&running, "agent", 1, CompletionReason::Crashed, 20) + .remove(0) + .into_append(); + + for (producer, park, reason) in [ + ( + "worker-reported", + worker_reported, + CompletionReason::LeaseExpired, + ), + ("abandoned lease", abandoned, CompletionReason::Crashed), + ] { + // Only the park landed; the wait.human did not. + let torn = RunState::fold("run", spec.clone(), &[started.clone(), park.clone()]).unwrap(); + assert_eq!( + torn.steps["agent"].state, + StepState::NeedsHuman { + wait_id: park_placeholder_wait_id("agent", 1) + }, + "{producer}: the torn prefix folds to the placeholder" + ); + assert!( + next_actions(&torn, 30).is_empty(), + "{producer}: a torn park must not dispatch" + ); + + let repaired = recovery_actions(&torn, 30); + assert_eq!( + repaired.len(), + 1, + "{producer}: recovery journals exactly the missing wait: {repaired:?}" + ); + let wait = repaired[0].clone().into_append(); + assert_eq!(wait.entry_type, EntryType::WaitHuman); + assert_eq!(wait.step_id.as_deref(), Some("agent")); + assert_eq!(wait.attempt, Some(1)); + let human: crate::entry::WaitHumanPayload = + serde_json::from_value(wait.payload.clone()).unwrap(); + assert_eq!(human.diff_ref.as_deref(), Some("repo@rev-clean..current")); + assert!( + human.prompt.contains(&format!("{reason:?}")), + "{producer}: the rebuilt prompt names the journaled reason: {}", + human.prompt + ); + + // Healed: the real wait id replaces the placeholder, and recovery has + // nothing further to add — the repair is idempotent across resumes. + let healed = RunState::fold("run", spec.clone(), &[started.clone(), park, wait]).unwrap(); + assert_eq!( + healed.steps["agent"].state, + StepState::NeedsHuman { + wait_id: human.wait_id + }, + "{producer}: the journaled wait names the parked step" + ); + assert!( + recovery_actions(&healed, 40).is_empty(), + "{producer}: a healed park must not be repaired twice" + ); + assert!(next_actions(&healed, 40).is_empty()); + } +} diff --git a/kernel/relayflowd-core/src/machine/tests.rs b/kernel/relayflowd-core/src/machine/tests.rs index 8db94883c..3bee96808 100644 --- a/kernel/relayflowd-core/src/machine/tests.rs +++ b/kernel/relayflowd-core/src/machine/tests.rs @@ -399,7 +399,7 @@ fn all_backing_off_steps_return_timers() { ); } -fn agent_spec(mode: &str) -> crate::RunSpec { +pub(super) fn agent_spec(mode: &str) -> crate::RunSpec { crate::RunSpec::parse(&json!({ "steps": [{ "id": "agent", @@ -414,7 +414,7 @@ fn agent_spec(mode: &str) -> crate::RunSpec { .unwrap() } -fn workspace_pins(revision_id: &str) -> Pins { +pub(super) fn workspace_pins(revision_id: &str) -> Pins { Pins { workspace: vec![crate::WorkspacePin { surface: "repo".to_owned(), @@ -424,7 +424,7 @@ fn workspace_pins(revision_id: &str) -> Pins { } } -fn started_agent(spec: &crate::RunSpec, pins: Pins) -> JournalEntry { +pub(super) fn started_agent(spec: &crate::RunSpec, pins: Pins) -> JournalEntry { let state = RunState::fold("run", spec.clone(), &[]).unwrap(); let Action::Append(mut started) = next_actions(&state, 10).remove(0) else { panic!("agent start must be journaled") @@ -667,216 +667,7 @@ fn manual_recovery_parks_needs_human_and_never_redispatches() { assert!(next_actions(&parked, 30).is_empty()); } -/// Regression (#501 review): `RecoveryMode::Manual` was honoured only when the -/// KERNEL noticed a dead lease (`abandonment_actions`). A worker that reported -/// its own transport loss through `step.complete` went through -/// `completion_actions`, which retried every budget-eligible `crashed` or -/// `lease_expired` without reading the recovery mode — so the same dead -/// attempt parked or redispatched depending on who noticed it first. -/// Appendix A rule 4 makes `manual` a park, whichever path records the death. -#[test] -fn manual_recovery_parks_a_worker_reported_transport_loss_instead_of_redispatching() { - for reason in [CompletionReason::Crashed, CompletionReason::LeaseExpired] { - let spec = agent_spec("manual"); - let started = started_agent(&spec, workspace_pins("rev-clean")); - let running = RunState::fold("run", spec.clone(), std::slice::from_ref(&started)).unwrap(); - let runtime = &running.steps["agent"]; - let mut result = AttemptResult::successful(Value::Null, "worker"); - result.failure_reason = Some(reason); - result.failure_detail = Some("direct transport closed by signal".to_owned()); - result.trajectory_tail = Some(json!({"transport": {"cause": "signal_close"}})); - // The worker's own claim about where the workspace ended up. The diff - // the human is handed must be anchored on the journaled START pin, not - // on this. - result.end_pins = Some(workspace_pins("rev-dirty")); - let actions = completion_actions( - "run", - &spec.steps[0], - 1, - runtime.semantic_executions, - runtime.last_start_pins.as_ref(), - result, - 20, - ); - assert_eq!( - actions.len(), - 2, - "{reason:?}: a park is a completion plus a wait.human and nothing else \ - (no retry timer): {actions:?}" - ); - let completed: StepCompletedPayload = - serde_json::from_value(actions[0].clone().into_append().payload).unwrap(); - assert_eq!(completed.completion_reason, reason); - assert_eq!(completed.disposition, Disposition::Park); - assert_eq!(completed.next_attempt_at_ms, None); - // The worker's evidence still travels with the completion; parking is - // not a reason to discard the account of what went wrong. - assert_eq!( - completed.trajectory_tail, - Some(json!({"transport": {"cause": "signal_close"}})) - ); - assert_eq!( - completed.verification.as_ref().map(|record| record.verdict), - Some(crate::VerificationVerdict::Fail) - ); - let wait = actions[1].clone().into_append(); - assert_eq!(wait.entry_type, EntryType::WaitHuman); - assert_eq!(wait.step_id.as_deref(), Some("agent")); - assert_eq!(wait.attempt, Some(1)); - let human: crate::entry::WaitHumanPayload = - serde_json::from_value(wait.payload.clone()).unwrap(); - assert_eq!(human.diff_ref.as_deref(), Some("repo@rev-clean..current")); - assert_eq!( - human.options, - Some(vec!["retry".to_owned(), "cancel".to_owned()]) - ); - assert!( - human.prompt.contains("agent") && human.prompt.contains("run"), - "the prompt must name the step and run it parked: {}", - human.prompt - ); - - let mut entries = vec![started]; - entries.extend(actions.into_iter().map(Action::into_append)); - let parked = RunState::fold("run", spec, &entries).unwrap(); - assert!( - matches!(parked.steps["agent"].state, StepState::NeedsHuman { .. }), - "{reason:?}: the step must be parked on a human, got {:?}", - parked.steps["agent"].state - ); - assert_eq!( - parked.steps["agent"].semantic_executions, 0, - "a transport loss must not consume a semantic iteration" - ); - assert!( - next_actions(&parked, 30).is_empty(), - "{reason:?}: a parked manual step must never be redispatched" - ); - } -} - -/// The park above is specific to `manual`. The same worker-reported crash -/// under `reset` still takes the bounded transport retry, so the two modes -/// stay distinguishable at the completion. -#[test] -fn reset_recovery_still_retries_a_worker_reported_transport_loss() { - let spec = agent_spec("reset"); - let started = started_agent(&spec, workspace_pins("rev-clean")); - let running = RunState::fold("run", spec.clone(), std::slice::from_ref(&started)).unwrap(); - let runtime = &running.steps["agent"]; - let mut result = AttemptResult::successful(Value::Null, "worker"); - result.failure_reason = Some(CompletionReason::Crashed); - result.failure_detail = Some("direct transport closed by signal".to_owned()); - let actions = completion_actions( - "run", - &spec.steps[0], - 1, - runtime.semantic_executions, - runtime.last_start_pins.as_ref(), - result, - 20, - ); - let completed: StepCompletedPayload = - serde_json::from_value(actions[0].clone().into_append().payload).unwrap(); - assert_eq!(completed.disposition, Disposition::Retry); - assert!( - actions - .iter() - .all(|action| !matches!(action, Action::Append(entry) if entry.entry_type == EntryType::WaitHuman)), - "reset must not park: {actions:?}" - ); -} - -/// A `manual` park is two appends — `step.completed` (`park`), then the -/// `wait.human` a human answers — and each append is its own transaction. A -/// process death between them used to leave the step folded to the placeholder -/// wait id with nothing answerable: a permanent park. Recovery must journal -/// the missing wait exactly once, for BOTH producers of a park. -#[test] -fn recovery_journals_the_wait_human_a_torn_manual_park_never_wrote() { - let spec = agent_spec("manual"); - let started = started_agent(&spec, workspace_pins("rev-clean")); - let running = RunState::fold("run", spec.clone(), std::slice::from_ref(&started)).unwrap(); - let runtime = &running.steps["agent"]; - - let mut reported = AttemptResult::successful(Value::Null, "worker"); - reported.failure_reason = Some(CompletionReason::LeaseExpired); - reported.failure_detail = Some("direct transport closed by signal".to_owned()); - let worker_reported = completion_actions( - "run", - &spec.steps[0], - 1, - runtime.semantic_executions, - runtime.last_start_pins.as_ref(), - reported, - 20, - ) - .remove(0) - .into_append(); - let abandoned = abandonment_actions(&running, "agent", 1, CompletionReason::Crashed, 20) - .remove(0) - .into_append(); - - for (producer, park, reason) in [ - ( - "worker-reported", - worker_reported, - CompletionReason::LeaseExpired, - ), - ("abandoned lease", abandoned, CompletionReason::Crashed), - ] { - // Only the park landed; the wait.human did not. - let torn = RunState::fold("run", spec.clone(), &[started.clone(), park.clone()]).unwrap(); - assert_eq!( - torn.steps["agent"].state, - StepState::NeedsHuman { - wait_id: crate::park_placeholder_wait_id("agent", 1) - }, - "{producer}: the torn prefix folds to the placeholder" - ); - assert!( - next_actions(&torn, 30).is_empty(), - "{producer}: a torn park must not dispatch" - ); - - let repaired = recovery_actions(&torn, 30); - assert_eq!( - repaired.len(), - 1, - "{producer}: recovery journals exactly the missing wait: {repaired:?}" - ); - let wait = repaired[0].clone().into_append(); - assert_eq!(wait.entry_type, EntryType::WaitHuman); - assert_eq!(wait.step_id.as_deref(), Some("agent")); - assert_eq!(wait.attempt, Some(1)); - let human: crate::entry::WaitHumanPayload = - serde_json::from_value(wait.payload.clone()).unwrap(); - assert_eq!(human.diff_ref.as_deref(), Some("repo@rev-clean..current")); - assert!( - human.prompt.contains(&format!("{reason:?}")), - "{producer}: the rebuilt prompt names the journaled reason: {}", - human.prompt - ); - - // Healed: the real wait id replaces the placeholder, and recovery has - // nothing further to add — the repair is idempotent across resumes. - let healed = RunState::fold("run", spec.clone(), &[started.clone(), park, wait]).unwrap(); - assert_eq!( - healed.steps["agent"].state, - StepState::NeedsHuman { - wait_id: human.wait_id - }, - "{producer}: the journaled wait names the parked step" - ); - assert!( - recovery_actions(&healed, 40).is_empty(), - "{producer}: a healed park must not be repaired twice" - ); - assert!(next_actions(&healed, 40).is_empty()); - } -} - -trait AppendAction { +pub(super) trait AppendAction { fn into_append(self) -> JournalEntry; } diff --git a/kernel/relayflowd-core/src/state.rs b/kernel/relayflowd-core/src/state.rs index af1061e60..3ba2fa543 100644 --- a/kernel/relayflowd-core/src/state.rs +++ b/kernel/relayflowd-core/src/state.rs @@ -88,7 +88,7 @@ pub struct RunState { /// answer. The two are separate journal appends, so a process death between /// them leaves the step folded to this placeholder with nothing answerable; /// recovery recognises it and journals the missing wait (`recovery_actions`). -pub fn park_placeholder_wait_id(step_id: &str, attempt: u32) -> String { +pub(crate) fn park_placeholder_wait_id(step_id: &str, attempt: u32) -> String { format!("park-{step_id}-{attempt}") } From 139690f754805ee961bb7f12e2c5d0f4f8b72f46 Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Sun, 20 Sep 2026 03:04:11 -0700 Subject: [PATCH 5/7] chore(evidence): literal journal-facts command for the codex live probe The journal-facts section of kernel/evidence/501/codex-live-probe.txt named the tool but not the command. It now carries the script, its literal invocation and the output verbatim (AGENTS.md evidence rule). Evidence only. Co-Authored-By: Claude Opus 5 (1M context) Session-Id: 144d3b43-0019-4de3-988a-7cd9ba4fc148 --- kernel/evidence/501/codex-live-probe.txt | 27 ++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/kernel/evidence/501/codex-live-probe.txt b/kernel/evidence/501/codex-live-probe.txt index 2ed374ac8..5085ce5f8 100644 --- a/kernel/evidence/501/codex-live-probe.txt +++ b/kernel/evidence/501/codex-live-probe.txt @@ -75,7 +75,30 @@ $ node /tmp/flows-fleet-501/packages/sdk/dist/cli.js logs --json --data-dir /tmp jq: parse error: Invalid numeric literal at line 1, column 8 exit_code=5 -# Journal facts (read directly from /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/codex-probe/data/runs/01M2Z44G4WB8BPC6JSMXT3NGZB.sqlite3 with python3 sqlite3; 'flows logs' takes no --data-dir): +# Journal facts, read directly from the run's SQLite journal ('flows logs' takes no --data-dir). Script used, then the literal invocation and its output: +$ cat /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/journal-facts.py +import sqlite3, sys, json +db = sys.argv[1] +cur = sqlite3.connect(db).cursor() +cols = [r[1] for r in cur.execute("pragma table_info(entries)")] +for row in cur.execute("select * from entries order by seq"): + e = dict(zip(cols, row)) + t = e["entry_type"] + if t not in ("step.attempt.started", "step.completed"): + continue + p = json.loads(e["payload"]) + if t == "step.attempt.started": + keep = ("step_type", "executor", "max_iterations", "max_transport_retries", "recovery_mode") + print(t, "attempt", e["attempt"], json.dumps({k: p.get(k) for k in keep})) + else: + out = p.get("output"); tail = p.get("trajectory_tail") or {} + print(t, "attempt", e["attempt"], json.dumps({ + "completionReason": p.get("completionReason"), "disposition": p.get("disposition"), + "verification": p.get("verification"), + "output": out if not isinstance(out, dict) else {k: out.get(k) for k in ("exit_code", "stdout_tail")}, + "transport": tail.get("transport") if isinstance(tail, dict) else None, + "spend": p.get("spend")}, indent=1)) +$ python3 /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/journal-facts.py /tmp/claude-1000/-tmp-flows-fleet-501/144d3b43-0019-4de3-988a-7cd9ba4fc148/scratchpad/codex-probe/data/runs/01M2Z44G4WB8BPC6JSMXT3NGZB.sqlite3 step.attempt.started attempt 1 {"step_type": "agent", "executor": "local-agent-98587efa-bcbb-4c24-8869-26d03ac86eb2", "max_iterations": 1, "max_transport_retries": 0, "recovery_mode": "reset"} step.completed attempt 1 { "completionReason": "success", @@ -97,7 +120,6 @@ step.completed attempt 1 { "signal": null, "stderr_tail": "Reading additional input from stdin...\n" }, - "transcript_model": null, "spend": { "dollars": 0, "dollars_unmetered": true, @@ -106,6 +128,7 @@ step.completed attempt 1 { "wallclock_ms": 3282 } } +exit_code=0 # Model: no model declared on the step and the codex adapter has no default (undefined below), so `codex exec` ran with NO --model flag: the model is whatever the installed Codex CLI's own configuration selects. The kernel journal does not record it for the direct transport. $ node -e "import('./dist/cli-adapter.js').then(m=>console.log(m.resolveCliModel('codex')))" From b26697909aaa6831a820c6f718c1648e46282e1b Mon Sep 17 00:00:00 2001 From: khaliqgant Date: Tue, 22 Sep 2026 23:25:28 -0700 Subject: [PATCH 6/7] merge(examples): keep the reviewed software-factory base source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Main's hosted-capability isolation pins software-factory.flow.ts by sha256; only the reviewed bytes are admitted as the extension base. Revert the branch's transportRetries additions there — adopting them requires a re-reviewed pin, which is a separate product decision. The transport-retry feature itself is unaffected. --- examples/software-factory/software-factory.flow.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/examples/software-factory/software-factory.flow.ts b/examples/software-factory/software-factory.flow.ts index fd229f3c5..fe8b61a4d 100644 --- a/examples/software-factory/software-factory.flow.ts +++ b/examples/software-factory/software-factory.flow.ts @@ -113,10 +113,6 @@ export default flow("software-factory", { await f.agent("implementer", { cli: "claude", - // Continue from the journaled dirty workspace only for a classified - // transport loss; an ordinary nonzero CLI exit remains terminal. - transportRetries: 1, - recoveryMode: "inspect", task: `Implement this ticket in the current repository, on the current branch, with regression tests. Commit as you go.\n` + `Write a PR description to ${WORK}/summary.md (what changed, how it was verified). Do not touch ${WORK}/ otherwise.\n\nTicket:\n${ticket}`, }).gate({ type: "subprocess_gate", command: `test -s ${WORK}/summary.md` }); @@ -127,8 +123,6 @@ export default flow("software-factory", { // what it finds; it must end with an explicit verdict file, not prose. await f.agent("adversary", { cli: "claude", - transportRetries: 1, - recoveryMode: "inspect", task: `Review the diff against the base branch as an adversary: find bugs, missing tests, unsafe defaults, and scope creep. ` + `Fix what is mechanical and re-run the tests. Write ${WORK}/review.md with your findings, then write ${WORK}/review.passed ` + `ONLY if the change is ready for a human to merge; otherwise write ${WORK}/review.blocked with the blocking findings.`, From ce2d4901c66c5dc5f74625d0545943ce6907d753 Mon Sep 17 00:00:00 2001 From: Relayflow Lead Date: Thu, 24 Sep 2026 23:12:11 -0700 Subject: [PATCH 7/7] fix(sdk): require a live drive at agent spawn --- packages/sdk/src/pty-sidechannel.ts | 25 ++++++------ packages/sdk/tests/pty-sidechannel.test.ts | 44 ++++++++++++++++++++++ 2 files changed, 55 insertions(+), 14 deletions(-) diff --git a/packages/sdk/src/pty-sidechannel.ts b/packages/sdk/src/pty-sidechannel.ts index 8934db609..abd694f62 100644 --- a/packages/sdk/src/pty-sidechannel.ts +++ b/packages/sdk/src/pty-sidechannel.ts @@ -31,10 +31,8 @@ export async function openSidechannel( canDrive: () => boolean = () => true, ) { const peers = new Map(); + const drivePeers = new Set(); let closed = false; - let driveConnected = false; - let announceDrive!: () => void; - const driveConnection = new Promise(resolve => { announceDrive = resolve; }); const server = createServer(socket => { if (peers.size >= 16) { socket.destroy(); return; } peers.set(socket, false); @@ -42,7 +40,10 @@ export async function openSidechannel( let mode: string | undefined; socket.setTimeout(2_000, () => socket.destroy()); socket.on('error', () => socket.destroy()); - socket.on('close', () => peers.delete(socket)); + socket.on('close', () => { + peers.delete(socket); + drivePeers.delete(socket); + }); socket.on('data', (bytes: Buffer) => { if (mode === undefined) { hello = Buffer.concat([hello, bytes]); @@ -56,8 +57,7 @@ export async function openSidechannel( peers.set(socket, true); // Passthrough is a passive raw-byte view in this initial slice. if (mode === 'drive') { - driveConnected = true; - announceDrive(); + drivePeers.add(socket); context.onDrive(); } bytes = hello.subarray(end + 1); @@ -106,14 +106,11 @@ export async function openSidechannel( * view and passthrough peers never change the child's stdin contract. */ async waitForDrive(timeoutMs: number): Promise { - if (driveConnected) return true; - let timer: NodeJS.Timeout | undefined; - await Promise.race([ - driveConnection, - new Promise(resolve => { timer = setTimeout(resolve, timeoutMs); }), - ]); - if (timer !== undefined) clearTimeout(timer); - return driveConnected; + // Enrollment is a live state, not a sticky historical event. Hold the + // whole bounded window so a peer that greets and then disconnects before + // spawn cannot leave the child with piped stdin and nobody to close it. + if (timeoutMs > 0) await new Promise(resolve => setTimeout(resolve, timeoutMs)); + return drivePeers.size > 0; }, close() { if (closed) return; diff --git a/packages/sdk/tests/pty-sidechannel.test.ts b/packages/sdk/tests/pty-sidechannel.test.ts index 36a3b65bb..ec20e1f91 100644 --- a/packages/sdk/tests/pty-sidechannel.test.ts +++ b/packages/sdk/tests/pty-sidechannel.test.ts @@ -1,4 +1,5 @@ import { EventEmitter } from 'node:events'; +import { spawn } from 'node:child_process'; import { createHash } from 'node:crypto'; import { mkdtempSync, writeFileSync, rmSync, existsSync } from 'node:fs'; import { connect, type Socket } from 'node:net'; @@ -125,6 +126,49 @@ setTimeout(() => { }); }); +it('a drive that disconnects before spawn leaves an unattended CLI at EOF', async () => { + const dataDir = dir(); + const cli = join(dataDir, 'claude'); + writeFileSync(cli, `#!/usr/bin/env node +const watchdog = setTimeout(() => process.exit(91), 2000); +process.stdin.resume(); +process.stdin.on('end', () => { + clearTimeout(watchdog); + process.stdout.write('eof-after-drive-disconnect'); +}); +`, { mode: 0o755 }); + let path = ''; + let peer: Socket | undefined; + let greeted = false; + const channel = await openSidechannel({ + dataDir, runId: 'r', stepId: 's', attempt: 1, + onDrive: () => { greeted = true; }, + onReady: value => { path = value; }, + }, () => true); + expect(channel).toBeDefined(); + try { + await new Promise((resolve, reject) => { + peer = connect(path); + peer.once('error', reject); + peer.once('connect', () => peer!.end('HELLO drive\n')); + peer.once('close', resolve); + }); + // Let the server consume its matching close event before the spawn-time + // enrollment decision. The peer greeted successfully, but is not live. + await new Promise(resolve => setImmediate(resolve)); + expect(greeted).toBe(true); + const driven = await channel!.waitForDrive(0); + expect(driven).toBe(false); + + let stdout = ''; + const child = spawn(cli, [], { stdio: [driven ? 'pipe' : 'ignore', 'pipe', 'pipe'] }); + child.stdout.on('data', bytes => { stdout += bytes.toString(); }); + const exitCode = await new Promise(resolve => child.once('close', resolve)); + expect(exitCode).toBe(0); + expect(stdout).toBe('eof-after-drive-disconnect'); + } finally { peer?.destroy(); channel?.close(); } +}); + it('rejects drive after EOF without marking human intervention', async () => { const dataDir = dir(); const cli = join(dataDir, 'claude');