From d10f6055f0d6c7336a46593c304cecfbddf0ef9c Mon Sep 17 00:00:00 2001 From: Yash Date: Sat, 5 Sep 2026 12:12:00 +0000 Subject: [PATCH] feat(harness): coordinate project bootstrap [Agent Map 06/15] --- .changeset/bootstrap-coordinator.md | 5 + .../src/core/project-bootstrap.test.ts | 4274 +++++++++++++++++ .../harness/src/core/project-bootstrap.ts | 3277 +++++++++++++ 3 files changed, 7556 insertions(+) create mode 100644 .changeset/bootstrap-coordinator.md create mode 100644 packages/harness/src/core/project-bootstrap.test.ts create mode 100644 packages/harness/src/core/project-bootstrap.ts diff --git a/.changeset/bootstrap-coordinator.md b/.changeset/bootstrap-coordinator.md new file mode 100644 index 00000000..f41f8b94 --- /dev/null +++ b/.changeset/bootstrap-coordinator.md @@ -0,0 +1,5 @@ +--- +"@sapiom/harness": patch +--- + +Internal groundwork for automatic Agent Map bootstrap, including recovery, FIFO delivery, and shutdown handling. Recovery events describe committed state. No user-facing behavior changes in this release. diff --git a/packages/harness/src/core/project-bootstrap.test.ts b/packages/harness/src/core/project-bootstrap.test.ts new file mode 100644 index 00000000..85a37f8c --- /dev/null +++ b/packages/harness/src/core/project-bootstrap.test.ts @@ -0,0 +1,4274 @@ +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import { createHash } from "node:crypto"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { + ProjectAgentSession, + ProjectBootstrapLifecycleEvent, + ProjectBootstrapMetadata, + ProjectBootstrapState, +} from "../shared/agent-map.js"; +import type { AnalyticsEvent, HarnessSession } from "../shared/types.js"; +import type { SessionManager } from "./session-manager.js"; +import { + SessionBackgroundInputPreemptedError, + SessionInputGuardRejectedError, + SessionNotReadyError, + type SessionInputWriteLifecycle, + type TerminalInputContext, +} from "./session-manager.js"; +import { + ProjectBootstrapCoordinator as ProjectBootstrapCoordinatorImpl, + ProjectBootstrapCoordinatorClosedError, + ProjectBootstrapDispatchForbiddenError, + ProjectBootstrapInputCapacityError, + ProjectBootstrapRequestIdConflictError, + ProjectBootstrapRetryUnavailableError, + projectBootstrapPrompt, + type ProjectBootstrapCoordinatorOptions, +} from "./project-bootstrap.js"; + +const activeCoordinators = new Set(); +const TEST_RUNTIME_EPOCH = "runtime-epoch-test"; + +class ProjectBootstrapCoordinator extends ProjectBootstrapCoordinatorImpl { + constructor(options: ProjectBootstrapCoordinatorOptions) { + super(options); + activeCoordinators.add(this); + } + + override async close(): Promise { + try { + await super.close(); + } finally { + activeCoordinators.delete(this); + } + } + + override async register( + session: HarnessSession, + context: Parameters[1], + runtimeEpoch: string | null = TEST_RUNTIME_EPOCH, + ): Promise { + if (runtimeEpoch !== null) { + await super.transitionRuntimeEpoch(session, runtimeEpoch); + } + return super.register(session, context, runtimeEpoch); + } + + override onSessionStatus( + session: HarnessSession, + runtimeEpoch: string | null = TEST_RUNTIME_EPOCH, + ): Promise { + return super.onSessionStatus(session, runtimeEpoch); + } + + override decorateLocalEvent( + event: AnalyticsEvent, + runtimeEpoch = TEST_RUNTIME_EPOCH, + ): AnalyticsEvent { + return super.decorateLocalEvent(event, runtimeEpoch); + } + + override onEventPersisted( + event: AnalyticsEvent, + runtimeEpoch = TEST_RUNTIME_EPOCH, + ): Promise { + return super.onEventPersisted(event, runtimeEpoch); + } + + override onTerminalInput( + sessionId: string, + context: Partial = {}, + ): void { + // Production admits the epoch before publishing the PTY. A few unit tests + // intentionally exercise raw input before register(), so mirror that + // already-completed SessionManager transition in the test adapter. + const runtimeEpoch = context.runtimeEpoch ?? TEST_RUNTIME_EPOCH; + const epochs = ( + this as unknown as { runtimeEpochs: Map } + ).runtimeEpochs; + if (!epochs.has(sessionId)) epochs.set(sessionId, runtimeEpoch); + super.onTerminalInput(sessionId, { + runtimeEpoch, + blockingPrompt: context.blockingPrompt ?? false, + }); + } +} + +const PROJECT_ID = "project_00000000-0000-7000-8000-000000000001"; +const USER_ID = "user-1"; +const NOW = "2026-09-01T00:00:00.000Z"; + +interface SubmittedInput { + sessionId: string; + text: string; + submit: boolean | undefined; + background: boolean | undefined; +} + +interface DurableBootstrapState { + schemaVersion: number; + metadata: ProjectBootstrapMetadata; + inputs: Array<{ + id: string; + sessionId: string; + text: string; + acceptedAt: string; + }>; + dispatchingInputId: string | null; + retryCount: number; + emptyProject: boolean; + attempts: Array<{ + attemptId: string; + retryOrdinal: number; + status: "active" | "retired" | "completed"; + phase?: "claimed" | "dispatching" | "not-submitted" | "submitted"; + }>; + uncertainInputIds?: string[]; + uncertainInputs?: Array<{ + id: string; + sessionId: string; + text: string; + acceptedAt: string; + }>; + receipts?: Array<{ + requestId: string | null; + inputId: string; + status: "queued" | "submitted" | "uncertain" | "completed"; + acceptedAt: string; + payloadDigest: string; + }>; +} + +function analyticsEvent( + sessionId: string, + type: AnalyticsEvent["type"], + payload: Record, + eventId = `event-${type}`, +): AnalyticsEvent { + return { + eventId, + seq: 1, + ts: NOW, + userId: USER_ID, + tenantId: null, + machineId: "machine-1", + harnessSessionId: sessionId, + agentSessionId: "provider-conversation-1", + harness: "codex", + type, + payload, + }; +} + +function projectBootstrapInputDigestForTest(text: string): string { + return createHash("sha256") + .update(JSON.stringify({ schemaVersion: 1, submit: true, text })) + .digest("hex"); +} + +function projectSession( + id = "session-1", + bootstrap: ProjectBootstrapState = { status: "pending" }, +): HarnessSession { + const identity: ProjectAgentSession = { + projectId: PROJECT_ID, + sessionId: id, + userId: USER_ID, + }; + return { + id, + agentSessionId: "provider-conversation-1", + harness: "codex", + cwd: "/private/project", + title: "Plan Agents", + status: "running", + createdAt: NOW, + lastActiveAt: NOW, + exitCode: null, + boundWorkflowPath: null, + ready: true, + agentMapIdentity: identity, + projectBootstrap: { + projectId: identity.projectId, + userId: identity.userId, + targetSessionId: identity.sessionId, + bootstrap: structuredClone(bootstrap), + queuedInputIds: [], + }, + }; +} + +function stateFile(root: string, sessionId: string): string { + return path.join(root, sessionId, "input-queue.json"); +} + +async function readState( + root: string, + sessionId: string, +): Promise { + return JSON.parse( + await fs.readFile(stateFile(root, sessionId), "utf8"), + ) as DurableBootstrapState; +} + +async function writeState( + root: string, + sessionId: string, + state: DurableBootstrapState, +): Promise { + await fs.mkdir(path.dirname(stateFile(root, sessionId)), { recursive: true }); + await fs.writeFile(stateFile(root, sessionId), `${JSON.stringify(state)}\n`); +} + +async function flushCoordinator( + coordinator: ProjectBootstrapCoordinator, + key: string, +): Promise { + const writes = ( + coordinator as unknown as { writes: Map> } + ).writes; + const pending = writes.get(key); + if (pending) await pending.catch(() => {}); + await Promise.resolve(); +} + +describe("ProjectBootstrapCoordinator", () => { + let root: string; + let legacyRoot: string; + let session: HarnessSession; + let sessions: Map; + let submitted: SubmittedInput[]; + let manager: SessionManager; + + beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), "project-bootstrap-")); + legacyRoot = await fs.mkdtemp( + path.join(os.tmpdir(), "planner-greeting-legacy-"), + ); + session = projectSession(); + sessions = new Map([[session.id, session]]); + submitted = []; + manager = { + get: (id: string) => sessions.get(id), + setProjectBootstrapMetadata: async ( + id: string, + metadata: ProjectBootstrapMetadata, + ) => { + const target = sessions.get(id); + if (!target) throw new Error("session missing"); + target.projectBootstrap = structuredClone(metadata); + }, + submitInput: async ( + id: string, + text: string, + submit?: boolean, + canWrite?: () => boolean | Promise, + background?: boolean, + lifecycle?: SessionInputWriteLifecycle, + ) => { + if (canWrite && !(await canWrite())) return false; + await lifecycle?.beforeFirstWrite?.(); + if (lifecycle?.canWriteNow && !lifecycle.canWriteNow()) return false; + submitted.push({ sessionId: id, text, submit, background }); + return true; + }, + preemptBackgroundInput: () => false, + getRuntimeEpoch: (id: string) => + sessions.get(id)?.status === "running" ? TEST_RUNTIME_EPOCH : null, + } as unknown as SessionManager; + }); + + afterEach(async () => { + await Promise.all( + [...activeCoordinators].map((coordinator) => coordinator.close()), + ); + vi.useRealTimers(); + await fs.rm(root, { recursive: true, force: true }); + await fs.rm(legacyRoot, { recursive: true, force: true }); + }); + + it("keeps pending readiness and model-turn deadlines distinct", async () => { + vi.useFakeTimers(); + session.ready = false; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + readinessTimeoutMs: 100, + deliveryTimeoutMs: 1_000, + }); + + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + await vi.advanceTimersByTimeAsync(60); + await coordinator.register(session, { emptyProject: true, mode: "live" }); + await vi.advanceTimersByTimeAsync(41); + await flushCoordinator(coordinator, session.id); + + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: true, + errorCode: "session_not_ready", + }); + expect(submitted).toEqual([]); + + session = projectSession("session-turn-timeout"); + sessions.set(session.id, session); + const turnCoordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + readinessTimeoutMs: 50, + deliveryTimeoutMs: 200, + generateId: () => "attempt-turn-timeout", + }); + await turnCoordinator.register(session, { + emptyProject: true, + mode: "created", + }); + await vi.advanceTimersByTimeAsync(51); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "generating", + attemptId: "attempt-turn-timeout", + }); + await vi.advanceTimersByTimeAsync(150); + await flushCoordinator(turnCoordinator, session.id); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: false, + errorCode: "delivery_timeout", + }); + }); + + it("rechecks durable map content and skips a no-longer-empty project without a model turn", async () => { + const lifecycle: ProjectBootstrapLifecycleEvent[] = []; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + isMeaningfullyEmpty: async (projectId) => { + expect(projectId).toBe(PROJECT_ID); + return false; + }, + onEvent: (event) => { + lifecycle.push(event); + }, + }); + + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + + expect(submitted).toEqual([]); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "skipped", + reason: "map-not-empty", + }); + expect(lifecycle).toContainEqual( + expect.objectContaining({ + name: "project_bootstrap.skipped", + reason: "map-not-empty", + queueDepth: 0, + }), + ); + expect((await readState(root, session.id)).emptyProject).toBe(false); + }); + + it("correlates one unique evidence-first bootstrap and ignores duplicate readiness and completion signals", async () => { + const lifecycle: ProjectBootstrapLifecycleEvent[] = []; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => "attempt-unique-1", + deliveryTimeoutMs: 60_000, + onEvent: (event) => { + lifecycle.push(event); + }, + }); + + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + await coordinator.register(session, { emptyProject: true, mode: "live" }); + await coordinator.onSessionStatus(session); + await coordinator.onSessionStatus(session); + + expect(submitted).toHaveLength(1); + expect(submitted[0]).toMatchObject({ + sessionId: session.id, + submit: true, + background: true, + }); + expect(submitted[0]!.text).toContain("attempt-unique-1"); + + const local = coordinator.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { + prompt: submitted[0]!.text, + }), + ); + expect(local.payload).toMatchObject({ + projectBootstrapOrigin: "infrastructure", + projectBootstrapAttemptId: "attempt-unique-1", + }); + + await coordinator.onEventPersisted( + analyticsEvent( + session.id, + "turn.completed", + { assistantText: "Evidence-supported map seed complete." }, + "turn-bootstrap-1", + ), + ); + await coordinator.onEventPersisted( + analyticsEvent( + session.id, + "turn.completed", + { assistantText: "Duplicate completion." }, + "turn-bootstrap-duplicate", + ), + ); + + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "delivered", + messageId: "turn-bootstrap-1", + }); + expect( + lifecycle.filter((event) => event.name === "project_bootstrap.attempted"), + ).toHaveLength(1); + expect( + lifecycle.filter((event) => event.name === "project_bootstrap.delivered"), + ).toHaveLength(1); + }); + + it("retains attempt tombstones so a late timed-out turn cannot complete its retry", async () => { + vi.useFakeTimers(); + const ids = ["attempt-1", "attempt-2"]; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + deliveryTimeoutMs: 100, + }); + + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + const firstPrompt = submitted[0]!.text; + coordinator.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { prompt: firstPrompt }), + ); + await vi.advanceTimersByTimeAsync(101); + await flushCoordinator(coordinator, session.id); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: false, + errorCode: "delivery_timeout", + }); + + await expect(coordinator.retry(session.id)).rejects.toBeInstanceOf( + ProjectBootstrapRetryUnavailableError, + ); + await coordinator.onEventPersisted( + analyticsEvent(session.id, "turn.completed", { + assistantText: "Late output from attempt one.", + }), + ); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: true, + errorCode: "delivery_timeout", + }); + + await coordinator.retry(session.id); + const retryPrompt = submitted[1]!.text; + expect(retryPrompt).not.toBe(firstPrompt); + expect(retryPrompt).toContain("automatic retry 1 of 2"); + coordinator.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { prompt: retryPrompt }), + ); + + await coordinator.onEventPersisted( + analyticsEvent( + session.id, + "turn.completed", + { assistantText: "Retry map seed complete." }, + "turn-retry", + ), + ); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "delivered", + messageId: "turn-retry", + }); + const durable = await readState(root, session.id); + expect(durable.retryCount).toBe(1); + expect(durable.attempts).toEqual([ + { + attemptId: "attempt-1", + retryOrdinal: 0, + status: "retired", + phase: "submitted", + }, + { + attemptId: "attempt-2", + retryOrdinal: 1, + status: "completed", + phase: "submitted", + }, + ]); + await expect(coordinator.retry(session.id)).rejects.toBeInstanceOf( + ProjectBootstrapRetryUnavailableError, + ); + }); + + it("releases input after a bounded timed-out turn and never replays it after restart", async () => { + vi.useFakeTimers(); + const ids = ["attempt-before-process-restart", "input-after-restart"]; + const first = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + deliveryTimeoutMs: 100, + }); + await first.register(session, { emptyProject: true, mode: "created" }); + const bootstrapPrompt = submitted[0]!.text; + first.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { + prompt: bootstrapPrompt, + }), + ); + + await vi.advanceTimersByTimeAsync(101); + await flushCoordinator(first, session.id); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: false, + errorCode: "delivery_timeout", + }); + + await expect( + first.enqueue(session.id, "implement the durable request directly"), + ).resolves.toMatchObject({ queuedInputIds: [] }); + expect(submitted.map((entry) => entry.text)).toEqual([ + bootstrapPrompt, + "implement the durable request directly", + ]); + + // Restart must not re-submit either already accepted turn. + session.status = "exited"; + session.ready = false; + await first.onSessionStatus(session); + await first.close(); + session.status = "running"; + session.ready = true; + const restarted = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + }); + await restarted.register(session, { emptyProject: true, mode: "boot" }); + await restarted.register(session, { emptyProject: true, mode: "live" }); + await restarted.onSessionStatus(session); + + expect(submitted.map((entry) => entry.text)).toEqual([ + bootstrapPrompt, + "implement the durable request directly", + ]); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "skipped", + reason: "user-proceeded", + }); + expect(session.projectBootstrap?.queuedInputIds).toEqual([]); + expect((await readState(root, session.id)).inputs).toEqual([]); + }); + + it("recovers an ambiguous generating restart as a non-retryable tombstone without blindly submitting again", async () => { + const first = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => "attempt-before-restart", + deliveryTimeoutMs: 60_000, + }); + await first.register(session, { emptyProject: true, mode: "created" }); + expect(submitted).toHaveLength(1); + expect(session.projectBootstrap?.bootstrap.status).toBe("generating"); + + const lifecycle: ProjectBootstrapLifecycleEvent[] = []; + const restarted = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + onEvent: (event) => { + lifecycle.push(event); + }, + }); + await restarted.register(session, { emptyProject: true, mode: "boot" }); + + expect(submitted).toHaveLength(1); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: false, + errorCode: "delivery_timeout", + }); + await expect(restarted.retry(session.id)).rejects.toBeInstanceOf( + ProjectBootstrapRetryUnavailableError, + ); + expect((await readState(root, session.id)).attempts).toEqual([ + { + attemptId: "attempt-before-restart", + retryOrdinal: 0, + status: "retired", + phase: "submitted", + }, + ]); + expect(lifecycle).toContainEqual( + expect.objectContaining({ + name: "project_bootstrap.recovered", + sessionId: session.id, + }), + ); + }); + + it.each(["claimed", "not-submitted"] as const)( + "publishes the committed retryable recovery for a %s attempt restored without a ready runtime", + async (phase) => { + session.ready = false; + session.status = "exited"; + session.projectBootstrap!.bootstrap = { + status: "generating", + attemptId: "attempt-unsubmitted-before-crash", + }; + await writeState(root, session.id, { + schemaVersion: 3, + metadata: structuredClone(session.projectBootstrap!), + inputs: [], + dispatchingInputId: null, + retryCount: 0, + emptyProject: true, + attempts: [ + { + attemptId: "attempt-unsubmitted-before-crash", + retryOrdinal: 0, + status: "active", + phase, + }, + ], + receipts: [], + }); + const lifecycle: ProjectBootstrapLifecycleEvent[] = []; + const eventsBeforeCommit: ProjectBootstrapLifecycleEvent[][] = []; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + onEvent: (event) => { lifecycle.push(event); }, + writeState: async (file, state) => { + eventsBeforeCommit.push([...lifecycle]); + await fs.writeFile(file, JSON.stringify(state)); + }, + }); + + await coordinator.register( + session, { emptyProject: true, mode: "boot" }, null, + ); + + expect(session.projectBootstrap!.bootstrap).toEqual({ + status: "failed", + retryable: true, + errorCode: "injection_failed", + }); + expect((await readState(root, session.id)).metadata.bootstrap).toEqual( + session.projectBootstrap!.bootstrap, + ); + expect( + lifecycle.filter((event) => event.name === "project_bootstrap.failed"), + ).toEqual([ + { + name: "project_bootstrap.failed", + projectId: PROJECT_ID, + sessionId: session.id, + attemptId: "attempt-unsubmitted-before-crash", + errorCode: "injection_failed", + retryable: true, + queueDepth: 0, + }, + ]); + expect(eventsBeforeCommit).toEqual([[]]); + expect(submitted).toEqual([]); + }, + ); + + it("publishes only the committed persistence failure when boot recovery cannot persist its classification", async () => { + session.ready = false; + session.status = "exited"; + session.projectBootstrap!.bootstrap = { + status: "generating", + attemptId: "attempt-before-storage-failure", + }; + await writeState(root, session.id, { + schemaVersion: 3, + metadata: structuredClone(session.projectBootstrap!), + inputs: [], + dispatchingInputId: null, + retryCount: 0, + emptyProject: true, + attempts: [ + { + attemptId: "attempt-before-storage-failure", + retryOrdinal: 0, + status: "active", + phase: "claimed", + }, + ], + receipts: [], + }); + const lifecycle: ProjectBootstrapLifecycleEvent[] = []; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + onEvent: (event) => { lifecycle.push(event); }, + writeState: vi.fn(async (file, state) => { + await fs.writeFile(file, JSON.stringify(state)); + }).mockRejectedValueOnce(new Error("storage unavailable")), + }); + + await expect(coordinator.register( + session, { emptyProject: true, mode: "boot" }, null, + )).rejects.toThrow("project bootstrap state persistence failed"); + + expect((await readState(root, session.id)).metadata.bootstrap).toEqual({ + status: "failed", + retryable: true, + errorCode: "persistence_failed", + }); + expect(lifecycle).toEqual([ + { + name: "project_bootstrap.failed", + projectId: PROJECT_ID, + sessionId: session.id, + errorCode: "persistence_failed", + retryable: true, + queueDepth: 0, + }, + ]); + expect(submitted).toEqual([]); + }); + + it("retries exactly once after restart when the durable attempt never reached its pre-write marker", async () => { + session.projectBootstrap!.bootstrap = { + status: "generating", + attemptId: "attempt-claimed-before-crash", + }; + await writeState(root, session.id, { + schemaVersion: 3, + metadata: structuredClone(session.projectBootstrap!), + inputs: [], + dispatchingInputId: null, + retryCount: 0, + emptyProject: true, + attempts: [ + { + attemptId: "attempt-claimed-before-crash", + retryOrdinal: 0, + status: "active", + phase: "claimed", + }, + ], + uncertainInputIds: [], + uncertainInputs: [], + receipts: [], + }); + + const restarted = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => "attempt-recovered-once", + deliveryTimeoutMs: 60_000, + }); + await restarted.register(session, { emptyProject: true, mode: "boot" }); + + expect(submitted).toHaveLength(1); + expect(submitted[0]?.text).toContain("attempt-recovered-once"); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "generating", + attemptId: "attempt-recovered-once", + }); + expect((await readState(root, session.id)).attempts).toEqual([ + { + attemptId: "attempt-claimed-before-crash", + retryOrdinal: 0, + status: "retired", + phase: "claimed", + }, + { + attemptId: "attempt-recovered-once", + retryOrdinal: 1, + status: "active", + phase: "submitted", + }, + ]); + + await restarted.register(session, { emptyProject: true, mode: "live" }); + expect(submitted).toHaveLength(1); + }); + + it("never replays a generating attempt whose pre-write dispatch marker is durable", async () => { + session.projectBootstrap!.bootstrap = { + status: "generating", + attemptId: "attempt-dispatch-uncertain", + }; + await writeState(root, session.id, { + schemaVersion: 3, + metadata: structuredClone(session.projectBootstrap!), + inputs: [], + dispatchingInputId: null, + retryCount: 0, + emptyProject: true, + attempts: [ + { + attemptId: "attempt-dispatch-uncertain", + retryOrdinal: 0, + status: "active", + phase: "dispatching", + }, + ], + uncertainInputIds: [], + uncertainInputs: [], + receipts: [], + }); + + const restarted = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + }); + await restarted.register(session, { emptyProject: true, mode: "boot" }); + + expect(submitted).toEqual([]); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: false, + errorCode: "delivery_timeout", + }); + expect((await readState(root, session.id)).attempts[0]).toMatchObject({ + attemptId: "attempt-dispatch-uncertain", + status: "retired", + phase: "dispatching", + }); + }); + + it("never replays a schema-3 bootstrap attempt already submitted before restart", async () => { + session.projectBootstrap!.bootstrap = { + status: "generating", + attemptId: "attempt-submitted-before-restart", + }; + await writeState(root, session.id, { + schemaVersion: 3, + metadata: structuredClone(session.projectBootstrap!), + inputs: [], + dispatchingInputId: null, + retryCount: 0, + emptyProject: true, + attempts: [ + { + attemptId: "attempt-submitted-before-restart", + retryOrdinal: 0, + status: "active", + phase: "submitted", + }, + ], + uncertainInputIds: [], + uncertainInputs: [], + receipts: [], + }); + + const restarted = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + }); + await restarted.register(session, { emptyProject: true, mode: "boot" }); + + expect(submitted).toEqual([]); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: false, + errorCode: "delivery_timeout", + }); + expect((await readState(root, session.id)).attempts[0]).toMatchObject({ + attemptId: "attempt-submitted-before-restart", + status: "retired", + phase: "submitted", + }); + await expect(restarted.retry(session.id)).rejects.toBeInstanceOf( + ProjectBootstrapRetryUnavailableError, + ); + }); + + it("keeps a post-Enter bootstrap state-write failure bounded before admitting user input", async () => { + vi.useFakeTimers(); + const ids = ["attempt-post-enter-write-failure", "input-after-bootstrap"]; + let failedSubmittedWrite = false; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + deliveryTimeoutMs: 100, + writeState: async (_file, value) => { + const durable = value as DurableBootstrapState; + if ( + !failedSubmittedWrite && + durable.metadata.bootstrap.status === "generating" && + durable.attempts.some( + (attempt) => + attempt.attemptId === "attempt-post-enter-write-failure" && + attempt.phase === "submitted", + ) + ) { + failedSubmittedWrite = true; + throw new Error("injected submitted bootstrap state failure"); + } + await writeState(root, session.id, durable); + }, + }); + + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + const bootstrapPrompt = submitted[0]!.text; + coordinator.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { + prompt: bootstrapPrompt, + }), + ); + const queued = await coordinator.enqueueWithReceipt( + session.id, + "build after the failed bootstrap state write", + "request-after-bootstrap-state-failure", + ); + + expect(failedSubmittedWrite).toBe(true); + expect(queued.receipt.status).toBe("queued"); + expect(submitted.map((entry) => entry.text)).toEqual([bootstrapPrompt]); + + await vi.advanceTimersByTimeAsync(101); + await flushCoordinator(coordinator, session.id); + + expect(submitted.map((entry) => entry.text)).toEqual([ + bootstrapPrompt, + "build after the failed bootstrap state write", + ]); + expect((await readState(root, session.id)).receipts).toContainEqual( + expect.objectContaining({ + inputId: "input-after-bootstrap", + status: "submitted", + }), + ); + + await coordinator.onEventPersisted( + analyticsEvent(session.id, "turn.completed", { + assistantText: "late bootstrap completion", + }), + ); + expect(coordinator.ownsInput(session.id)).toBe(true); + expect(submitted).toHaveLength(2); + }); + + it("refuses to retry a submitted bootstrap after its durable state write fails", async () => { + vi.useFakeTimers(); + let failedSubmittedWrite = false; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => "attempt-submitted-write-failure", + deliveryTimeoutMs: 100, + writeState: async (_file, value) => { + const durable = value as DurableBootstrapState; + if ( + !failedSubmittedWrite && + durable.metadata.bootstrap.status === "generating" && + durable.attempts.some( + (attempt) => + attempt.attemptId === "attempt-submitted-write-failure" && + attempt.phase === "submitted", + ) + ) { + failedSubmittedWrite = true; + throw new Error("injected submitted bootstrap state failure"); + } + await writeState(root, session.id, durable); + }, + }); + + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: false, + errorCode: "persistence_failed", + }); + expect(submitted).toHaveLength(1); + + await vi.advanceTimersByTimeAsync(101); + await flushCoordinator(coordinator, session.id); + + await expect(coordinator.retry(session.id)).rejects.toBeInstanceOf( + ProjectBootstrapRetryUnavailableError, + ); + expect(submitted).toHaveLength(1); + expect((await readState(root, session.id)).attempts.at(-1)).toMatchObject({ + attemptId: "attempt-submitted-write-failure", + status: "retired", + phase: "submitted", + }); + }); + + it("migrates a planner-era schema-1 FIFO in place without quarantine or input loss", async () => { + session.ready = false; + const legacyDirectory = path.join(legacyRoot, session.id); + await fs.mkdir(legacyDirectory, { recursive: true }); + await fs.writeFile( + path.join(legacyDirectory, "input-queue.json"), + `${JSON.stringify({ + schemaVersion: 1, + metadata: { + identity: { + projectId: PROJECT_ID, + userId: USER_ID, + sessionId: session.id, + role: "map-planner", + }, + greeting: { status: "delivered", messageId: "legacy-greeting" }, + queuedInputIds: ["legacy-input-1", "legacy-input-2"], + }, + inputs: [ + { + id: "legacy-input-1", + sessionId: session.id, + text: "first durable user request", + acceptedAt: NOW, + }, + { + id: "legacy-input-2", + sessionId: session.id, + text: "second durable user request", + acceptedAt: NOW, + }, + ], + dispatchingInputId: null, + retryCount: 0, + emptyProject: true, + // Schema 1 never defined this field. Migration must ignore it rather + // than accepting forged keyed receipt authority. + receipts: [ + { + requestId: "forged-legacy-key", + inputId: "legacy-input-1", + status: "queued", + acceptedAt: NOW, + payloadDigest: "f".repeat(64), + }, + ], + })}\n`, + ); + + const coordinator = new ProjectBootstrapCoordinator({ + root, + legacyStateRoot: legacyRoot, + sessionManager: manager, + }); + await coordinator.register(session, { emptyProject: true, mode: "boot" }); + + const migrated = await readState(root, session.id); + expect(migrated).toMatchObject({ + schemaVersion: 3, + metadata: { + projectId: PROJECT_ID, + userId: USER_ID, + targetSessionId: session.id, + bootstrap: { status: "delivered", messageId: "legacy-greeting" }, + queuedInputIds: ["legacy-input-1", "legacy-input-2"], + }, + }); + expect(migrated.metadata).not.toHaveProperty("identity"); + expect(migrated.metadata).not.toHaveProperty("greeting"); + expect(migrated.receipts).toHaveLength(2); + expect(migrated.receipts?.map((receipt) => receipt.requestId)).toEqual([ + null, + null, + ]); + expect( + new Set(migrated.receipts?.map((receipt) => receipt.inputId)).size, + ).toBe(2); + expect(await fs.readdir(legacyDirectory)).toEqual(["input-queue.json"]); + + session.ready = true; + await coordinator.onSessionStatus(session); + expect(submitted.map((entry) => entry.text)).toEqual([ + "first durable user request", + ]); + coordinator.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { + prompt: "first durable user request", + }), + ); + await coordinator.onEventPersisted( + analyticsEvent(session.id, "turn.completed", { + assistantText: "First request complete.", + }), + ); + expect(submitted.map((entry) => entry.text)).toEqual([ + "first durable user request", + "second durable user request", + ]); + expect(session.projectBootstrap?.queuedInputIds).toEqual([]); + expect((await readState(root, session.id)).inputs).toEqual([]); + }); + + it("lets API input preempt a pending bootstrap and preserves its FIFO order", async () => { + session.ready = false; + const ids = ["input-1", "input-2"]; + const lifecycle: ProjectBootstrapLifecycleEvent[] = []; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + onEvent: (event) => { + lifecycle.push(event); + }, + }); + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + + await coordinator.enqueue(session.id, "first user request"); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "skipped", + reason: "user-proceeded", + }); + expect(submitted).toEqual([]); + + session.ready = true; + await coordinator.onSessionStatus(session); + expect(submitted.map((entry) => entry.text)).toEqual([ + "first user request", + ]); + expect(session.projectBootstrap?.queuedInputIds).toEqual([]); + expect(coordinator.ownsInput(session.id)).toBe(true); + + await coordinator.enqueue(session.id, "second user request"); + await coordinator.onSessionStatus(session); + expect(submitted.map((entry) => entry.text)).toEqual([ + "first user request", + ]); + coordinator.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { + prompt: "first user request", + }), + ); + await coordinator.onEventPersisted( + analyticsEvent(session.id, "turn.completed", { + assistantText: "First request complete.", + }), + ); + expect(submitted.map((entry) => entry.text)).toEqual([ + "first user request", + "second user request", + ]); + expect(submitted.every((entry) => entry.background !== true)).toBe(true); + expect(lifecycle).toContainEqual( + expect.objectContaining({ + name: "project_bootstrap.preempted", + reason: "user-proceeded", + queueDepth: 1, + }), + ); + }); + + it("durably prioritizes API input that arrives between background text and Enter", async () => { + let announceStaged!: () => void; + const staged = new Promise((resolve) => { + announceStaged = resolve; + }); + let releaseStaged!: () => void; + const released = new Promise((resolve) => { + releaseStaged = resolve; + }); + manager.preemptBackgroundInput = () => { + releaseStaged(); + return true; + }; + manager.submitInput = async ( + id: string, + text: string, + submit?: boolean, + canWrite?: () => boolean | Promise, + background?: boolean, + lifecycle?: SessionInputWriteLifecycle, + ) => { + if (background) { + await lifecycle?.beforeFirstWrite?.(); + announceStaged(); + await released; + if (canWrite && !(await canWrite())) { + await lifecycle?.onNotSubmitted?.(); + throw new SessionInputGuardRejectedError(true); + } + } + submitted.push({ sessionId: id, text, submit, background }); + return true; + }; + const ids = ["attempt-staged", "input-priority"]; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + deliveryTimeoutMs: 60_000, + }); + + const registering = coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + await staged; + const enqueueing = coordinator.enqueue( + session.id, + "implement the requested change now", + ); + await Promise.all([registering, enqueueing]); + + expect(submitted).toEqual([ + { + sessionId: session.id, + text: "implement the requested change now", + submit: true, + background: false, + }, + ]); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "skipped", + reason: "user-proceeded", + }); + expect((await readState(root, session.id)).attempts).toEqual([ + { + attemptId: "attempt-staged", + retryOrdinal: 0, + status: "retired", + phase: "not-submitted", + }, + ]); + }); + + it("tombstones an in-flight bootstrap when durable API input arrives", async () => { + const ids = ["attempt-1", "input-1"]; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + deliveryTimeoutMs: 60_000, + }); + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + const bootstrapPrompt = submitted[0]!.text; + coordinator.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { + prompt: bootstrapPrompt, + }), + ); + + await coordinator.enqueue(session.id, "build this directly now"); + expect(submitted.map((entry) => entry.text)).toEqual([bootstrapPrompt]); + expect(session.projectBootstrap?.queuedInputIds).toEqual(["input-1"]); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "skipped", + reason: "user-proceeded", + }); + + await coordinator.onEventPersisted( + analyticsEvent(session.id, "turn.completed", { + assistantText: "Late bootstrap output.", + }), + ); + expect(submitted.map((entry) => entry.text)).toEqual([ + bootstrapPrompt, + "build this directly now", + ]); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "skipped", + reason: "user-proceeded", + }); + }); + + it("keeps durable API input queued through raw preemption and resumes it after the user turn", async () => { + session.ready = false; + let preemptOnce = true; + manager.submitInput = async ( + id: string, + text: string, + submit?: boolean, + canWrite?: () => boolean | Promise, + background?: boolean, + lifecycle?: SessionInputWriteLifecycle, + ) => { + if (preemptOnce) { + preemptOnce = false; + coordinator.onTerminalInput(id); + await lifecycle?.onNotSubmitted?.(); + throw new SessionBackgroundInputPreemptedError(true); + } + if (canWrite && !(await canWrite())) return false; + submitted.push({ sessionId: id, text, submit, background }); + return true; + }; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => "input-after-raw-turn", + }); + manager.setProjectBootstrapMetadata = async ( + id: string, + metadata: ProjectBootstrapMetadata, + ) => { + const target = sessions.get(id); + if (!target) throw new Error("session missing"); + target.projectBootstrap = structuredClone(metadata); + // Match production SessionManager: every durable projection emits a + // re-entrant status callback. The coordinator must keep the raw turn's + // ownership even when this callback is queued during persistence. + void coordinator.onSessionStatus(target); + }; + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + await coordinator.enqueue(session.id, "durable API request"); + + session.ready = true; + await coordinator.onSessionStatus(session); + expect(submitted).toEqual([]); + expect(session.projectBootstrap?.queuedInputIds).toEqual([ + "input-after-raw-turn", + ]); + expect((await readState(root, session.id)).dispatchingInputId).toBeNull(); + await flushCoordinator(coordinator, session.id); + expect(submitted).toEqual([]); + + coordinator.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { + prompt: "raw terminal request", + }), + ); + await coordinator.onEventPersisted( + analyticsEvent(session.id, "turn.completed", { + assistantText: "The user's raw terminal turn completed.", + }), + ); + expect(submitted.map((entry) => entry.text)).toEqual([ + "durable API request", + ]); + expect(session.projectBootstrap?.queuedInputIds).toEqual([]); + }); + + it("raw terminal input synchronously preempts bootstrap before or during dispatch", async () => { + const beforeRegistration = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + }); + beforeRegistration.onTerminalInput(session.id); + await beforeRegistration.register(session, { + emptyProject: true, + mode: "created", + }); + expect(submitted).toEqual([]); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "skipped", + reason: "user-proceeded", + }); + + session = projectSession("session-raw-in-flight"); + sessions.set(session.id, session); + const lifecycle: ProjectBootstrapLifecycleEvent[] = []; + const inFlight = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => "attempt-raw", + deliveryTimeoutMs: 60_000, + onEvent: (event) => { + lifecycle.push(event); + }, + }); + await inFlight.register(session, { emptyProject: true, mode: "created" }); + expect(submitted).toHaveLength(1); + inFlight.onTerminalInput(session.id); + inFlight.onTerminalInput(session.id); + await flushCoordinator(inFlight, session.id); + + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "skipped", + reason: "user-proceeded", + }); + expect( + lifecycle.filter((event) => event.name === "project_bootstrap.preempted"), + ).toHaveLength(1); + }); + + it("tombstones an orphaned dispatch without replay and lets the later FIFO progress", async () => { + session.projectBootstrap!.bootstrap = { + status: "delivered", + messageId: "bootstrap-turn", + }; + await writeState(root, session.id, { + schemaVersion: 2, + metadata: { + ...structuredClone(session.projectBootstrap!), + queuedInputIds: ["input-uncertain", "input-safe-next"], + }, + inputs: [ + { + id: "input-uncertain", + sessionId: session.id, + text: "possibly accepted already", + acceptedAt: NOW, + }, + { + id: "input-safe-next", + sessionId: session.id, + text: "definitely send next", + acceptedAt: NOW, + }, + ], + dispatchingInputId: "input-uncertain", + retryCount: 0, + emptyProject: true, + attempts: [], + }); + const lifecycle: ProjectBootstrapLifecycleEvent[] = []; + const restarted = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + onEvent: (event) => { + lifecycle.push(event); + }, + }); + + await restarted.register(session, { emptyProject: true, mode: "boot" }); + await restarted.register(session, { emptyProject: true, mode: "live" }); + + expect(submitted.map((entry) => entry.text)).toEqual([ + "definitely send next", + ]); + expect(lifecycle).toContainEqual({ + name: "project_bootstrap.input_delivery_uncertain", + projectId: PROJECT_ID, + sessionId: session.id, + inputId: "input-uncertain", + errorCode: "delivery_uncertain", + queueDepth: 2, + }); + expect(JSON.stringify(lifecycle)).not.toContain( + "possibly accepted already", + ); + const persisted = await readState(root, session.id); + expect(persisted.inputs).toEqual([]); + expect(persisted.uncertainInputIds).toEqual(["input-uncertain"]); + expect(persisted.uncertainInputs).toEqual([ + expect.objectContaining({ + id: "input-uncertain", + text: "possibly accepted already", + }), + ]); + expect( + lifecycle.filter( + (event) => event.name === "project_bootstrap.input_delivery_uncertain", + ), + ).toHaveLength(1); + }); + + it("classifies false dispatch, provider rejection, empty model output, and session exit", async () => { + manager.submitInput = async () => false; + const falseDispatch = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + }); + await falseDispatch.register(session, { + emptyProject: true, + mode: "created", + }); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: false, + errorCode: "scope_unavailable", + }); + + session = projectSession("session-injection-failure"); + sessions.set(session.id, session); + manager.submitInput = async ( + _id, + _text, + _submit, + _canWrite, + _background, + lifecycle, + ) => { + await lifecycle?.beforeFirstWrite?.(); + await lifecycle?.onNotSubmitted?.(); + throw new Error("raw provider failure"); + }; + const injectionFailure = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + }); + await injectionFailure.register(session, { + emptyProject: true, + mode: "created", + }); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: true, + errorCode: "injection_failed", + }); + + session = projectSession("session-empty-turn"); + sessions.set(session.id, session); + submitted = []; + manager.submitInput = async (id: string, text: string) => { + submitted.push({ sessionId: id, text, submit: true, background: true }); + return true; + }; + const emptyTurn = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => "attempt-empty-turn", + deliveryTimeoutMs: 60_000, + }); + await emptyTurn.register(session, { emptyProject: true, mode: "created" }); + emptyTurn.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { + prompt: submitted[0]!.text, + }), + ); + await emptyTurn.onEventPersisted( + analyticsEvent(session.id, "turn.completed", { assistantText: " " }), + ); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: true, + errorCode: "model_turn_failed", + }); + + session = projectSession("session-exited-pending"); + session.ready = false; + sessions.set(session.id, session); + const exited = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + readinessTimeoutMs: 60_000, + }); + await exited.register(session, { emptyProject: true, mode: "created" }); + session.status = "exited"; + await exited.onSessionStatus(session); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: false, + errorCode: "session_exited", + }); + + session = projectSession("session-already-exited"); + session.ready = false; + session.status = "exited"; + sessions.set(session.id, session); + const alreadyExited = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + readinessTimeoutMs: 60_000, + }); + await alreadyExited.register(session, { + emptyProject: true, + mode: "boot", + }); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: false, + errorCode: "session_exited", + }); + expect( + (alreadyExited as unknown as { timers: Map }).timers + .size, + ).toBe(0); + }); + + it("allows one explicit retry after SessionNotReadyError proves Enter did not cross", async () => { + const ids = ["attempt-not-ready", "attempt-ready-retry"]; + let submitCalls = 0; + manager.submitInput = async ( + id: string, + text: string, + submit?: boolean, + canWrite?: () => boolean | Promise, + background?: boolean, + lifecycle?: SessionInputWriteLifecycle, + ) => { + submitCalls += 1; + if (submitCalls === 1) throw new SessionNotReadyError(id); + if (canWrite && !(await canWrite())) return false; + await lifecycle?.beforeFirstWrite?.(); + if (lifecycle?.canWriteNow && !lifecycle.canWriteNow()) return false; + submitted.push({ sessionId: id, text, submit, background }); + return true; + }; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + deliveryTimeoutMs: 60_000, + }); + + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: true, + errorCode: "session_not_ready", + }); + expect((await readState(root, session.id)).attempts.at(-1)).toMatchObject({ + attemptId: "attempt-not-ready", + status: "retired", + phase: "claimed", + }); + + await coordinator.retry(session.id); + + expect(submitted).toHaveLength(1); + expect(submitted[0]?.text).toContain("attempt-ready-retry"); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "generating", + attemptId: "attempt-ready-retry", + }); + }); + + it("allows one explicit retry after a pre-Enter provider rejection", async () => { + const ids = ["attempt-provider-rejected", "attempt-provider-retry"]; + let submitCalls = 0; + manager.submitInput = async ( + id: string, + text: string, + submit?: boolean, + canWrite?: () => boolean | Promise, + background?: boolean, + lifecycle?: SessionInputWriteLifecycle, + ) => { + submitCalls += 1; + if (canWrite && !(await canWrite())) return false; + await lifecycle?.beforeFirstWrite?.(); + if (submitCalls === 1) { + await lifecycle?.onNotSubmitted?.(); + throw new Error("provider rejected before the first PTY byte"); + } + if (lifecycle?.canWriteNow && !lifecycle.canWriteNow()) return false; + submitted.push({ sessionId: id, text, submit, background }); + return true; + }; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + deliveryTimeoutMs: 60_000, + }); + + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: true, + errorCode: "injection_failed", + }); + expect((await readState(root, session.id)).attempts.at(-1)).toMatchObject({ + attemptId: "attempt-provider-rejected", + status: "retired", + phase: "not-submitted", + }); + + await coordinator.close(); + const restarted = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + deliveryTimeoutMs: 60_000, + }); + await restarted.register(session, { emptyProject: true, mode: "boot" }); + await restarted.retry(session.id); + + expect(submitted).toHaveLength(1); + expect(submitted[0]?.text).toContain("attempt-provider-retry"); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "generating", + attemptId: "attempt-provider-retry", + }); + }); + + it("fences an ambiguous provider rejection without durable not-submitted proof", async () => { + vi.useFakeTimers(); + manager.submitInput = async ( + _id, + _text, + _submit, + _canWrite, + _background, + lifecycle, + ) => { + await lifecycle?.beforeFirstWrite?.(); + throw new Error("provider rejected at the Enter boundary"); + }; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => "attempt-ambiguous-provider-rejection", + }); + + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "generating", + attemptId: "attempt-ambiguous-provider-rejection", + }); + expect( + ( + coordinator as unknown as { + activeTurnTimers: Map; + runtimeEpochs: Map; + } + ).activeTurnTimers.size, + ).toBe(1); + expect( + ( + coordinator as unknown as { + runtimeEpochs: Map; + } + ).runtimeEpochs.get(session.id), + ).toBe(TEST_RUNTIME_EPOCH); + await vi.advanceTimersByTimeAsync(300_000); + await flushCoordinator(coordinator, session.id); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: false, + errorCode: "delivery_timeout", + }); + expect((await readState(root, session.id)).attempts.at(-1)).toMatchObject({ + attemptId: "attempt-ambiguous-provider-rejection", + status: "retired", + phase: "dispatching", + }); + await expect(coordinator.retry(session.id)).rejects.toBeInstanceOf( + ProjectBootstrapRetryUnavailableError, + ); + expect(submitted).toEqual([]); + }); + + it("allows retry after a current-schema persistence failure before attempt allocation", async () => { + let rejectFirstWrite = true; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => "attempt-after-persistence-recovery", + deliveryTimeoutMs: 60_000, + writeState: async (_file, value) => { + if (rejectFirstWrite) { + rejectFirstWrite = false; + throw new Error("injected pre-attempt persistence failure"); + } + await writeState(root, session.id, value as DurableBootstrapState); + }, + }); + + await expect( + coordinator.register(session, { emptyProject: true, mode: "created" }), + ).rejects.toThrow("project bootstrap state persistence failed"); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: true, + errorCode: "persistence_failed", + }); + expect((await readState(root, session.id)).attempts).toEqual([]); + + await coordinator.retry(session.id); + + expect(submitted).toHaveLength(1); + expect(submitted[0]?.text).toContain("attempt-after-persistence-recovery"); + }); + + it("allows retry only after a correlated submitted turn reports an empty model result", async () => { + const ids = ["attempt-empty-model", "attempt-after-empty-model"]; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + deliveryTimeoutMs: 60_000, + }); + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + coordinator.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { + prompt: submitted[0]!.text, + }), + ); + await coordinator.onEventPersisted( + analyticsEvent(session.id, "turn.completed", { assistantText: " " }), + ); + + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: true, + errorCode: "model_turn_failed", + }); + expect((await readState(root, session.id)).attempts.at(-1)).toMatchObject({ + attemptId: "attempt-empty-model", + status: "retired", + phase: "submitted", + }); + + await coordinator.retry(session.id); + + expect(submitted).toHaveLength(2); + expect(submitted[1]?.text).toContain("attempt-after-empty-model"); + }); + + it.each(["injection_failed", "persistence_failed"] as const)( + "does not trust legacy phase-less %s retry metadata", + async (errorCode) => { + session.projectBootstrap!.bootstrap = { + status: "failed", + retryable: true, + errorCode, + }; + await writeState(root, session.id, { + schemaVersion: 2, + metadata: structuredClone(session.projectBootstrap!), + inputs: [], + dispatchingInputId: null, + retryCount: 0, + emptyProject: true, + attempts: [], + }); + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + }); + + await coordinator.register(session, { + emptyProject: true, + mode: "boot", + }); + + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: false, + errorCode, + }); + await expect(coordinator.retry(session.id)).rejects.toBeInstanceOf( + ProjectBootstrapRetryUnavailableError, + ); + expect(submitted).toEqual([]); + }, + ); + + it("durably bounds persistence failures without exposing storage content", async () => { + const lifecycle: ProjectBootstrapLifecycleEvent[] = []; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + writeState: async () => { + throw new Error("/private/customer/path provider-secret"); + }, + onEvent: (event) => { + lifecycle.push(event); + }, + }); + + await expect( + coordinator.register(session, { emptyProject: true, mode: "created" }), + ).rejects.toThrow("project bootstrap state persistence failed"); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: true, + errorCode: "persistence_failed", + }); + const serialized = JSON.stringify(lifecycle); + expect(serialized).toContain("project_bootstrap.failed"); + expect(serialized).not.toContain("private/customer"); + expect(serialized).not.toContain("provider-secret"); + }); + + it("revalidates dispatch authority before bootstrap and queued user input", async () => { + session.ready = false; + let authorized = true; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + canDispatch: async () => authorized, + readinessTimeoutMs: 60_000, + }); + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + await coordinator.enqueue(session.id, "durable user request"); + authorized = false; + session.ready = true; + await coordinator.onSessionStatus(session); + + expect(submitted).toEqual([]); + expect(session.projectBootstrap?.queuedInputIds).toHaveLength(1); + await expect( + coordinator.enqueue(session.id, "foreign follow-up"), + ).rejects.toBeInstanceOf(ProjectBootstrapDispatchForbiddenError); + }); + + it("acknowledges a durable enqueue when authority changes after its commit", async () => { + session.ready = false; + let authorized = true; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => "input-committed-before-rebind", + canDispatch: () => authorized, + readinessTimeoutMs: 60_000, + }); + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + manager.setProjectBootstrapMetadata = async ( + id: string, + metadata: ProjectBootstrapMetadata, + ) => { + const target = sessions.get(id); + if (!target) throw new Error("session missing"); + target.projectBootstrap = structuredClone(metadata); + if (metadata.queuedInputIds.length > 0) authorized = false; + }; + session.ready = true; + + await expect( + coordinator.enqueue(session.id, "durably accepted before rebind"), + ).resolves.toMatchObject({ + queuedInputIds: ["input-committed-before-rebind"], + }); + expect(submitted).toEqual([]); + expect((await readState(root, session.id)).inputs).toEqual([ + expect.objectContaining({ + id: "input-committed-before-rebind", + text: "durably accepted before rebind", + }), + ]); + }); + + it("deduplicates only explicit durable request IDs and rejects changed payloads", async () => { + session.ready = false; + const ids = ["input-keyed", "input-unkeyed-1", "input-unkeyed-2"]; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + readinessTimeoutMs: 60_000, + }); + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + + const first = await coordinator.enqueueWithReceipt( + session.id, + "implement the scoped request", + "request-1", + ); + const replay = await coordinator.enqueueWithReceipt( + session.id, + "implement the scoped request", + "request-1", + ); + expect(replay).toEqual(first); + expect(first.receipt).toMatchObject({ + requestId: "request-1", + inputId: "input-keyed", + status: "queued", + }); + await expect( + coordinator.enqueueWithReceipt( + session.id, + "different payload", + "request-1", + ), + ).rejects.toBeInstanceOf(ProjectBootstrapRequestIdConflictError); + + await coordinator.enqueue(session.id, "same text without a key"); + await coordinator.enqueue(session.id, "same text without a key"); + const durable = await readState(root, session.id); + expect(durable.inputs.map((input) => input.id)).toEqual([ + "input-keyed", + "input-unkeyed-1", + "input-unkeyed-2", + ]); + expect(durable.receipts).toHaveLength(3); + }); + + it("bounds and compacts keyed and unkeyed receipt storage without persisting payload copies", async () => { + session.ready = false; + session.projectBootstrap!.bootstrap = { + status: "delivered", + messageId: "bootstrap-complete", + }; + const oldReceipts = Array.from({ length: 128 }, (_, index) => ({ + requestId: index % 2 === 0 ? null : `old-request-${index}`, + inputId: `old-input-${index}`, + status: "completed" as const, + acceptedAt: NOW, + payloadDigest: index.toString(16).padStart(64, "0"), + leakedText: "receipt-only secret", + })); + await writeState(root, session.id, { + schemaVersion: 3, + metadata: structuredClone(session.projectBootstrap!), + inputs: [], + dispatchingInputId: null, + retryCount: 0, + emptyProject: false, + attempts: [], + uncertainInputIds: [], + uncertainInputs: [], + receipts: oldReceipts, + }); + const lifecycle: ProjectBootstrapLifecycleEvent[] = []; + const ids = ["new-unkeyed-input", "new-keyed-input"]; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + onEvent: (event) => { + lifecycle.push(event); + }, + }); + await coordinator.register(session, { emptyProject: false, mode: "boot" }); + + await coordinator.enqueueWithReceipt(session.id, "first private payload"); + await coordinator.enqueueWithReceipt( + session.id, + "second private payload", + "new-keyed-request", + ); + + const durable = await readState(root, session.id); + expect(durable.receipts).toHaveLength(128); + expect(durable.receipts).not.toContainEqual( + expect.objectContaining({ inputId: "old-input-0" }), + ); + expect(durable.receipts).not.toContainEqual( + expect.objectContaining({ inputId: "old-input-2" }), + ); + expect(durable.receipts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + requestId: null, + inputId: "new-unkeyed-input", + status: "queued", + }), + expect.objectContaining({ + requestId: "new-keyed-request", + inputId: "new-keyed-input", + status: "queued", + }), + ]), + ); + for (const receipt of durable.receipts ?? []) { + expect(Object.keys(receipt).sort()).toEqual([ + "acceptedAt", + "inputId", + "payloadDigest", + "requestId", + "status", + ]); + expect(receipt.payloadDigest).toMatch(/^[0-9a-f]{64}$/); + } + expect(JSON.stringify(durable.receipts)).not.toContain( + "first private payload", + ); + expect(JSON.stringify(durable.receipts)).not.toContain( + "second private payload", + ); + expect(JSON.stringify(durable.receipts)).not.toContain( + "receipt-only secret", + ); + expect(JSON.stringify(lifecycle)).not.toContain("private payload"); + expect(JSON.stringify(lifecycle)).not.toContain("receipt-only secret"); + }); + + it("fails closed when every bounded receipt still owns queued work", async () => { + session.ready = false; + session.projectBootstrap!.bootstrap = { + status: "delivered", + messageId: "bootstrap-complete", + }; + const inputs = Array.from({ length: 128 }, (_, index) => ({ + id: `queued-input-${index}`, + sessionId: session.id, + text: `queued payload ${index}`, + acceptedAt: NOW, + })); + session.projectBootstrap!.queuedInputIds = inputs.map((input) => input.id); + await writeState(root, session.id, { + schemaVersion: 3, + metadata: structuredClone(session.projectBootstrap!), + inputs, + dispatchingInputId: null, + retryCount: 0, + emptyProject: false, + attempts: [], + uncertainInputIds: [], + uncertainInputs: [], + receipts: inputs.map((input, index) => ({ + requestId: index % 2 === 0 ? null : `queued-request-${index}`, + inputId: input.id, + status: "queued" as const, + acceptedAt: NOW, + payloadDigest: createHash("sha256") + .update( + JSON.stringify({ + schemaVersion: 1, + submit: true, + text: input.text, + }), + ) + .digest("hex"), + })), + }); + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + }); + await coordinator.register(session, { emptyProject: false, mode: "boot" }); + + await expect( + coordinator.enqueueWithReceipt( + session.id, + "queued payload 1", + "queued-request-1", + ), + ).resolves.toMatchObject({ + receipt: { + requestId: "queued-request-1", + inputId: "queued-input-1", + status: "queued", + }, + }); + await expect( + coordinator.enqueueWithReceipt( + session.id, + "changed payload at capacity", + "queued-request-1", + ), + ).rejects.toBeInstanceOf(ProjectBootstrapRequestIdConflictError); + await expect( + coordinator.enqueueWithReceipt( + session.id, + "one beyond the bound", + "genuinely-new-request", + ), + ).rejects.toBeInstanceOf(ProjectBootstrapInputCapacityError); + expect((await readState(root, session.id)).receipts).toHaveLength(128); + }); + + it("returns the same uncertain receipt after restart without replaying a response-loss retry", async () => { + session.projectBootstrap!.bootstrap = { + status: "skipped", + reason: "user-proceeded", + }; + session.projectBootstrap!.queuedInputIds = ["input-response-lost"]; + const payload = "durable request whose response was lost"; + const payloadDigest = createHash("sha256") + .update(JSON.stringify({ schemaVersion: 1, submit: true, text: payload })) + .digest("hex"); + await writeState(root, session.id, { + schemaVersion: 3, + metadata: structuredClone(session.projectBootstrap!), + inputs: [ + { + id: "input-response-lost", + sessionId: session.id, + text: payload, + acceptedAt: NOW, + }, + ], + dispatchingInputId: "input-response-lost", + retryCount: 0, + emptyProject: true, + attempts: [], + uncertainInputIds: [], + uncertainInputs: [], + receipts: [ + { + requestId: "request-response-lost", + inputId: "input-response-lost", + status: "queued", + acceptedAt: NOW, + payloadDigest, + }, + ], + }); + + const restarted = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + }); + await restarted.register(session, { emptyProject: true, mode: "boot" }); + const replay = await restarted.enqueueWithReceipt( + session.id, + payload, + "request-response-lost", + ); + + expect(submitted).toEqual([]); + expect(replay.receipt).toEqual({ + requestId: "request-response-lost", + inputId: "input-response-lost", + status: "uncertain", + acceptedAt: NOW, + }); + expect(replay.metadata.queuedInputIds).toEqual([]); + expect(restarted.ownsInput(session.id, "request-response-lost")).toBe(true); + }); + + it("returns one completed keyed receipt before and after restart without another PTY submission", async () => { + session.projectBootstrap!.bootstrap = { + status: "delivered", + messageId: "bootstrap-complete", + }; + const first = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => "input-completed-once", + now: () => NOW, + deliveryTimeoutMs: 60_000, + }); + await first.register(session, { emptyProject: false, mode: "boot" }); + const accepted = await first.enqueueWithReceipt( + session.id, + "complete this logical input once", + "request-completed-once", + ); + expect(accepted.receipt.status).toBe("submitted"); + first.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { + prompt: "complete this logical input once", + }), + ); + await first.onEventPersisted( + analyticsEvent(session.id, "turn.completed", { + assistantText: "done", + }), + ); + + const replayBeforeRestart = await first.enqueueWithReceipt( + session.id, + "complete this logical input once", + "request-completed-once", + ); + expect(replayBeforeRestart.receipt).toEqual({ + requestId: "request-completed-once", + inputId: "input-completed-once", + status: "completed", + acceptedAt: NOW, + }); + expect(submitted.map((entry) => entry.text)).toEqual([ + "complete this logical input once", + ]); + + await first.close(); + const restarted = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + }); + await restarted.register(session, { emptyProject: false, mode: "boot" }); + const replayAfterRestart = await restarted.enqueueWithReceipt( + session.id, + "complete this logical input once", + "request-completed-once", + ); + expect(replayAfterRestart).toEqual(replayBeforeRestart); + expect(submitted.map((entry) => entry.text)).toEqual([ + "complete this logical input once", + ]); + expect((await readState(root, session.id)).inputs).toEqual([]); + }); + + it("reconciles a durable PTY acknowledgement after queue cleanup fails without resubmitting", async () => { + session.projectBootstrap!.bootstrap = { + status: "delivered", + messageId: "bootstrap-complete", + }; + let failCommittedDequeue = false; + let failedOnce = false; + const first = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => "input-accepted-before-cleanup-failure", + writeState: async (_file, value) => { + const durable = value as DurableBootstrapState; + if ( + failCommittedDequeue && + !failedOnce && + durable.inputs.length === 0 && + durable.receipts?.some( + (receipt) => + receipt.inputId === "input-accepted-before-cleanup-failure" && + receipt.status === "submitted", + ) + ) { + failedOnce = true; + throw new Error("injected dequeue persistence failure"); + } + await writeState(root, session.id, durable); + }, + }); + await first.register(session, { emptyProject: false, mode: "boot" }); + failCommittedDequeue = true; + + await first.enqueueWithReceipt( + session.id, + "send exactly once", + "request-cleanup-recovery", + ); + expect(submitted.map((entry) => entry.text)).toEqual(["send exactly once"]); + expect((await readState(root, session.id)).dispatchingInputId).toBe( + "input-accepted-before-cleanup-failure", + ); + expect( + JSON.parse( + await fs.readFile( + path.join(root, session.id, "accepted-inputs.json"), + "utf8", + ), + ), + ).toMatchObject({ + inputIds: ["input-accepted-before-cleanup-failure"], + }); + + await first.close(); + const restarted = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + }); + await restarted.register(session, { emptyProject: false, mode: "boot" }); + const replay = await restarted.enqueueWithReceipt( + session.id, + "send exactly once", + "request-cleanup-recovery", + ); + + expect(submitted.map((entry) => entry.text)).toEqual(["send exactly once"]); + expect(replay.receipt.status).toBe("uncertain"); + const recovered = await readState(root, session.id); + expect(recovered.inputs).toEqual([]); + expect(recovered.uncertainInputs).toEqual([ + expect.objectContaining({ + id: "input-accepted-before-cleanup-failure", + text: "send exactly once", + }), + ]); + expect( + JSON.parse( + await fs.readFile( + path.join(root, session.id, "accepted-inputs.json"), + "utf8", + ), + ), + ).toEqual({ schemaVersion: 1, inputIds: [] }); + }); + + it("retries a durable FIFO head only after a pre-Enter rejection is durably rolled back", async () => { + session.projectBootstrap!.bootstrap = { + status: "delivered", + messageId: "bootstrap-complete", + }; + const ids = ["input-pre-enter-a", "input-after-pre-enter-b"]; + let firstAttempts = 0; + manager.submitInput = async ( + id: string, + text: string, + submit?: boolean, + canWrite?: () => boolean | Promise, + background?: boolean, + lifecycle?: SessionInputWriteLifecycle, + ) => { + if (canWrite && !(await canWrite())) return false; + await lifecycle?.beforeFirstWrite?.(); + if (text === "turn A rejected before Enter") { + firstAttempts += 1; + if (firstAttempts === 1) { + await lifecycle?.onNotSubmitted?.(); + throw new Error("injected pre-Enter text rejection"); + } + } + if (lifecycle?.canWriteNow && !lifecycle.canWriteNow()) return false; + submitted.push({ sessionId: id, text, submit, background }); + return true; + }; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + deliveryTimeoutMs: 60_000, + }); + await coordinator.register(session, { emptyProject: false, mode: "boot" }); + + const first = await coordinator.enqueueWithReceipt( + session.id, + "turn A rejected before Enter", + "request-pre-enter-a", + ); + expect(first.receipt.status).toBe("queued"); + expect(firstAttempts).toBe(1); + expect(submitted).toEqual([]); + expect(await readState(root, session.id)).toMatchObject({ + dispatchingInputId: null, + inputs: [expect.objectContaining({ id: "input-pre-enter-a" })], + }); + + await coordinator.enqueueWithReceipt( + session.id, + "turn B waits for A", + "request-pre-enter-b", + ); + expect(firstAttempts).toBe(2); + expect(submitted.map((entry) => entry.text)).toEqual([ + "turn A rejected before Enter", + ]); + + coordinator.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { + prompt: "turn A rejected before Enter", + }), + ); + await coordinator.onEventPersisted( + analyticsEvent(session.id, "turn.completed", { + assistantText: "A completed after its one safe retry", + }), + ); + + expect(submitted.map((entry) => entry.text)).toEqual([ + "turn A rejected before Enter", + "turn B waits for A", + ]); + const durable = await readState(root, session.id); + expect(durable.receipts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + inputId: "input-pre-enter-a", + status: "completed", + }), + expect.objectContaining({ + inputId: "input-after-pre-enter-b", + status: "submitted", + }), + ]), + ); + }); + + it("holds an ambiguous FIFO Enter rejection until A's deadline and admits B once", async () => { + vi.useFakeTimers(); + session.projectBootstrap!.bootstrap = { + status: "delivered", + messageId: "bootstrap-complete", + }; + const ids = ["input-ambiguous-a", "input-after-ambiguous-b"]; + const enterAttempts: string[] = []; + manager.submitInput = async ( + id: string, + text: string, + submit?: boolean, + canWrite?: () => boolean | Promise, + background?: boolean, + lifecycle?: SessionInputWriteLifecycle, + ) => { + if (canWrite && !(await canWrite())) return false; + await lifecycle?.beforeFirstWrite?.(); + if (lifecycle?.canWriteNow && !lifecycle.canWriteNow()) return false; + enterAttempts.push(text); + if (text === "turn A has an ambiguous Enter rejection") { + throw new Error("injected ambiguous Enter rejection"); + } + submitted.push({ sessionId: id, text, submit, background }); + return true; + }; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + deliveryTimeoutMs: 100, + }); + await coordinator.register(session, { emptyProject: false, mode: "boot" }); + + const first = await coordinator.enqueueWithReceipt( + session.id, + "turn A has an ambiguous Enter rejection", + "request-ambiguous-a", + ); + expect(first.receipt.status).toBe("submitted"); + await coordinator.enqueueWithReceipt( + session.id, + "turn B waits behind ambiguous A", + "request-ambiguous-b", + ); + expect(enterAttempts).toEqual(["turn A has an ambiguous Enter rejection"]); + expect(await readState(root, session.id)).toMatchObject({ + dispatchingInputId: "input-ambiguous-a", + receipts: expect.arrayContaining([ + expect.objectContaining({ + inputId: "input-ambiguous-a", + status: "submitted", + }), + expect.objectContaining({ + inputId: "input-after-ambiguous-b", + status: "queued", + }), + ]), + }); + + await vi.advanceTimersByTimeAsync(101); + await flushCoordinator(coordinator, session.id); + + expect(enterAttempts).toEqual([ + "turn A has an ambiguous Enter rejection", + "turn B waits behind ambiguous A", + ]); + let durable = await readState(root, session.id); + expect(durable.dispatchingInputId).toBeNull(); + expect(durable.inputs).toEqual([]); + expect(durable.uncertainInputs).toContainEqual( + expect.objectContaining({ id: "input-ambiguous-a" }), + ); + expect(durable.receipts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + inputId: "input-ambiguous-a", + status: "uncertain", + }), + expect.objectContaining({ + inputId: "input-after-ambiguous-b", + status: "submitted", + }), + ]), + ); + + // A's late completion consumes only A's correlation. It cannot clear B's + // active ownership, alter A's terminal uncertainty, or submit B twice. + coordinator.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { + prompt: "turn A has an ambiguous Enter rejection", + }), + ); + await coordinator.onEventPersisted( + analyticsEvent(session.id, "turn.completed", { + assistantText: "A eventually completed", + }), + ); + durable = await readState(root, session.id); + expect( + durable.receipts?.find( + (receipt) => receipt.inputId === "input-ambiguous-a", + )?.status, + ).toBe("uncertain"); + expect(enterAttempts).toEqual([ + "turn A has an ambiguous Enter rejection", + "turn B waits behind ambiguous A", + ]); + expect(coordinator.ownsInput(session.id)).toBe(true); + }); + + it("atomically completes an ambiguous FIFO dispatch before admitting B", async () => { + vi.useFakeTimers(); + session.projectBootstrap!.bootstrap = { + status: "delivered", + messageId: "bootstrap-complete", + }; + const ids = ["input-completed-ambiguous-a", "input-after-completed-b"]; + const enterAttempts: string[] = []; + manager.submitInput = async ( + id: string, + text: string, + submit?: boolean, + canWrite?: () => boolean | Promise, + background?: boolean, + lifecycle?: SessionInputWriteLifecycle, + ) => { + if (canWrite && !(await canWrite())) return false; + await lifecycle?.beforeFirstWrite?.(); + if (lifecycle?.canWriteNow && !lifecycle.canWriteNow()) return false; + enterAttempts.push(text); + if (text === "turn A completes despite ambiguous Enter") { + throw new Error("injected ambiguous Enter rejection"); + } + submitted.push({ sessionId: id, text, submit, background }); + return true; + }; + const first = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + deliveryTimeoutMs: 100, + }); + await first.register(session, { emptyProject: false, mode: "boot" }); + await first.enqueueWithReceipt( + session.id, + "turn A completes despite ambiguous Enter", + "request-completed-ambiguous-a", + ); + await first.enqueueWithReceipt( + session.id, + "turn B follows completed A", + "request-after-completed-b", + ); + + first.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { + prompt: "turn A completes despite ambiguous Enter", + }), + ); + await first.onEventPersisted( + analyticsEvent(session.id, "turn.completed", { + assistantText: "A completed before its deadline", + }), + ); + + expect(enterAttempts).toEqual([ + "turn A completes despite ambiguous Enter", + "turn B follows completed A", + ]); + let durable = await readState(root, session.id); + expect(durable.dispatchingInputId).toBeNull(); + expect(durable.inputs).toEqual([]); + expect(durable.receipts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + inputId: "input-completed-ambiguous-a", + status: "completed", + }), + expect.objectContaining({ + inputId: "input-after-completed-b", + status: "submitted", + }), + ]), + ); + + // A's timer was removed only after its atomic completion commit. Advancing + // the old deadline can expire B, but can never downgrade or replay A. + await vi.advanceTimersByTimeAsync(101); + await flushCoordinator(first, session.id); + durable = await readState(root, session.id); + expect( + durable.receipts?.find( + (receipt) => receipt.inputId === "input-completed-ambiguous-a", + )?.status, + ).toBe("completed"); + expect(enterAttempts).toEqual([ + "turn A completes despite ambiguous Enter", + "turn B follows completed A", + ]); + + await first.close(); + const restarted = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + deliveryTimeoutMs: 100, + }); + await restarted.register(session, { emptyProject: false, mode: "boot" }); + const replay = await restarted.enqueueWithReceipt( + session.id, + "turn A completes despite ambiguous Enter", + "request-completed-ambiguous-a", + ); + expect(replay.receipt.status).toBe("completed"); + expect(enterAttempts).toEqual([ + "turn A completes despite ambiguous Enter", + "turn B follows completed A", + ]); + durable = await readState(root, session.id); + expect( + durable.receipts?.find( + (receipt) => receipt.inputId === "input-completed-ambiguous-a", + )?.status, + ).toBe("completed"); + }); + + it("retains A's deadline when its correlated completion cannot be persisted", async () => { + vi.useFakeTimers(); + session.projectBootstrap!.bootstrap = { + status: "delivered", + messageId: "bootstrap-complete", + }; + const ids = [ + "input-failed-completion-a", + "input-after-failed-completion-b", + ]; + const enterAttempts: string[] = []; + let failCompletionCommit = false; + let completionCommitFailed = false; + manager.submitInput = async ( + id: string, + text: string, + submit?: boolean, + canWrite?: () => boolean | Promise, + background?: boolean, + lifecycle?: SessionInputWriteLifecycle, + ) => { + if (canWrite && !(await canWrite())) return false; + await lifecycle?.beforeFirstWrite?.(); + if (lifecycle?.canWriteNow && !lifecycle.canWriteNow()) return false; + enterAttempts.push(text); + if (text === "turn A completion cannot commit") { + throw new Error("injected ambiguous Enter rejection"); + } + submitted.push({ sessionId: id, text, submit, background }); + return true; + }; + const first = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + deliveryTimeoutMs: 100, + writeState: async (_file, value) => { + const durable = value as DurableBootstrapState; + if ( + failCompletionCommit && + !completionCommitFailed && + durable.inputs.every( + (input) => input.id !== "input-failed-completion-a", + ) && + durable.receipts?.some( + (receipt) => + receipt.inputId === "input-failed-completion-a" && + receipt.status === "completed", + ) + ) { + completionCommitFailed = true; + throw new Error("injected completed dequeue persistence failure"); + } + await writeState(root, session.id, durable); + }, + }); + await first.register(session, { emptyProject: false, mode: "boot" }); + await first.enqueueWithReceipt( + session.id, + "turn A completion cannot commit", + "request-failed-completion-a", + ); + await first.enqueueWithReceipt( + session.id, + "turn B waits for bounded completion recovery", + "request-after-failed-completion-b", + ); + first.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { + prompt: "turn A completion cannot commit", + }), + ); + failCompletionCommit = true; + + await expect( + first.onEventPersisted( + analyticsEvent(session.id, "turn.completed", { + assistantText: "A completed but its dequeue write failed", + }), + ), + ).rejects.toThrow("project bootstrap state persistence failed"); + expect(completionCommitFailed).toBe(true); + expect(enterAttempts).toEqual(["turn A completion cannot commit"]); + expect(await readState(root, session.id)).toMatchObject({ + dispatchingInputId: "input-failed-completion-a", + receipts: expect.arrayContaining([ + expect.objectContaining({ + inputId: "input-failed-completion-a", + status: "submitted", + }), + ]), + }); + + await vi.advanceTimersByTimeAsync(101); + await flushCoordinator(first, session.id); + + expect(enterAttempts).toEqual([ + "turn A completion cannot commit", + "turn B waits for bounded completion recovery", + ]); + let durable = await readState(root, session.id); + expect(durable.inputs).toEqual([]); + expect(durable.receipts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + inputId: "input-failed-completion-a", + status: "uncertain", + }), + expect.objectContaining({ + inputId: "input-after-failed-completion-b", + status: "submitted", + }), + ]), + ); + const replayBeforeRestart = await first.enqueueWithReceipt( + session.id, + "turn A completion cannot commit", + "request-failed-completion-a", + ); + expect(replayBeforeRestart.receipt.status).toBe("uncertain"); + expect(enterAttempts).toHaveLength(2); + + await first.close(); + const restarted = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + deliveryTimeoutMs: 100, + }); + await restarted.register(session, { emptyProject: false, mode: "boot" }); + const replayAfterRestart = await restarted.enqueueWithReceipt( + session.id, + "turn A completion cannot commit", + "request-failed-completion-a", + ); + expect(replayAfterRestart.receipt.status).toBe("uncertain"); + expect(enterAttempts).toEqual([ + "turn A completion cannot commit", + "turn B waits for bounded completion recovery", + ]); + durable = await readState(root, session.id); + expect( + durable.receipts?.find( + (receipt) => receipt.inputId === "input-failed-completion-a", + )?.status, + ).toBe("uncertain"); + }); + + it("does not replay an ambiguous FIFO Enter rejection after restart", async () => { + session.projectBootstrap!.bootstrap = { + status: "delivered", + messageId: "bootstrap-complete", + }; + const ids = ["input-restart-ambiguous-a", "input-restart-successor-b"]; + const enterAttempts: string[] = []; + manager.submitInput = async ( + id: string, + text: string, + submit?: boolean, + canWrite?: () => boolean | Promise, + background?: boolean, + lifecycle?: SessionInputWriteLifecycle, + ) => { + if (canWrite && !(await canWrite())) return false; + await lifecycle?.beforeFirstWrite?.(); + if (lifecycle?.canWriteNow && !lifecycle.canWriteNow()) return false; + enterAttempts.push(text); + if (text === "turn A is ambiguous across restart") { + throw new Error("injected ambiguous Enter rejection"); + } + submitted.push({ sessionId: id, text, submit, background }); + return true; + }; + const first = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + deliveryTimeoutMs: 60_000, + }); + await first.register(session, { emptyProject: false, mode: "boot" }); + await first.enqueueWithReceipt( + session.id, + "turn A is ambiguous across restart", + "request-restart-ambiguous-a", + ); + await first.enqueueWithReceipt( + session.id, + "turn B may run after restart", + "request-restart-successor-b", + ); + expect(enterAttempts).toEqual(["turn A is ambiguous across restart"]); + await first.close(); + + const restarted = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + deliveryTimeoutMs: 60_000, + }); + await restarted.register(session, { emptyProject: false, mode: "boot" }); + + expect(enterAttempts).toEqual([ + "turn A is ambiguous across restart", + "turn B may run after restart", + ]); + const durable = await readState(root, session.id); + expect(durable.inputs).toEqual([]); + expect(durable.receipts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + inputId: "input-restart-ambiguous-a", + status: "uncertain", + }), + expect.objectContaining({ + inputId: "input-restart-successor-b", + status: "submitted", + }), + ]), + ); + }); + + it("keeps a bounded turn deadline when the first post-Enter state write fails", async () => { + vi.useFakeTimers(); + session.projectBootstrap!.bootstrap = { + status: "delivered", + messageId: "bootstrap-complete", + }; + const ids = ["input-post-enter-failure", "input-after-failure"]; + let failSubmittedState = false; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + deliveryTimeoutMs: 100, + writeState: async (_file, value) => { + const durable = value as DurableBootstrapState; + if ( + failSubmittedState && + durable.dispatchingInputId === "input-post-enter-failure" && + durable.receipts?.some( + (receipt) => + receipt.inputId === "input-post-enter-failure" && + receipt.status === "submitted", + ) + ) { + throw new Error("injected submitted-state persistence failure"); + } + await writeState(root, session.id, durable); + }, + }); + await coordinator.register(session, { emptyProject: false, mode: "boot" }); + failSubmittedState = true; + + await coordinator.enqueueWithReceipt( + session.id, + "turn whose post-Enter write fails", + "request-post-enter-failure", + ); + failSubmittedState = false; + await coordinator.enqueueWithReceipt( + session.id, + "turn waiting behind the failure", + "request-after-failure", + ); + + expect(submitted.map((entry) => entry.text)).toEqual([ + "turn whose post-Enter write fails", + ]); + expect(coordinator.ownsInput(session.id)).toBe(true); + + await vi.advanceTimersByTimeAsync(101); + await flushCoordinator(coordinator, session.id); + + expect(submitted.map((entry) => entry.text)).toEqual([ + "turn whose post-Enter write fails", + "turn waiting behind the failure", + ]); + const durable = await readState(root, session.id); + expect(durable.receipts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + inputId: "input-post-enter-failure", + status: "uncertain", + }), + expect.objectContaining({ + inputId: "input-after-failure", + status: "submitted", + }), + ]), + ); + expect(durable.inputs).toEqual([]); + }); + + it("releases one successor when the accepted-input ledger write fails after Enter", async () => { + vi.useFakeTimers(); + session.projectBootstrap!.bootstrap = { + status: "delivered", + messageId: "bootstrap-complete", + }; + const ids = ["input-ledger-failure", "input-after-ledger-failure"]; + let failAcceptedLedger = true; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + deliveryTimeoutMs: 100, + writeAcceptedLedger: async (file, value) => { + const ledger = value as { inputIds: string[] }; + if ( + failAcceptedLedger && + ledger.inputIds.includes("input-ledger-failure") + ) { + failAcceptedLedger = false; + throw new Error("injected accepted-input ledger failure"); + } + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile(file, `${JSON.stringify(value)}\n`); + }, + }); + await coordinator.register(session, { emptyProject: false, mode: "boot" }); + + await coordinator.enqueueWithReceipt( + session.id, + "turn whose accepted ledger write fails", + "request-ledger-failure", + ); + await coordinator.enqueueWithReceipt( + session.id, + "turn waiting behind the ledger failure", + "request-after-ledger-failure", + ); + + expect(failAcceptedLedger).toBe(false); + expect(submitted.map((entry) => entry.text)).toEqual([ + "turn whose accepted ledger write fails", + ]); + expect(await readState(root, session.id)).toMatchObject({ + dispatchingInputId: "input-ledger-failure", + receipts: expect.arrayContaining([ + expect.objectContaining({ + inputId: "input-ledger-failure", + status: "submitted", + }), + ]), + }); + await vi.advanceTimersByTimeAsync(101); + await flushCoordinator(coordinator, session.id); + + expect(submitted.map((entry) => entry.text)).toEqual([ + "turn whose accepted ledger write fails", + "turn waiting behind the ledger failure", + ]); + const durable = await readState(root, session.id); + expect(durable.receipts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + inputId: "input-ledger-failure", + status: "uncertain", + }), + expect.objectContaining({ + inputId: "input-after-ledger-failure", + status: "submitted", + }), + ]), + ); + expect(durable.inputs).toEqual([]); + expect(durable.uncertainInputs).toContainEqual( + expect.objectContaining({ id: "input-ledger-failure" }), + ); + }); + + it("reconciles an accepted input after live dequeue persistence fails before admitting one successor", async () => { + vi.useFakeTimers(); + session.projectBootstrap!.bootstrap = { + status: "delivered", + messageId: "bootstrap-complete", + }; + const ids = ["input-dequeue-failure", "input-after-dequeue-failure"]; + let failCommittedDequeue = true; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + deliveryTimeoutMs: 100, + writeState: async (_file, value) => { + const durable = value as DurableBootstrapState; + if ( + failCommittedDequeue && + durable.inputs.every( + (input) => input.id !== "input-dequeue-failure", + ) && + durable.receipts?.some( + (receipt) => + receipt.inputId === "input-dequeue-failure" && + receipt.status === "submitted", + ) + ) { + failCommittedDequeue = false; + throw new Error("injected dequeue persistence failure"); + } + await writeState(root, session.id, durable); + }, + }); + await coordinator.register(session, { emptyProject: false, mode: "boot" }); + + await coordinator.enqueueWithReceipt( + session.id, + "turn whose dequeue persistence fails", + "request-dequeue-failure", + ); + await coordinator.enqueueWithReceipt( + session.id, + "turn waiting behind the dequeue failure", + "request-after-dequeue-failure", + ); + + expect(submitted.map((entry) => entry.text)).toEqual([ + "turn whose dequeue persistence fails", + ]); + expect( + JSON.parse( + await fs.readFile( + path.join(root, session.id, "accepted-inputs.json"), + "utf8", + ), + ), + ).toMatchObject({ inputIds: ["input-dequeue-failure"] }); + expect(failCommittedDequeue).toBe(false); + + await vi.advanceTimersByTimeAsync(101); + await flushCoordinator(coordinator, session.id); + + expect(submitted.map((entry) => entry.text)).toEqual([ + "turn whose dequeue persistence fails", + "turn waiting behind the dequeue failure", + ]); + const durable = await readState(root, session.id); + expect(durable.receipts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + inputId: "input-dequeue-failure", + status: "uncertain", + }), + expect.objectContaining({ + inputId: "input-after-dequeue-failure", + status: "submitted", + }), + ]), + ); + expect(durable.inputs).toEqual([]); + expect( + JSON.parse( + await fs.readFile( + path.join(root, session.id, "accepted-inputs.json"), + "utf8", + ), + ), + ).toEqual({ schemaVersion: 1, inputIds: [] }); + }); + + it("does not retain a timer after completion observed during submission", async () => { + vi.useFakeTimers(); + session.projectBootstrap!.bootstrap = { + status: "delivered", + messageId: "bootstrap-complete", + }; + let completion: Promise | undefined; + manager.submitInput = async ( + id: string, + text: string, + submit?: boolean, + canWrite?: () => boolean | Promise, + background?: boolean, + lifecycle?: SessionInputWriteLifecycle, + ) => { + if (canWrite && !(await canWrite())) return false; + await lifecycle?.beforeFirstWrite?.(); + if (lifecycle?.canWriteNow && !lifecycle.canWriteNow()) return false; + submitted.push({ sessionId: id, text, submit, background }); + coordinator.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { prompt: text }), + ); + completion = coordinator.onEventPersisted( + analyticsEvent(session.id, "turn.completed", { + assistantText: "completed immediately", + }), + ); + return true; + }; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => "input-immediate-completion", + deliveryTimeoutMs: 100, + }); + await coordinator.register(session, { emptyProject: false, mode: "boot" }); + + await coordinator.enqueueWithReceipt( + session.id, + "complete while submission unwinds", + "request-immediate-completion", + ); + await completion; + + const internals = coordinator as unknown as { + activeTurns: Map; + activeTurnTimers: Map; + }; + expect(internals.activeTurns.has(session.id)).toBe(false); + expect(internals.activeTurnTimers.has(session.id)).toBe(false); + await vi.advanceTimersByTimeAsync(101); + expect( + ( + await coordinator.enqueueWithReceipt( + session.id, + "complete while submission unwinds", + "request-immediate-completion", + ) + ).receipt.status, + ).toBe("completed"); + expect(submitted).toHaveLength(1); + }); + + it("redrains once after recognized setup input becomes ready without a model event", async () => { + session.ready = false; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => "input-after-setup", + readinessTimeoutMs: 60_000, + }); + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + await coordinator.enqueue(session.id, "build immediately after setup"); + + coordinator.onTerminalInput(session.id, { blockingPrompt: true }); + session.ready = true; + await coordinator.onSessionStatus(session); + + expect(submitted.map((entry) => entry.text)).toEqual([ + "build immediately after setup", + ]); + expect(session.projectBootstrap?.queuedInputIds).toEqual([]); + expect( + ( + coordinator as unknown as { + terminalPreemptions: Set; + } + ).terminalPreemptions.has(session.id), + ).toBe(false); + }); + + it("releases durable user input after a submitted bootstrap reaches its bounded timeout", async () => { + vi.useFakeTimers(); + const ids = ["attempt-hung", "input-behind-hung-bootstrap"]; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + deliveryTimeoutMs: 100, + }); + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + await coordinator.enqueue(session.id, "user build request wins next"); + expect(submitted.map((entry) => entry.text)).toHaveLength(1); + + await vi.advanceTimersByTimeAsync(101); + await flushCoordinator(coordinator, session.id); + + expect(submitted.map((entry) => entry.text)).toEqual([ + expect.stringContaining("Agent Studio project bootstrap"), + "user build request wins next", + ]); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "skipped", + reason: "user-proceeded", + }); + }); + + it("holds each API turn after dequeue and releases one successor per bounded timeout", async () => { + vi.useFakeTimers(); + session.projectBootstrap!.bootstrap = { + status: "delivered", + messageId: "bootstrap-complete", + }; + const ids = [ + "first-active-input", + "second-waiting-input", + "third-waiting-input", + ]; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => ids.shift() ?? "unexpected-id", + deliveryTimeoutMs: 100, + }); + await coordinator.register(session, { emptyProject: false, mode: "boot" }); + + const first = await coordinator.enqueueWithReceipt( + session.id, + "first server-owned turn", + "request-first-active", + ); + expect(first.receipt.status).toBe("submitted"); + expect((await readState(root, session.id)).inputs).toEqual([]); + expect(coordinator.ownsInput(session.id)).toBe(true); + + const second = await coordinator.enqueueWithReceipt( + session.id, + "second server-owned turn", + "request-second-waiting", + ); + expect(second.receipt.status).toBe("queued"); + const third = await coordinator.enqueueWithReceipt( + session.id, + "third server-owned turn", + "request-third-waiting", + ); + expect(third.receipt.status).toBe("queued"); + expect(submitted.map((entry) => entry.text)).toEqual([ + "first server-owned turn", + ]); + + await vi.advanceTimersByTimeAsync(101); + await flushCoordinator(coordinator, session.id); + + expect(submitted.map((entry) => entry.text)).toEqual([ + "first server-owned turn", + "second server-owned turn", + ]); + const durable = await readState(root, session.id); + expect(durable.receipts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + inputId: "first-active-input", + status: "uncertain", + }), + expect.objectContaining({ + inputId: "second-waiting-input", + status: "submitted", + }), + expect.objectContaining({ + inputId: "third-waiting-input", + status: "queued", + }), + ]), + ); + expect(durable.inputs).toEqual([ + expect.objectContaining({ id: "third-waiting-input" }), + ]); + + // A late completion for A consumes only A's correlation. It cannot change + // A's terminal uncertainty, clear B, or advance C. + coordinator.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { + prompt: "first server-owned turn", + }), + ); + await coordinator.onEventPersisted( + analyticsEvent(session.id, "turn.completed", { + assistantText: "first eventually finished", + }), + ); + let afterLateCompletion = await readState(root, session.id); + expect(afterLateCompletion.receipts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + inputId: "first-active-input", + status: "uncertain", + }), + expect.objectContaining({ + inputId: "second-waiting-input", + status: "submitted", + }), + ]), + ); + expect(afterLateCompletion.inputs).toEqual([ + expect.objectContaining({ id: "third-waiting-input" }), + ]); + expect(submitted.map((entry) => entry.text)).toEqual([ + "first server-owned turn", + "second server-owned turn", + ]); + + // B owns a distinct deadline. Its timeout terminalizes only B, then admits + // exactly one successor C. + await vi.advanceTimersByTimeAsync(101); + await flushCoordinator(coordinator, session.id); + expect(submitted.map((entry) => entry.text)).toEqual([ + "first server-owned turn", + "second server-owned turn", + "third server-owned turn", + ]); + afterLateCompletion = await readState(root, session.id); + expect(afterLateCompletion.inputs).toEqual([]); + expect(afterLateCompletion.receipts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + inputId: "first-active-input", + status: "uncertain", + }), + expect.objectContaining({ + inputId: "second-waiting-input", + status: "uncertain", + }), + expect.objectContaining({ + inputId: "third-waiting-input", + status: "submitted", + }), + ]), + ); + }); + + it("fences the final PTY boundary and leaves no acknowledgement when close wins", async () => { + const writes: string[] = []; + let finalAuthorizationReached!: () => void; + const atFinalAuthorization = new Promise((resolve) => { + finalAuthorizationReached = resolve; + }); + let releaseFinalBoundary!: () => void; + const released = new Promise((resolve) => { + releaseFinalBoundary = resolve; + }); + manager.submitInput = async ( + _id: string, + text: string, + _submit?: boolean, + canWrite?: () => boolean | Promise, + _background?: boolean, + lifecycle?: SessionInputWriteLifecycle, + ) => { + await lifecycle?.beforeFirstWrite?.(); + if (lifecycle?.canWriteNow && !lifecycle.canWriteNow()) { + throw new SessionInputGuardRejectedError(false); + } + writes.push(text); + if (canWrite && !(await canWrite())) { + throw new SessionInputGuardRejectedError(true); + } + finalAuthorizationReached(); + await released; + if (lifecycle?.canWriteNow && !lifecycle.canWriteNow()) { + writes.push("\x15"); + throw new SessionInputGuardRejectedError(true); + } + writes.push("\r"); + return true; + }; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => "attempt-close-fence", + deliveryTimeoutMs: 60_000, + }); + const registering = coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + await atFinalAuthorization; + const closing = coordinator.close(); + releaseFinalBoundary(); + await Promise.all([registering, closing]); + + expect(writes).toEqual([ + expect.stringContaining("Agent Studio project bootstrap"), + "\x15", + ]); + const durable = await readState(root, session.id); + expect(durable.attempts[0]).toMatchObject({ + attemptId: "attempt-close-fence", + phase: "dispatching", + }); + await expect( + fs.readFile(path.join(root, session.id, "accepted-inputs.json"), "utf8"), + ).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("leaves a durable uncertain intent when shutdown starts after Enter but before acknowledgement", async () => { + session.projectBootstrap!.bootstrap = { + status: "delivered", + messageId: "bootstrap-complete", + }; + let closing: Promise | undefined; + manager.submitInput = async ( + _id: string, + text: string, + _submit?: boolean, + canWrite?: () => boolean | Promise, + _background?: boolean, + lifecycle?: SessionInputWriteLifecycle, + ) => { + await lifecycle?.beforeFirstWrite?.(); + if (canWrite && !(await canWrite())) return false; + submitted.push({ + sessionId: session.id, + text, + submit: true, + background: false, + }); + closing = coordinator.close(); + return true; + }; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => "input-entered-before-close", + }); + await coordinator.register(session, { emptyProject: false, mode: "boot" }); + + await coordinator.enqueueWithReceipt( + session.id, + "entered before shutdown", + "request-entered-before-close", + ); + await closing; + + expect(submitted.map((entry) => entry.text)).toEqual([ + "entered before shutdown", + ]); + const durable = await readState(root, session.id); + expect(durable.dispatchingInputId).toBe("input-entered-before-close"); + expect(durable.receipts).toContainEqual( + expect.objectContaining({ + requestId: "request-entered-before-close", + inputId: "input-entered-before-close", + status: "submitted", + }), + ); + await expect( + fs.readFile(path.join(root, session.id, "accepted-inputs.json"), "utf8"), + ).rejects.toMatchObject({ code: "ENOENT" }); + + const restarted = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + }); + await restarted.register(session, { emptyProject: false, mode: "boot" }); + const replay = await restarted.enqueueWithReceipt( + session.id, + "entered before shutdown", + "request-entered-before-close", + ); + expect(replay.receipt.status).toBe("uncertain"); + expect(submitted.map((entry) => entry.text)).toEqual([ + "entered before shutdown", + ]); + const recovered = await readState(root, session.id); + expect(recovered.dispatchingInputId).toBeNull(); + expect(recovered.inputs).toEqual([]); + expect(recovered.uncertainInputs).toEqual([ + expect.objectContaining({ id: "input-entered-before-close" }), + ]); + await expect( + fs.readFile(path.join(root, session.id, "accepted-inputs.json"), "utf8"), + ).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("recovers only the pending claim transition after both claim stores reject", async () => { + vi.useFakeTimers(); + let rejectClaimStores = true; + let allocatedAttempts = 0; + const setMetadata = manager.setProjectBootstrapMetadata.bind(manager); + manager.setProjectBootstrapMetadata = async (id, metadata) => { + if (rejectClaimStores && metadata.bootstrap.status === "failed") { + throw new Error("injected sessions projection failure"); + } + await setMetadata(id, metadata); + }; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + readinessTimeoutMs: 50, + generateId: () => { + allocatedAttempts += 1; + return "claim-that-never-committed"; + }, + writeState: async (_file, value) => { + const durable = value as DurableBootstrapState; + if ( + rejectClaimStores && + (durable.metadata.bootstrap.status === "generating" || + durable.metadata.bootstrap.status === "failed") + ) { + throw new Error("injected queue-store failure"); + } + await writeState(root, session.id, durable); + }, + }); + + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + expect(allocatedAttempts).toBe(1); + expect(submitted).toEqual([]); + expect((await readState(root, session.id)).metadata.bootstrap).toEqual({ + status: "pending", + }); + + rejectClaimStores = false; + await vi.advanceTimersByTimeAsync(51); + await flushCoordinator(coordinator, session.id); + + expect(allocatedAttempts).toBe(1); + expect(submitted).toEqual([]); + expect(session.projectBootstrap?.bootstrap).toEqual({ + status: "failed", + retryable: true, + errorCode: "persistence_failed", + }); + expect( + (coordinator as unknown as { timers: Map }).timers.size, + ).toBe(0); + expect( + ( + coordinator as unknown as { + pendingBootstrapFailureTransitions: Map; + } + ).pendingBootstrapFailureTransitions.size, + ).toBe(0); + }); + + it("fences coordinator-owned completion and status events to one runtime epoch", async () => { + let liveEpoch: string | null = "epoch-a"; + manager.getRuntimeEpoch = () => liveEpoch; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => "attempt-epoch-a", + }); + await coordinator.register( + session, + { emptyProject: true, mode: "created" }, + "epoch-a", + ); + const prompt = submitted[0]!.text; + coordinator.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { prompt }), + "epoch-a", + ); + + liveEpoch = null; + await coordinator.transitionRuntimeEpoch( + { ...session, status: "starting", ready: false }, + "epoch-b", + ); + liveEpoch = "epoch-b"; + session.status = "running"; + session.ready = true; + await coordinator.register( + session, + { emptyProject: true, mode: "resumed" }, + "epoch-b", + ); + const afterTransition = await readState(root, session.id); + expect(afterTransition.metadata.bootstrap).toEqual({ + status: "failed", + retryable: false, + errorCode: "session_exited", + }); + + const late = analyticsEvent( + session.id, + "turn.completed", + { assistantText: "late old-process completion" }, + "late-epoch-a-completion", + ); + await coordinator.onEventPersisted(late, "epoch-a"); + await coordinator.onSessionStatus( + { ...session, status: "exited", ready: false }, + "epoch-a", + ); + expect(await readState(root, session.id)).toEqual(afterTransition); + expect( + coordinator.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { prompt }), + "epoch-a", + ).payload, + ).not.toHaveProperty("projectBootstrapAttemptId"); + }); + + it("terminalizes an old user turn before admitting a replacement runtime", async () => { + session.projectBootstrap!.bootstrap = { + status: "delivered", + messageId: "bootstrap-complete", + }; + let liveEpoch: string | null = "epoch-a"; + manager.getRuntimeEpoch = () => liveEpoch; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => "input-owned-by-epoch-a", + }); + await coordinator.register( + session, + { emptyProject: false, mode: "created" }, + "epoch-a", + ); + const accepted = await coordinator.enqueueWithReceipt( + session.id, + "build from runtime A", + "request-runtime-a", + ); + expect(accepted.receipt.status).toBe("submitted"); + + liveEpoch = null; + await coordinator.transitionRuntimeEpoch( + { ...session, status: "starting", ready: false }, + "epoch-b", + ); + liveEpoch = "epoch-b"; + session.status = "running"; + session.ready = true; + await coordinator.register( + session, + { emptyProject: false, mode: "resumed" }, + "epoch-b", + ); + const durable = await readState(root, session.id); + expect(durable.receipts).toContainEqual( + expect.objectContaining({ + requestId: "request-runtime-a", + status: "uncertain", + }), + ); + + await coordinator.onEventPersisted( + analyticsEvent( + session.id, + "turn.completed", + { assistantText: "late runtime A reply" }, + "runtime-a-late-turn", + ), + "epoch-a", + ); + expect((await readState(root, session.id)).receipts).toEqual( + durable.receipts, + ); + }); + + it("clears an old raw-terminal hold before admitting the replacement runtime", async () => { + session.projectBootstrap!.bootstrap = { + status: "delivered", + messageId: "bootstrap-complete", + }; + let liveEpoch: string | null = "epoch-a"; + manager.getRuntimeEpoch = () => liveEpoch; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => "input-owned-by-epoch-b", + }); + await coordinator.register( + session, + { emptyProject: false, mode: "created" }, + "epoch-a", + ); + coordinator.onTerminalInput(session.id, { + runtimeEpoch: "epoch-a", + blockingPrompt: false, + }); + coordinator.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { + prompt: "raw input owned by runtime A", + }), + "epoch-a", + ); + await flushCoordinator(coordinator, session.id); + + liveEpoch = null; + await coordinator.transitionRuntimeEpoch( + { ...session, status: "starting", ready: false }, + "epoch-b", + ); + liveEpoch = "epoch-b"; + await coordinator.register( + session, + { emptyProject: false, mode: "resumed" }, + "epoch-b", + ); + const accepted = await coordinator.enqueueWithReceipt( + session.id, + "build on runtime B", + "request-runtime-b", + ); + expect(accepted.receipt.status).toBe("submitted"); + expect(submitted.map((entry) => entry.text)).toEqual([ + "build on runtime B", + ]); + + const beforeLateCompletion = await readState(root, session.id); + await coordinator.onEventPersisted( + analyticsEvent( + session.id, + "turn.completed", + { assistantText: "late runtime A reply" }, + "runtime-a-late-raw-turn", + ), + "epoch-a", + ); + await coordinator.onSessionStatus( + { ...session, status: "exited", ready: false }, + "epoch-a", + ); + expect(await readState(root, session.id)).toEqual(beforeLateCompletion); + }); + + it("returns an exited keyed receipt idempotently but denies new or rebound input", async () => { + session.projectBootstrap!.bootstrap = { + status: "delivered", + messageId: "bootstrap-complete", + }; + let liveEpoch: string | null = "epoch-a"; + let authorized = true; + manager.getRuntimeEpoch = () => liveEpoch; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + canDispatch: () => authorized, + generateId: () => "keyed-input", + }); + await coordinator.register( + session, + { emptyProject: false, mode: "created" }, + "epoch-a", + ); + await coordinator.enqueueWithReceipt( + session.id, + "durable keyed request", + "stable-request-id", + ); + liveEpoch = null; + session.status = "exited"; + session.ready = false; + await coordinator.onSessionStatus(session, "epoch-a"); + + expect(coordinator.ownsInput(session.id, "stable-request-id")).toBe(true); + const replay = await coordinator.enqueueWithReceipt( + session.id, + "durable keyed request", + "stable-request-id", + ); + expect(replay.receipt.status).toBe("uncertain"); + await expect( + coordinator.enqueueWithReceipt( + session.id, + "changed request", + "stable-request-id", + ), + ).rejects.toBeInstanceOf(ProjectBootstrapRequestIdConflictError); + await expect( + coordinator.enqueueWithReceipt( + session.id, + "new request", + "different-request-id", + ), + ).rejects.toBeInstanceOf(ProjectBootstrapDispatchForbiddenError); + + authorized = false; + await expect( + coordinator.enqueueWithReceipt( + session.id, + "durable keyed request", + "stable-request-id", + ), + ).rejects.toBeInstanceOf(ProjectBootstrapDispatchForbiddenError); + expect(submitted).toHaveLength(1); + }); + + it("fails closed on malformed current attempts, nonterminal FIFO state, and receipt reordering", async () => { + const cases: Array<{ id: string; state: Record }> = [ + { + id: "missing-attempts", + state: { + schemaVersion: 3, + metadata: { + ...structuredClone(session.projectBootstrap!), + bootstrap: { + status: "failed", + retryable: true, + errorCode: "persistence_failed", + }, + }, + inputs: [], + dispatchingInputId: null, + retryCount: 0, + emptyProject: true, + uncertainInputIds: [], + uncertainInputs: [], + receipts: [], + }, + }, + { + id: "nonterminal-fifo", + state: { + schemaVersion: 3, + metadata: { + ...structuredClone(session.projectBootstrap!), + queuedInputIds: ["queued-a"], + }, + inputs: [ + { + id: "queued-a", + sessionId: session.id, + text: "must not replay", + acceptedAt: NOW, + }, + ], + dispatchingInputId: null, + retryCount: 0, + emptyProject: true, + attempts: [], + uncertainInputIds: [], + uncertainInputs: [], + receipts: [ + { + requestId: "queued-a-request", + inputId: "queued-a", + status: "queued", + acceptedAt: NOW, + payloadDigest: createHash("sha256") + .update( + JSON.stringify({ + schemaVersion: 1, + submit: true, + text: "must not replay", + }), + ) + .digest("hex"), + }, + ], + }, + }, + { + id: "receipt-after-live-fifo", + state: { + schemaVersion: 3, + metadata: { + ...structuredClone(session.projectBootstrap!), + bootstrap: { status: "delivered", messageId: "bootstrap-done" }, + queuedInputIds: ["queued-a"], + }, + inputs: [ + { + id: "queued-a", + sessionId: session.id, + text: "must remain fenced", + acceptedAt: NOW, + }, + ], + dispatchingInputId: null, + retryCount: 0, + emptyProject: false, + attempts: [], + uncertainInputIds: [], + uncertainInputs: [], + receipts: [ + { + requestId: "queued-a-request", + inputId: "queued-a", + status: "queued", + acceptedAt: NOW, + payloadDigest: projectBootstrapInputDigestForTest( + "must remain fenced", + ), + }, + { + requestId: "later-request", + inputId: "later-submitted", + status: "completed", + acceptedAt: NOW, + payloadDigest: projectBootstrapInputDigestForTest("later"), + }, + ], + }, + }, + ]; + + for (const testCase of cases) { + const target = projectSession(`session-${testCase.id}`); + const raw = structuredClone(testCase.state); + const metadata = (raw.metadata ?? {}) as Record; + metadata.projectId = PROJECT_ID; + metadata.userId = USER_ID; + metadata.targetSessionId = target.id; + if (Array.isArray(raw.inputs)) { + for (const input of raw.inputs as Array>) { + input.sessionId = target.id; + } + } + sessions.set(target.id, target); + await fs.mkdir(path.dirname(stateFile(root, target.id)), { + recursive: true, + }); + await fs.writeFile( + stateFile(root, target.id), + `${JSON.stringify(raw)}\n`, + ); + const original = await fs.readFile(stateFile(root, target.id), "utf8"); + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + }); + await expect( + coordinator.register( + target, + { emptyProject: true, mode: "boot" }, + TEST_RUNTIME_EPOCH, + ), + ).rejects.toThrow("project bootstrap state is unavailable"); + expect(await fs.readFile(stateFile(root, target.id), "utf8")).toBe( + original, + ); + } + }); + + it("emits neutral content-free lifecycle events and redacts local hook content", async () => { + vi.useFakeTimers(); + const lifecycle: ProjectBootstrapLifecycleEvent[] = []; + manager.submitInput = async () => { + throw new Error("provider said /private/customer secret-token"); + }; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => "attempt-telemetry", + onEvent: (event) => { + lifecycle.push(event); + }, + }); + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + + expect(lifecycle.map((event) => event.name)).toEqual([ + "project_bootstrap.scheduled", + "project_bootstrap.attempted", + ]); + await vi.advanceTimersByTimeAsync(300_000); + await flushCoordinator(coordinator, session.id); + expect(lifecycle.map((event) => event.name)).toEqual([ + "project_bootstrap.scheduled", + "project_bootstrap.attempted", + "project_bootstrap.failed", + ]); + const serializedLifecycle = JSON.stringify(lifecycle); + expect(serializedLifecycle).not.toMatch(/planner|builder/i); + expect(serializedLifecycle).not.toContain("Agent Studio project bootstrap"); + expect(serializedLifecycle).not.toContain("private/customer"); + expect(serializedLifecycle).not.toContain("secret-token"); + + session = projectSession("session-telemetry-redaction"); + sessions.set(session.id, session); + submitted = []; + manager.submitInput = async (id: string, text: string) => { + submitted.push({ sessionId: id, text, submit: true, background: true }); + return true; + }; + const redactor = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + generateId: () => "attempt-local-only", + }); + await redactor.register(session, { emptyProject: true, mode: "created" }); + const local = redactor.decorateLocalEvent( + analyticsEvent(session.id, "prompt.submitted", { + prompt: submitted[0]!.text, + path: "/private/source.ts", + connectorPayload: "secret connector body", + credential: "sk-cutover-secret", + compiledBrief: "private focused brief body", + }), + ); + expect(local.payload.prompt).toBe(submitted[0]!.text); + expect(local.payload.projectBootstrapAttemptId).toBe("attempt-local-only"); + + const remotePrompt = redactor.redactForTelemetry(local); + expect(remotePrompt.agentSessionId).toBeNull(); + expect(remotePrompt.payload).toEqual({ + projectBootstrap: true, + origin: "infrastructure", + projectBootstrapAttemptId: "attempt-local-only", + }); + expect(JSON.stringify(remotePrompt)).not.toContain("private/source"); + expect(JSON.stringify(remotePrompt)).not.toContain("connector body"); + expect(JSON.stringify(remotePrompt)).not.toContain("sk-cutover-secret"); + expect(JSON.stringify(remotePrompt)).not.toContain("focused brief body"); + + const remoteTurn = redactor.redactForTelemetry( + analyticsEvent(session.id, "turn.completed", { + assistantText: "raw provider output secret", + model: "covert-provider-channel", + usage: { inputTokens: 10, outputTokens: 20 }, + sourceText: "customer source", + }), + ); + expect(remoteTurn.payload).toEqual({ + projectBootstrap: true, + hasAssistantText: true, + modelReported: true, + usage: { inputTokens: 10, outputTokens: 20 }, + }); + expect(JSON.stringify(remoteTurn)).not.toContain("provider output"); + expect(JSON.stringify(remoteTurn)).not.toContain("covert-provider"); + expect(JSON.stringify(remoteTurn)).not.toContain("customer source"); + + session = projectSession("session-ordinary-after-bootstrap"); + session.projectBootstrap!.bootstrap = { + status: "delivered", + messageId: "message-bootstrap-complete", + }; + sessions.set(session.id, session); + const ordinaryEvent = analyticsEvent(session.id, "prompt.submitted", { + prompt: "ordinary project work", + }); + expect(redactor.redactForTelemetry(ordinaryEvent)).toEqual(ordinaryEvent); + }); + + it("closes timer and work admission without allowing a late lifecycle transition", async () => { + vi.useFakeTimers(); + session.ready = false; + const lifecycle: ProjectBootstrapLifecycleEvent[] = []; + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + readinessTimeoutMs: 10, + onEvent: (event) => { + lifecycle.push(event); + }, + }); + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + expect( + (coordinator as unknown as { timers: Map }).timers.size, + ).toBe(1); + + await coordinator.close(); + await coordinator.close(); + expect( + (coordinator as unknown as { timers: Map }).timers.size, + ).toBe(0); + await vi.advanceTimersByTimeAsync(20); + expect( + lifecycle.filter((event) => event.name === "project_bootstrap.failed"), + ).toEqual([]); + await expect(coordinator.retry(session.id)).rejects.toBeInstanceOf( + ProjectBootstrapCoordinatorClosedError, + ); + await expect( + coordinator.enqueue(session.id, "must not be admitted"), + ).rejects.toBeInstanceOf(ProjectBootstrapCoordinatorClosedError); + await expect( + coordinator.scheduleProject(PROJECT_ID, USER_ID), + ).rejects.toBeInstanceOf(ProjectBootstrapCoordinatorClosedError); + }); + + it("tracks an in-flight status authorization check and clears all lifecycle state on close", async () => { + session.ready = false; + let blockAuthorization = false; + let authorizationStarted!: () => void; + const started = new Promise((resolve) => { + authorizationStarted = resolve; + }); + let releaseAuthorization!: () => void; + const released = new Promise((resolve) => { + releaseAuthorization = resolve; + }); + const coordinator = new ProjectBootstrapCoordinator({ + root, + sessionManager: manager, + readinessTimeoutMs: 60_000, + canDispatch: async () => { + if (!blockAuthorization) return true; + authorizationStarted(); + await released; + return true; + }, + }); + await coordinator.register(session, { + emptyProject: true, + mode: "created", + }); + session.ready = true; + blockAuthorization = true; + const status = coordinator.onSessionStatus(session); + await started; + + let closeSettled = false; + const closing = coordinator.close().then(() => { + closeSettled = true; + }); + await Promise.resolve(); + expect(closeSettled).toBe(false); + releaseAuthorization(); + await Promise.all([status, closing]); + expect(closeSettled).toBe(true); + + const internals = coordinator as unknown as Record< + string, + Map | Set + >; + for (const key of [ + "states", + "writes", + "expected", + "observedAttempts", + "activeTurns", + "correlationOverflow", + "timers", + "activeTurnTimers", + "blockingInputRedrainTimers", + "provisionalProjectClaims", + "provisionalSessionClaims", + "pendingApiPreemptions", + "registeredSessions", + "terminalPreemptions", + "reportedTerminalPreemptions", + ]) { + expect(internals[key]?.size, key).toBe(0); + } + }); +}); + +describe("projectBootstrapPrompt", () => { + it("is evidence-first, tool-capable, role-neutral, and direct-build safe", () => { + const prompt = projectBootstrapPrompt(); + + expect(prompt).toContain("Read the current Agent Map first"); + expect(prompt).toContain("meaningfully empty"); + expect(prompt).toContain("explicit evidence"); + expect(prompt).toContain("structured Agent Map tools"); + expect(prompt).toContain("Validate before proposing"); + expect(prompt).toContain("Never guess"); + expect(prompt).toContain("prioritize"); + expect(prompt).toContain("proceed directly with implementation"); + expect(prompt).toContain("no confirmation or mode transition is required"); + expect(prompt).not.toMatch( + /map-planner|agent-builder|planning-only|no-code/i, + ); + }); + + it("makes retries uniquely correlatable without exposing authority in the prompt", () => { + const first = projectBootstrapPrompt(0, "attempt-1"); + const retry = projectBootstrapPrompt(1, "attempt-2"); + + expect(first).not.toBe(retry); + expect(first).toContain("Internal correlation key: attempt-1"); + expect(retry).toContain("Internal correlation key: attempt-2"); + expect(retry).toContain("automatic retry 1 of 2"); + expect(retry).toContain("Never repeat or expose this key"); + expect(first).not.toMatch(/projectId|userId|sessionId|capability|bearer/i); + }); +}); diff --git a/packages/harness/src/core/project-bootstrap.ts b/packages/harness/src/core/project-bootstrap.ts new file mode 100644 index 00000000..d57ba6cf --- /dev/null +++ b/packages/harness/src/core/project-bootstrap.ts @@ -0,0 +1,3277 @@ +import { + ProjectBootstrapStore, + ProjectBootstrapDispatchForbiddenError, + ProjectBootstrapCoordinatorClosedError, + MAX_RETRIES, + compactInputReceipts, + projectBootstrapInputDigest, + isRecord, + isTerminal, +} from "./project-bootstrap-store.js"; +import type { + ProjectBootstrapAttemptPhase, + PersistedProjectBootstrapInputReceipt, + PersistedProjectBootstrapState, +} from "./project-bootstrap-store.js"; +export { + ProjectBootstrapDispatchForbiddenError, + ProjectBootstrapCoordinatorClosedError, + ProjectBootstrapInputCapacityError, +} from "./project-bootstrap-store.js"; +import { randomUUID } from "node:crypto"; + +import type { + ProjectBootstrapErrorCode, + ProjectBootstrapLifecycleEvent, + ProjectBootstrapInputReceipt, + ProjectBootstrapQueuedInput, + ProjectBootstrapMetadata, +} from "../shared/agent-map.js"; +import type { AnalyticsEvent, HarnessSession } from "../shared/types.js"; +import { + SessionBackgroundInputPreemptedError, + SessionInputGuardRejectedError, + SessionManager, + SessionNotReadyError, +} from "./session-manager.js"; + +interface ExpectedPrompt { + kind: "bootstrap" | "user"; + id: string; + text: string; + /** A failed/timed-out greeting stays as a FIFO tombstone so a late hook + * cannot be mistaken for a later retry. */ + retired?: boolean; +} + +type ObservedProjectTurn = + | { kind: "bootstrap"; id: string; retired: boolean } + | { kind: "user"; id: string } + | { kind: "external" }; + +type ActiveCoordinatorTurn = + | { kind: "bootstrap"; id: string } + | { kind: "user"; id: string }; + +interface AttemptTimer { + key: "pending" | string; + runtimeEpoch: string; + handle: ReturnType; +} + +interface ActiveTurnTimer { + turn: ActiveCoordinatorTurn; + runtimeEpoch: string; + handle: ReturnType; +} + +type ProjectBootstrapDrainOutcome = + | "progressed" + | "empty" + | "owned" + | "not-runnable" + | "authorization-denied" + | "transient-failure"; + +interface BootstrapFailureTransitionObligation { + attemptId: string; + errorCode: ProjectBootstrapErrorCode; + retryable: boolean; + correlationRelease: "remove" | "tombstone" | "consume-observed-or-tombstone"; +} + +interface PendingBootstrapFailureTransitionObligation { + errorCode: ProjectBootstrapErrorCode; + retryable: boolean; +} + +export type ProjectBootstrapRegistrationMode = + | "boot" + | "created" + | "live" + | "resumed"; + +export interface ProjectBootstrapRegistrationContext { + emptyProject: boolean; + mode: ProjectBootstrapRegistrationMode; +} + +export interface ProjectBootstrapCoordinatorOptions { + root: string; + /** @deprecated Read-only migration source for pre-SAP-3148 queue files. */ + legacyStateRoot?: string; + sessionManager: SessionManager; + now?: () => string; + generateId?: () => string; + /** Maximum wait for the target session to become interactive. */ + readinessTimeoutMs?: number; + /** Maximum wait for the model turn after the request reaches the PTY. */ + deliveryTimeoutMs?: number; + /** Test seam for classifying queue-store failures without exposing raw errors. */ + writeState?: (file: string, state: unknown) => Promise; + /** Test seam for accepted-ledger cleanup/commit failures. */ + writeAcceptedLedger?: (file: string, state: unknown) => Promise; + /** Live authorization gate checked immediately before every PTY dispatch. */ + canDispatch?: (session: HarnessSession) => boolean | Promise; + /** Rechecks E2 semantic state immediately before the bootstrap attempt. */ + isMeaningfullyEmpty?: (projectId: string) => boolean | Promise; + onEvent?: (event: ProjectBootstrapLifecycleEvent) => Promise | void; +} + +export class ProjectBootstrapRetryUnavailableError extends Error { + readonly code = "project_bootstrap_retry_unavailable"; + + constructor() { + super("project bootstrap retry is not available"); + this.name = "ProjectBootstrapRetryUnavailableError"; + } +} + +export class ProjectBootstrapRequestIdConflictError extends Error { + readonly code = "project_bootstrap_request_id_reused"; + + constructor() { + super("project bootstrap request id was reused with different input"); + this.name = "ProjectBootstrapRequestIdConflictError"; + } +} +const MAX_CORRELATION_BARRIERS = 256; +const MAX_COMPLETION_EVENT_RECEIPTS = 256; + +function hasSafeBootstrapRetryEvidence( + state: PersistedProjectBootstrapState, +): boolean { + const bootstrap = state.metadata.bootstrap; + if (bootstrap.status !== "failed" || !bootstrap.retryable) return false; + const latest = state.attempts.at(-1); + if (!latest) { + // Readiness and current-schema persistence can fail before an attempt is + // allocated. Legacy unsafe combinations are normalized non-retryable. + return ( + bootstrap.errorCode === "session_not_ready" || + bootstrap.errorCode === "persistence_failed" + ); + } + if (latest.status !== "retired") return false; + if (latest.phase === "claimed") { + return ( + bootstrap.errorCode === "session_not_ready" || + bootstrap.errorCode === "injection_failed" || + bootstrap.errorCode === "persistence_failed" + ); + } + if (latest.phase === "not-submitted") { + return ( + bootstrap.errorCode === "injection_failed" || + bootstrap.errorCode === "persistence_failed" + ); + } + return ( + latest.phase === "submitted" && + (bootstrap.errorCode === "delivery_timeout" || + bootstrap.errorCode === "model_turn_failed") + ); +} + +/** + * Whether the bootstrap coordinator still owns submitted user input. Once its + * FIFO is empty, the ordinary SessionManager input path resumes ownership. + */ +export function projectBootstrapOwnsInput( + metadata: ProjectBootstrapMetadata | null | undefined, +): boolean { + return Boolean( + metadata && (!isTerminal(metadata) || metadata.queuedInputIds.length > 0), + ); +} + +export function projectBootstrapPrompt( + retryOrdinal = 0, + attemptId?: string, +): string { + const suffix = + retryOrdinal > 0 + ? ` This is automatic retry ${Math.min(retryOrdinal, MAX_RETRIES)} of ${MAX_RETRIES}.` + : ""; + const correlation = attemptId + ? ` Internal correlation key: ${attemptId}. Never repeat or expose this key.` + : ""; + return `Agent Studio project bootstrap: Read the current Agent Map first. Only if it is still meaningfully empty, inspect available project context for explicit evidence of agents, meaningful subagents, responsibilities, contracts, resources, connectors, artifacts, and cross-agent data flow. Validate before proposing one honest initial map with the structured Agent Map tools. Never guess, invent placeholder nodes, or overwrite concurrent work; on conflict, reread and reconcile. Summarize what the evidence supports and clearly identify uncertainty. This bootstrap is ordinary project work: if a real user request is present, prioritize it and proceed directly with implementation when it is build-ready; no confirmation or mode transition is required.${suffix}${correlation}`; +} + +const PROJECT_SESSION_SOURCES = new Set([ + "startup", + "resume", + "clear", + "compact", + "codex", +]); +const MAX_TELEMETRY_TOKEN_COUNT = 1_000_000_000_000; + +function telemetrySource(value: unknown): string { + return typeof value === "string" && PROJECT_SESSION_SOURCES.has(value) + ? value + : "unknown"; +} + +function telemetryTokenCount(value: unknown): number | null { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + return null; + } + return Math.min(Math.trunc(value), MAX_TELEMETRY_TOKEN_COUNT); +} + +function telemetryUsage(value: unknown): Record | null { + if (!isRecord(value)) return null; + const inputTokens = telemetryTokenCount(value.inputTokens); + const outputTokens = telemetryTokenCount(value.outputTokens); + if (inputTokens === null && outputTokens === null) return null; + return { inputTokens, outputTokens }; +} + +function telemetryPayload(event: AnalyticsEvent): Record { + switch (event.type) { + case "session.start": + return { + source: telemetrySource(event.payload.source), + projectBootstrap: true, + }; + case "prompt.submitted": + return { + projectBootstrap: true, + origin: event.payload.projectBootstrapOrigin ?? "user", + ...(typeof event.payload.projectBootstrapInputId === "string" + ? { projectBootstrapInputId: event.payload.projectBootstrapInputId } + : {}), + ...(typeof event.payload.projectBootstrapAttemptId === "string" + ? { + projectBootstrapAttemptId: + event.payload.projectBootstrapAttemptId, + } + : {}), + }; + case "tool.call": + return { projectBootstrap: true, toolObserved: true }; + case "turn.completed": + return { + projectBootstrap: true, + hasAssistantText: + typeof event.payload.assistantText === "string" && + event.payload.assistantText.length > 0, + // Provider/model text is never remotely projected. Even a syntactically + // plausible model identifier is an attacker-controlled covert channel. + modelReported: + typeof event.payload.model === "string" && + event.payload.model.length > 0, + usage: telemetryUsage(event.payload.usage), + }; + default: + return { projectBootstrap: true }; + } +} + +export class ProjectBootstrapCoordinator extends ProjectBootstrapStore { + private readonly generateId: () => string; + private readonly readinessTimeoutMs: number; + private readonly deliveryTimeoutMs: number; + /** Exact server-issued PTY generation currently allowed to own volatile + * correlation, input holds, timers, and completion dedupe state. */ + private readonly runtimeEpochs = new Map(); + private readonly expected = new Map(); + private readonly observedAttempts = new Map(); + /** Exact normalized completion IDs already applied in this live ingest + * epoch. The ingest pipeline never replays archived events into a newly + * constructed coordinator, so restart is the safe epoch boundary. Separate + * provider Stop invocations receive distinct IDs and carry no stable turn + * token; those intrinsically indistinguishable events are handled only by + * conservative correlation barriers/timeouts, never text/timing guesses. */ + private readonly processedCompletionEvents = new Map(); + /** Once the fixed exact-ID window is full, stop trusting completion events + * for the remainder of this live coordinator epoch. Ingest callbacks are + * detached from HTTP processing and therefore have no finite delay bound; + * evicting an old ID would make its replay capable of completing a newer + * turn. Timeouts preserve FIFO progress until restart establishes a fresh + * event epoch. */ + private readonly completionDedupeOverflow = new Set(); + /** At most one coordinator-owned prompt may have crossed Enter without a + * correlated turn completion. This is deliberately separate from durable + * FIFO acceptance: accepted user input remains durable even while its turn + * temporarily owns the live CLI. */ + private readonly activeTurns = new Map(); + private readonly correlationOverflow = new Set(); + private readonly timers = new Map(); + private readonly activeTurnTimers = new Map(); + /** Exact persistence-only transition retained when an active bootstrap + * failure cannot commit. Its retry never writes prompt bytes and preserves + * the original public error classification. */ + private readonly bootstrapFailureTransitions = new Map< + string, + BootstrapFailureTransitionObligation + >(); + private readonly pendingBootstrapFailureTransitions = new Map< + string, + PendingBootstrapFailureTransitionObligation + >(); + /** Positive no-Enter user submissions whose dispatch-marker rollback still + * needs to commit. The active owner remains until this persistence-only + * obligation succeeds; its retry never writes prompt bytes. */ + private readonly userNotSubmittedTransitions = new Map(); + private readonly blockingInputRedrainTimers = new Map< + string, + ReturnType + >(); + /** Durable work that lost its final external/status wakeup to a transient + * local persistence/load failure. Only this classification may poll on a + * bounded timer; authorization denial explicitly clears it. */ + private readonly inputRedrainNeeded = new Set(); + /** Synchronous API-arrival signal used to cancel a staged background Enter + * before the durable FIFO operation reaches this coordinator's lock. */ + private readonly pendingApiPreemptions = new Map< + string, + { runtimeEpoch: string; count: number } + >(); + /** Status hooks can race a freshly spawned PTY ahead of registration. */ + private readonly registeredSessions = new Set(); + /** Set synchronously by raw terminal input, including before registration. */ + private readonly terminalPreemptions = new Set(); + /** Raw input creates a durable user-proceeded obligation independently of + * the raw model-turn hold. A completion may release the latter but never the + * former before `skipped` is committed. */ + private readonly terminalPreemptionObligations = new Set(); + /** Blocking trust/login input releases its raw hold after the durable + * user-proceeded transition; ordinary terminal input keeps the hold until a + * correlated external completion. */ + private readonly blockingTerminalPreemptions = new Set(); + private readonly terminalPreemptionRetryTimers = new Map< + string, + ReturnType + >(); + private readonly reportedTerminalPreemptions = new Set(); + private admissionGeneration = 0; + + constructor(private readonly options: ProjectBootstrapCoordinatorOptions) { + super(options); + this.generateId = options.generateId ?? randomUUID; + this.readinessTimeoutMs = + options.readinessTimeoutMs ?? options.deliveryTimeoutMs ?? 45_000; + this.deliveryTimeoutMs = options.deliveryTimeoutMs ?? 300_000; + } + + private isAdmissionCurrent(generation: number): boolean { + return !this.closed && this.admissionGeneration === generation; + } + + private async canDispatch( + session: HarnessSession, + generation = this.admissionGeneration, + ): Promise { + if (!this.isAdmissionCurrent(generation)) return false; + try { + const allowed = (await this.options.canDispatch?.(session)) ?? true; + return allowed && this.isAdmissionCurrent(generation); + } catch { + return false; + } + } + + private isRuntimeEpochCurrent( + sessionId: string, + runtimeEpoch: string, + ): boolean { + return !this.closed && this.runtimeEpochs.get(sessionId) === runtimeEpoch; + } + + /** Current trusted live epoch for server-originated API/retry actions. */ + private currentRuntimeEpoch(sessionId: string): string | null { + const runtimeEpoch = this.options.sessionManager.getRuntimeEpoch(sessionId); + return runtimeEpoch !== null && + this.isRuntimeEpochCurrent(sessionId, runtimeEpoch) + ? runtimeEpoch + : null; + } + + private hasPendingApiInput( + sessionId: string, + runtimeEpoch = this.runtimeEpochs.get(sessionId), + ): boolean { + const pending = this.pendingApiPreemptions.get(sessionId); + return Boolean( + runtimeEpoch && + pending?.runtimeEpoch === runtimeEpoch && + pending.count > 0, + ); + } + + private notePendingApiInput(sessionId: string, runtimeEpoch: string): void { + if (!this.isRuntimeEpochCurrent(sessionId, runtimeEpoch)) return; + const pending = this.pendingApiPreemptions.get(sessionId); + this.pendingApiPreemptions.set(sessionId, { + runtimeEpoch, + count: pending?.runtimeEpoch === runtimeEpoch ? pending.count + 1 : 1, + }); + } + + private clearPendingApiInput(sessionId: string, runtimeEpoch: string): void { + const pending = this.pendingApiPreemptions.get(sessionId); + if (pending?.runtimeEpoch !== runtimeEpoch) return; + const remaining = pending.count - 1; + if (remaining <= 0) { + this.pendingApiPreemptions.delete(sessionId); + // Releasing the final API-admission fence is itself a progress edge. A + // transient drain failure may have tried to schedule recovery while this + // counter still suppressed it. + this.scheduleInputRedrain(sessionId, runtimeEpoch); + } else { + this.pendingApiPreemptions.set(sessionId, { + runtimeEpoch, + count: remaining, + }); + } + } + + private hasInputHold(sessionId: string): boolean { + return ( + this.activeTurns.has(sessionId) || this.terminalPreemptions.has(sessionId) + ); + } + + /** + * Whether the single durable input authority must handle this request. + * Ownership extends through the whole coordinated model turn, even after + * its FIFO row has been dequeued. A known request ID also routes back here + * so response-loss retries resolve to the original durable receipt. + */ + ownsInput(sessionId: string, requestId?: string): boolean { + const session = this.options.sessionManager.get(sessionId); + const state = this.states.get(sessionId); + return Boolean( + projectBootstrapOwnsInput(session?.projectBootstrap) || + state?.dispatchingInputId || + this.activeTurns.has(sessionId) || + this.hasPendingApiInput(sessionId) || + (requestId && + state?.receipts.some((receipt) => receipt.requestId === requestId)), + ); + } + + private inputPayloadDigest(text: string): string { + return projectBootstrapInputDigest(text); + } + + private publicReceipt( + receipt: PersistedProjectBootstrapInputReceipt, + ): ProjectBootstrapInputReceipt { + return { + requestId: receipt.requestId, + inputId: receipt.inputId, + status: receipt.status, + acceptedAt: receipt.acceptedAt, + }; + } + + private clearActiveTurn( + sessionId: string, + kind: ActiveCoordinatorTurn["kind"], + id: string, + ): void { + const active = this.activeTurns.get(sessionId); + if (active?.kind === kind && active.id === id) { + this.activeTurns.delete(sessionId); + const timer = this.activeTurnTimers.get(sessionId); + if (timer?.turn.kind === kind && timer.turn.id === id) { + clearTimeout(timer.handle); + this.activeTurnTimers.delete(sessionId); + } + } + } + + /** Drop only volatile ownership belonging to one proven PTY generation. */ + private clearRuntimeEpochState( + sessionId: string, + runtimeEpoch: string, + ): void { + if (this.runtimeEpochs.get(sessionId) !== runtimeEpoch) return; + this.clearTimer(sessionId); + const activeTimer = this.activeTurnTimers.get(sessionId); + if (activeTimer) clearTimeout(activeTimer.handle); + this.activeTurnTimers.delete(sessionId); + const redrain = this.blockingInputRedrainTimers.get(sessionId); + if (redrain) clearTimeout(redrain); + this.blockingInputRedrainTimers.delete(sessionId); + const preemptionRetry = this.terminalPreemptionRetryTimers.get(sessionId); + if (preemptionRetry) clearTimeout(preemptionRetry); + this.terminalPreemptionRetryTimers.delete(sessionId); + this.expected.delete(sessionId); + this.observedAttempts.delete(sessionId); + this.processedCompletionEvents.delete(sessionId); + this.completionDedupeOverflow.delete(sessionId); + this.activeTurns.delete(sessionId); + this.correlationOverflow.delete(sessionId); + this.bootstrapFailureTransitions.delete(sessionId); + this.pendingBootstrapFailureTransitions.delete(sessionId); + this.userNotSubmittedTransitions.delete(sessionId); + this.inputRedrainNeeded.delete(sessionId); + this.pendingApiPreemptions.delete(sessionId); + this.registeredSessions.delete(sessionId); + this.terminalPreemptions.delete(sessionId); + this.terminalPreemptionObligations.delete(sessionId); + this.blockingTerminalPreemptions.delete(sessionId); + this.reportedTerminalPreemptions.delete(sessionId); + this.runtimeEpochs.delete(sessionId); + } + + private async resolveUncertainDispatch( + state: PersistedProjectBootstrapState, + ): Promise { + const inputId = state.dispatchingInputId; + if (inputId === null || state.inputs[0]?.id !== inputId) return state; + const input = state.inputs[0]; + if (!input) return state; + const resolved: PersistedProjectBootstrapState = { + ...structuredClone(state), + inputs: state.inputs.slice(1), + dispatchingInputId: null, + metadata: { + ...structuredClone(state.metadata), + queuedInputIds: state.metadata.queuedInputIds.slice(1), + }, + uncertainInputIds: [...state.uncertainInputIds, inputId], + uncertainInputs: [...state.uncertainInputs, structuredClone(input)], + receipts: structuredClone(state.receipts), + }; + this.updateReceiptStatus(resolved, inputId, "uncertain"); + await this.persist(state.metadata.targetSessionId, resolved); + this.emit({ + name: "project_bootstrap.input_delivery_uncertain", + projectId: state.metadata.projectId, + sessionId: state.metadata.targetSessionId, + inputId, + errorCode: "delivery_uncertain", + queueDepth: state.inputs.length, + }); + return resolved; + } + + private async normalizeBootInputState( + state: PersistedProjectBootstrapState, + ): Promise { + const sessionId = state.metadata.targetSessionId; + const accepted = await this.acceptedInputIds(sessionId); + if (accepted === null) + return this.terminalizeUnreadableAcceptedLedger(state); + const receiptIndexByInput = new Map( + state.receipts.map((receipt, index) => [receipt.inputId, index]), + ); + let lastUnsafeReceiptIndex = -1; + state.receipts.forEach((receipt, index) => { + if ( + receipt.status === "submitted" || + receipt.status === "uncertain" || + receipt.status === "completed" || + accepted.has(receipt.inputId) || + state.dispatchingInputId === receipt.inputId + ) { + lastUnsafeReceiptIndex = index; + } + }); + // A terminal receipt that no longer has a content row is still positive + // evidence that a later logical turn crossed the PTY. Every earlier live + // FIFO row belongs to the same causal prefix and cannot be replayed after + // restart without reversing arrival order or duplicating work. + let lastUnsafeIndex = -1; + state.inputs.forEach((input, index) => { + const receiptIndex = receiptIndexByInput.get(input.id); + if ( + receiptIndex !== undefined && + receiptIndex <= lastUnsafeReceiptIndex + ) { + lastUnsafeIndex = index; + } + }); + + const terminal = structuredClone(state); + const uncertainById = new Map( + terminal.uncertainInputs.map((input) => [input.id, input]), + ); + // If a later FIFO row is positively accepted or already marked submitted, + // every earlier row belongs to the same unresolved delivery prefix. A new + // process cannot replay that prefix without reversing durable arrival + // order or duplicating a turn, so terminalize it in FIFO order. + for (const input of terminal.inputs.slice(0, lastUnsafeIndex + 1)) { + uncertainById.set(input.id, structuredClone(input)); + this.updateReceiptStatus(terminal, input.id, "uncertain"); + } + terminal.inputs = terminal.inputs.slice(lastUnsafeIndex + 1); + terminal.metadata.queuedInputIds = terminal.inputs.map((input) => input.id); + if (lastUnsafeIndex >= 0) terminal.dispatchingInputId = null; + + // Submitted receipts whose payload row was already durably removed still + // cannot recover live completion correlation in a new process. Likewise, + // accepted-ledger proof is submission acknowledgement, not completion. + for (const receipt of terminal.receipts) { + if ( + receipt.status === "submitted" || + (accepted.has(receipt.inputId) && + receipt.status !== "completed" && + receipt.status !== "uncertain") + ) { + receipt.status = "uncertain"; + } + } + terminal.uncertainInputs = [...uncertainById.values()]; + terminal.uncertainInputIds = terminal.uncertainInputs.map( + (input) => input.id, + ); + + const changed = JSON.stringify(terminal) !== JSON.stringify(state); + if (changed) { + await this.persist(sessionId, terminal); + for (const input of state.inputs.slice(0, lastUnsafeIndex + 1)) { + this.emit({ + name: "project_bootstrap.input_delivery_uncertain", + projectId: state.metadata.projectId, + sessionId, + inputId: input.id, + errorCode: "delivery_uncertain", + queueDepth: state.inputs.length, + }); + } + } + // Never discard positive PTY acknowledgement until the canonical + // non-replayable state is durable. Cleanup is retry-safe and may lag. + if (accepted.size > 0) + await this.writeAcceptedInputIds(sessionId, []).catch(() => {}); + return terminal; + } + + /** Stop timers and settle all queued persistence before server teardown. */ + async close(): Promise { + if (this.closed) return; + this.closed = true; + this.admissionGeneration += 1; + for (const { handle } of this.timers.values()) clearTimeout(handle); + this.timers.clear(); + for (const { handle } of this.activeTurnTimers.values()) + clearTimeout(handle); + this.activeTurnTimers.clear(); + this.bootstrapFailureTransitions.clear(); + this.pendingBootstrapFailureTransitions.clear(); + this.userNotSubmittedTransitions.clear(); + for (const handle of this.blockingInputRedrainTimers.values()) + clearTimeout(handle); + this.blockingInputRedrainTimers.clear(); + this.inputRedrainNeeded.clear(); + while (this.writes.size > 0) { + await Promise.allSettled([...this.writes.values()]); + } + // An operation that was already between awaits when close began must not + // leave any late timer or correlation state behind. + for (const { handle } of this.timers.values()) clearTimeout(handle); + this.timers.clear(); + for (const { handle } of this.activeTurnTimers.values()) + clearTimeout(handle); + this.activeTurnTimers.clear(); + for (const handle of this.blockingInputRedrainTimers.values()) + clearTimeout(handle); + this.blockingInputRedrainTimers.clear(); + this.expected.clear(); + this.observedAttempts.clear(); + this.processedCompletionEvents.clear(); + this.completionDedupeOverflow.clear(); + this.activeTurns.clear(); + this.correlationOverflow.clear(); + this.pendingApiPreemptions.clear(); + this.provisionalProjectClaims.clear(); + this.provisionalSessionClaims.clear(); + this.states.clear(); + this.registeredSessions.clear(); + this.terminalPreemptions.clear(); + this.terminalPreemptionObligations.clear(); + this.blockingTerminalPreemptions.clear(); + for (const handle of this.terminalPreemptionRetryTimers.values()) + clearTimeout(handle); + this.terminalPreemptionRetryTimers.clear(); + this.reportedTerminalPreemptions.clear(); + this.runtimeEpochs.clear(); + } + + private clearTimer(sessionId: string, key?: string): void { + const timer = this.timers.get(sessionId); + if (!timer || (key !== undefined && timer.key !== key)) return; + clearTimeout(timer.handle); + this.timers.delete(sessionId); + } + + private needsLifecycleTimer( + sessionId: string, + key: "pending" | string, + runtimeEpoch: string, + ): boolean { + if (!this.isRuntimeEpochCurrent(sessionId, runtimeEpoch)) return false; + if ( + key === "pending" && + this.pendingBootstrapFailureTransitions.has(sessionId) + ) { + return true; + } + if ( + key !== "pending" && + this.bootstrapFailureTransitions.get(sessionId)?.attemptId === key + ) { + return true; + } + const state = this.states.get(sessionId); + const bootstrap = + state?.metadata.bootstrap ?? + this.options.sessionManager.get(sessionId)?.projectBootstrap?.bootstrap; + return key === "pending" + ? bootstrap?.status === "pending" + : bootstrap?.status === "generating" && bootstrap.attemptId === key; + } + + private armTimer( + sessionId: string, + key: "pending" | string, + runtimeEpoch = this.runtimeEpochs.get(sessionId), + ): void { + if ( + runtimeEpoch === undefined || + !this.isRuntimeEpochCurrent(sessionId, runtimeEpoch) + ) + return; + const existing = this.timers.get(sessionId); + if (existing?.key === key && existing.runtimeEpoch === runtimeEpoch) return; + this.clearTimer(sessionId); + const handle = setTimeout( + () => { + const fired = this.timers.get(sessionId); + if ( + fired?.handle !== handle || + !this.isRuntimeEpochCurrent(sessionId, runtimeEpoch) + ) + return; + this.timers.delete(sessionId); + void this.fail( + sessionId, + key, + key === "pending" ? "session_not_ready" : "delivery_timeout", + key === "pending", + runtimeEpoch, + ).catch(() => { + if (this.needsLifecycleTimer(sessionId, key, runtimeEpoch)) + this.armTimer(sessionId, key, runtimeEpoch); + // Timer callbacks have no request boundary to receive a rejection. + // persist() already projects/emits `persistence_failed` wherever one + // durable store remains; log only a fixed local classification here, + // never the provider/storage error or planner content. + console.error( + "[harness] project bootstrap timeout transition failed: persistence_failed", + ); + }); + }, + key === "pending" ? this.readinessTimeoutMs : this.deliveryTimeoutMs, + ); + handle.unref?.(); + this.timers.set(sessionId, { key, runtimeEpoch, handle }); + } + + private armActiveTurnTimer( + sessionId: string, + turn: ActiveCoordinatorTurn, + runtimeEpoch = this.runtimeEpochs.get(sessionId), + ): void { + if ( + runtimeEpoch === undefined || + !this.isRuntimeEpochCurrent(sessionId, runtimeEpoch) + ) + return; + const active = this.activeTurns.get(sessionId); + // A very fast provider can persist completion before submitInput returns. + // Never install a timer after that completion already released ownership. + if (active?.kind !== turn.kind || active.id !== turn.id) return; + const existing = this.activeTurnTimers.get(sessionId); + if ( + existing?.turn.kind === turn.kind && + existing.turn.id === turn.id && + existing.runtimeEpoch === runtimeEpoch + ) + return; + if (existing) clearTimeout(existing.handle); + const handle = setTimeout(() => { + const fired = this.activeTurnTimers.get(sessionId); + if ( + fired?.handle !== handle || + !this.isRuntimeEpochCurrent(sessionId, runtimeEpoch) + ) + return; + this.activeTurnTimers.delete(sessionId); + void this.expireActiveTurn(sessionId, turn, runtimeEpoch).catch(() => { + // The timeout is only a request to persist a conservative terminal + // boundary. A storage rejection must retain the exact live owner and + // try that persistence again after another bounded interval; it must + // never release a successor merely because storage was unavailable. + const active = this.activeTurns.get(sessionId); + if ( + this.isRuntimeEpochCurrent(sessionId, runtimeEpoch) && + active?.kind === turn.kind && + active.id === turn.id + ) { + this.armActiveTurnTimer(sessionId, turn, runtimeEpoch); + } + console.error( + "[harness] project bootstrap turn timeout transition failed: persistence_failed", + ); + }); + }, this.deliveryTimeoutMs); + handle.unref?.(); + this.activeTurnTimers.set(sessionId, { + turn: structuredClone(turn), + runtimeEpoch, + handle, + }); + } + + private async commitBootstrapFailureTransition( + state: PersistedProjectBootstrapState, + transition: BootstrapFailureTransitionObligation, + ): Promise { + const sessionId = state.metadata.targetSessionId; + const active = this.activeTurns.get(sessionId); + if ( + active && + (active.kind !== "bootstrap" || active.id !== transition.attemptId) + ) { + return state; + } + const knownAttempt = state.attempts.some( + (attempt) => attempt.attemptId === transition.attemptId, + ); + if (!active && !knownAttempt) { + return state; + } + const releaseCorrelation = (): void => { + if (transition.correlationRelease === "remove") { + this.removeExpectedGreeting(sessionId, transition.attemptId); + } else if (transition.correlationRelease === "tombstone") { + this.tombstoneMissingPromptBarrier(sessionId, { + kind: "bootstrap", + id: transition.attemptId, + }); + } else { + const observed = this.observedAttempts.get(sessionId)?.[0]; + if ( + observed?.kind === "bootstrap" && + observed.id === transition.attemptId + ) { + this.consumeObservedTurn(sessionId, observed); + } else { + this.tombstoneMissingPromptBarrier(sessionId, { + kind: "bootstrap", + id: transition.attemptId, + }); + } + } + }; + if (isTerminal(state.metadata)) { + // persist() may have committed a terminal fallback before throwing. That + // satisfies only the state half of this obligation; exact correlation + // retirement must still land before ownership can be released. + releaseCorrelation(); + this.bootstrapFailureTransitions.delete(sessionId); + this.clearActiveTurn(sessionId, "bootstrap", transition.attemptId); + this.clearTimer(sessionId, transition.attemptId); + if (state.inputs.length > 0) await this.drainWithRecovery(state); + return state; + } + + const next = structuredClone(state); + this.markAttempt(next, transition.attemptId, "retired"); + next.metadata.bootstrap = next.inputs.length + ? { status: "skipped", reason: "user-proceeded" } + : { + status: "failed", + retryable: transition.retryable, + errorCode: transition.errorCode, + }; + await this.persist(sessionId, next); + + releaseCorrelation(); + this.bootstrapFailureTransitions.delete(sessionId); + this.clearActiveTurn(sessionId, "bootstrap", transition.attemptId); + this.clearTimer(sessionId, transition.attemptId); + if (next.metadata.bootstrap.status === "skipped") { + this.emit({ + name: "project_bootstrap.skipped", + projectId: next.metadata.projectId, + sessionId, + attemptId: transition.attemptId, + reason: "user-proceeded", + queueDepth: next.inputs.length, + }); + if (next.inputs.length > 0) await this.drainWithRecovery(next); + } else { + this.emit({ + name: "project_bootstrap.failed", + projectId: next.metadata.projectId, + sessionId, + attemptId: transition.attemptId, + errorCode: transition.errorCode, + retryable: transition.retryable, + queueDepth: 0, + }); + } + return next; + } + + /** + * Persist a pre-attempt lifecycle failure without surrendering the one + * readiness owner first. A failed primary write can project the bounded + * `persistence_failed` fallback, but this obligation retains the original + * public cause/retryability and retries only that state transition; it never + * writes bootstrap prompt bytes. + */ + private async commitPendingBootstrapFailureTransition( + state: PersistedProjectBootstrapState, + transition: PendingBootstrapFailureTransitionObligation, + drainAfterCommit = true, + ): Promise { + const sessionId = state.metadata.targetSessionId; + const bootstrap = state.metadata.bootstrap; + if (bootstrap.status === "delivered" || bootstrap.status === "skipped") { + this.pendingBootstrapFailureTransitions.delete(sessionId); + this.clearTimer(sessionId, "pending"); + if (drainAfterCommit && state.inputs.length > 0) + await this.drainWithRecovery(state); + return state; + } + if (bootstrap.status === "generating") { + // A claim that won after the readiness callback was queued supersedes + // that stale pending transition. Never overwrite the active attempt. + this.pendingBootstrapFailureTransitions.delete(sessionId); + this.clearTimer(sessionId, "pending"); + return state; + } + if ( + bootstrap.status === "failed" && + bootstrap.errorCode !== "persistence_failed" + ) { + // Another exact terminal transition already committed. + this.pendingBootstrapFailureTransitions.delete(sessionId); + this.clearTimer(sessionId, "pending"); + return state; + } + + const next = structuredClone(state); + next.metadata.bootstrap = next.inputs.length + ? { status: "skipped", reason: "user-proceeded" } + : { + status: "failed", + retryable: transition.retryable, + errorCode: transition.errorCode, + }; + try { + await this.persist(sessionId, next); + } catch (error) { + if (!this.closed) this.armTimer(sessionId, "pending"); + throw error; + } + + Object.assign(state, structuredClone(next)); + this.pendingBootstrapFailureTransitions.delete(sessionId); + this.clearTimer(sessionId, "pending"); + if (next.metadata.bootstrap.status === "skipped") { + this.emit({ + name: "project_bootstrap.skipped", + projectId: next.metadata.projectId, + sessionId, + reason: "user-proceeded", + queueDepth: next.inputs.length, + }); + if (drainAfterCommit && next.inputs.length > 0) + await this.drainWithRecovery(next); + } else { + this.emit({ + name: "project_bootstrap.failed", + projectId: next.metadata.projectId, + sessionId, + errorCode: transition.errorCode, + retryable: transition.retryable, + queueDepth: 0, + }); + } + return next; + } + + private async commitUserNotSubmittedTransition( + state: PersistedProjectBootstrapState, + inputId: string, + ): Promise { + const sessionId = state.metadata.targetSessionId; + const active = this.activeTurns.get(sessionId); + if (active?.kind !== "user" || active.id !== inputId) return state; + const next = structuredClone(state); + if (next.dispatchingInputId === inputId) next.dispatchingInputId = null; + await this.persist(sessionId, next); + this.userNotSubmittedTransitions.delete(sessionId); + this.removeExpectedPrompt(sessionId, "user", inputId); + this.clearActiveTurn(sessionId, "user", inputId); + return next; + } + + private async expireActiveTurn( + sessionId: string, + turn: ActiveCoordinatorTurn, + runtimeEpoch: string, + ): Promise { + await this.serialize(sessionId, async () => { + if (!this.isRuntimeEpochCurrent(sessionId, runtimeEpoch)) return; + const current = this.activeTurns.get(sessionId); + if (current?.kind !== turn.kind || current.id !== turn.id) return; + const session = this.options.sessionManager.get(sessionId); + if (!session?.projectBootstrap) { + throw new Error("project bootstrap session unavailable"); + } + let state = await this.load(session); + if (turn.kind === "bootstrap") { + if (this.terminalPreemptionObligations.has(sessionId)) { + state = await this.commitTerminalPreemption(state); + } else if ( + this.bootstrapFailureTransitions.get(sessionId)?.attemptId === turn.id + ) { + state = await this.commitBootstrapFailureTransition( + state, + this.bootstrapFailureTransitions.get(sessionId)!, + ); + } else if ( + state.metadata.bootstrap.status === "generating" && + state.metadata.bootstrap.attemptId === turn.id + ) { + const attempt = state.attempts.find( + (candidate) => candidate.attemptId === turn.id, + ); + const positivelyNotSubmitted = + attempt?.phase === "claimed" || attempt?.phase === "not-submitted"; + const transition: BootstrapFailureTransitionObligation = { + attemptId: turn.id, + errorCode: positivelyNotSubmitted + ? "injection_failed" + : "delivery_timeout", + retryable: positivelyNotSubmitted, + correlationRelease: positivelyNotSubmitted ? "remove" : "tombstone", + }; + if (positivelyNotSubmitted) + this.removeExpectedGreeting(sessionId, turn.id); + this.bootstrapFailureTransitions.set(sessionId, transition); + state = await this.commitBootstrapFailureTransition( + state, + transition, + ); + } else { + // A post-Enter persistence failure may already have projected a + // bounded `failed` classification through persist()'s fallback + // store. The deadline still owns retirement of that submitted + // attempt. Make the retirement durable even though setFailure() no + // longer matches the now-failed metadata; otherwise retry admission + // must infer safety from a stale active attempt after restart. + const retired = structuredClone(state); + this.markAttempt(retired, turn.id, "retired"); + await this.persist(sessionId, retired); + state = retired; + } + } else { + if (this.userNotSubmittedTransitions.get(sessionId) === turn.id) { + state = await this.commitUserNotSubmittedTransition(state, turn.id); + if (isTerminal(state.metadata) && state.inputs.length > 0) + await this.drainWithRecovery(state); + return; + } + if ( + state.dispatchingInputId === turn.id && + state.inputs[0]?.id === turn.id + ) { + // Timeout is the bounded uncertainty boundary. Move the exact head + // out of the replayable FIFO and into its tombstone in one durable + // write; an intermediate `uncertain` receipt beside a dispatchable + // row would be both crash-unsafe and rejected by the schema parser. + state = await this.resolveUncertainDispatch(state); + } else { + this.updateReceiptStatus(state, turn.id, "uncertain"); + await this.persist(sessionId, state); + this.emit({ + name: "project_bootstrap.input_delivery_uncertain", + projectId: state.metadata.projectId, + sessionId, + inputId: turn.id, + errorCode: "delivery_uncertain", + queueDepth: state.inputs.length, + }); + } + } + // Release only after the terminal/non-replayable state is durable. The + // fired timer was removed by its callback, so clearActiveTurn cannot + // accidentally erase a newly armed successor deadline. + const remainingActive = this.activeTurns.get(sessionId); + if ( + remainingActive?.kind === turn.kind && + remainingActive.id === turn.id + ) { + this.tombstoneMissingPromptBarrier(sessionId, turn); + this.clearActiveTurn(sessionId, turn.kind, turn.id); + } + if (isTerminal(state.metadata) && state.inputs.length > 0) { + await this.drainWithRecovery(state); + } + }); + } + + private hasRunnableQueuedInput( + sessionId: string, + runtimeEpoch: string, + ): boolean { + if ( + !this.isRuntimeEpochCurrent(sessionId, runtimeEpoch) || + this.hasInputHold(sessionId) || + this.hasPendingApiInput(sessionId, runtimeEpoch) + ) + return false; + const session = this.options.sessionManager.get(sessionId); + if ( + !session?.projectBootstrap || + !session.ready || + session.status !== "running" + ) { + return false; + } + const state = this.states.get(sessionId); + const metadata = state?.metadata ?? session.projectBootstrap; + const queueDepth = + state?.inputs.length ?? session.projectBootstrap.queuedInputIds.length; + return isTerminal(metadata) && queueDepth > 0; + } + + private requestInputRedrain( + sessionId: string, + runtimeEpoch = this.currentRuntimeEpoch(sessionId), + ): void { + if ( + runtimeEpoch === null || + !this.isRuntimeEpochCurrent(sessionId, runtimeEpoch) + ) + return; + this.inputRedrainNeeded.add(sessionId); + this.scheduleInputRedrain(sessionId, runtimeEpoch); + } + + /** + * One bounded, non-overlapping wakeup for durable FIFO work. Both event and + * status paths may consume their immediate wakeup before a transient store + * failure is observable; the durable queue remains the predicate that keeps + * this retry alive. Active/raw owners, exit, close, and authorization are + * rechecked by the serialized drain before any PTY byte. + */ + private scheduleInputRedrain( + sessionId: string, + runtimeEpoch = this.currentRuntimeEpoch(sessionId), + ): void { + if ( + runtimeEpoch === null || + !this.isRuntimeEpochCurrent(sessionId, runtimeEpoch) || + !this.inputRedrainNeeded.has(sessionId) || + !this.hasRunnableQueuedInput(sessionId, runtimeEpoch) || + this.blockingInputRedrainTimers.has(sessionId) + ) { + return; + } + const handle = setTimeout(() => { + if (this.blockingInputRedrainTimers.get(sessionId) !== handle) return; + this.blockingInputRedrainTimers.delete(sessionId); + if (!this.isRuntimeEpochCurrent(sessionId, runtimeEpoch)) return; + void this.drainSessionWithRecovery(sessionId, runtimeEpoch); + }, this.readinessTimeoutMs); + handle.unref?.(); + this.blockingInputRedrainTimers.set(sessionId, handle); + } + + private async drainWithRecovery( + state: PersistedProjectBootstrapState, + runtimeEpoch = this.runtimeEpochs.get(state.metadata.targetSessionId), + ): Promise { + const sessionId = state.metadata.targetSessionId; + if ( + runtimeEpoch === undefined || + !this.isRuntimeEpochCurrent(sessionId, runtimeEpoch) + ) + return "not-runnable"; + let outcome: ProjectBootstrapDrainOutcome; + try { + outcome = await this.drain(state, runtimeEpoch); + } catch { + outcome = "transient-failure"; + } + if (outcome === "transient-failure") { + this.inputRedrainNeeded.add(sessionId); + } else { + this.inputRedrainNeeded.delete(sessionId); + } + this.scheduleInputRedrain(sessionId, runtimeEpoch); + return outcome; + } + + private async drainSessionWithRecovery( + sessionId: string, + runtimeEpoch: string, + ): Promise { + if (!this.isRuntimeEpochCurrent(sessionId, runtimeEpoch)) + return "not-runnable"; + let outcome: ProjectBootstrapDrainOutcome; + try { + outcome = await this.drainSession(sessionId, runtimeEpoch); + } catch { + outcome = "transient-failure"; + console.error( + "[harness] project bootstrap input redrain failed: persistence_failed", + ); + } + if (outcome === "transient-failure") { + this.inputRedrainNeeded.add(sessionId); + } else { + this.inputRedrainNeeded.delete(sessionId); + } + this.scheduleInputRedrain(sessionId, runtimeEpoch); + return outcome; + } + + private retireAttemptCorrelation(sessionId: string, attemptId: string): void { + const expected = this.expected.get(sessionId); + if (expected) { + this.expected.set( + sessionId, + expected.map((entry) => + entry.kind === "bootstrap" && entry.id === attemptId + ? { ...entry, retired: true } + : entry, + ), + ); + } + const observed = this.observedAttempts.get(sessionId); + if (observed) { + this.observedAttempts.set( + sessionId, + observed.map((entry) => + entry.kind === "bootstrap" && entry.id === attemptId + ? { ...entry, retired: true } + : entry, + ), + ); + } + } + + /** + * A provider prompt hook can be lost even though Enter crossed. Before a + * timed-out turn releases its successor, reserve its completion slot ahead + * of later observed prompts. A late completion then consumes only this + * terminal turn; if it never arrives, the successor reaches its own timeout + * conservatively instead of being falsely marked complete. + */ + private tombstoneMissingPromptBarrier( + sessionId: string, + turn: ActiveCoordinatorTurn, + ): void { + const expected = this.expected.get(sessionId); + const index = expected?.findIndex( + (entry) => entry.kind === turn.kind && entry.id === turn.id, + ); + if (expected && index !== undefined && index >= 0) { + expected.splice(index, 1); + if (expected.length === 0) this.expected.delete(sessionId); + const observed = this.observedAttempts.get(sessionId) ?? []; + if (observed.length >= MAX_CORRELATION_BARRIERS) { + this.clearCorrelation(sessionId); + this.correlationOverflow.add(sessionId); + return; + } + observed.unshift( + turn.kind === "bootstrap" + ? { kind: "bootstrap", id: turn.id, retired: true } + : { kind: "user", id: turn.id }, + ); + this.observedAttempts.set(sessionId, observed); + } else if (turn.kind === "bootstrap") { + this.retireAttemptCorrelation(sessionId, turn.id); + } + } + + private markAttempt( + state: PersistedProjectBootstrapState, + attemptId: string, + status: "active" | "retired" | "completed", + ): void { + const attempt = state.attempts.find( + (candidate) => candidate.attemptId === attemptId, + ); + if (attempt) attempt.status = status; + } + + private markAttemptPhase( + state: PersistedProjectBootstrapState, + attemptId: string, + phase: ProjectBootstrapAttemptPhase, + ): void { + const attempt = state.attempts.find( + (candidate) => candidate.attemptId === attemptId, + ); + if (attempt) attempt.phase = phase; + } + + /** + * An API request reached the server before the background Enter crossed the + * PTY. Return the lifecycle to pending until enqueue() durably commits that + * user input; if enqueue fails, a later registration can safely retry because + * this attempt is proven not to have been submitted. + */ + private async yieldStagedBootstrapToApiInput( + state: PersistedProjectBootstrapState, + attemptId: string, + ): Promise { + const sessionId = state.metadata.targetSessionId; + // The prompt is positively known not to have crossed Enter. Removing only + // this exact expectation immediately prevents an identical newer raw + // prompt from being mislabeled while the lifecycle commit is pending. + this.removeExpectedGreeting(sessionId, attemptId); + const pending = structuredClone(state); + this.markAttempt(pending, attemptId, "retired"); + pending.metadata.bootstrap = { status: "pending" }; + await this.persist(sessionId, pending); + // Positive no-Enter evidence makes retry safe, but owner/correlation + // release still follows the durable lifecycle commit. + this.clearActiveTurn(sessionId, "bootstrap", attemptId); + this.clearTimer(sessionId, attemptId); + this.retireAttemptCorrelation(sessionId, attemptId); + } + + private removeExpectedGreeting(sessionId: string, attemptId: string): void { + this.removeExpectedPrompt(sessionId, "bootstrap", attemptId); + } + + private removeExpectedPrompt( + sessionId: string, + kind: ExpectedPrompt["kind"], + id: string, + ): void { + const expected = this.expected.get(sessionId); + if (!expected) return; + const remaining = expected.filter( + (entry) => !(entry.kind === kind && entry.id === id), + ); + if (remaining.length === 0) this.expected.delete(sessionId); + else this.expected.set(sessionId, remaining); + } + + private clearCorrelation(sessionId: string): void { + this.expected.delete(sessionId); + this.observedAttempts.delete(sessionId); + this.correlationOverflow.delete(sessionId); + } + + private consumeObservedTurn( + sessionId: string, + turn: ObservedProjectTurn | undefined, + ): void { + if (!turn) return; + const observed = this.observedAttempts.get(sessionId); + const head = observed?.[0]; + if ( + !head || + head.kind !== turn.kind || + (head.kind !== "external" && + turn.kind !== "external" && + head.id !== turn.id) + ) { + return; + } + observed.shift(); + if (observed.length === 0) this.observedAttempts.delete(sessionId); + } + + private claimCompletionEvent(sessionId: string, eventId: string): boolean { + if (this.completionDedupeOverflow.has(sessionId)) return false; + const processed = this.processedCompletionEvents.get(sessionId) ?? []; + if (processed.includes(eventId)) return false; + if (processed.length >= MAX_COMPLETION_EVENT_RECEIPTS) { + this.completionDedupeOverflow.add(sessionId); + return false; + } + processed.push(eventId); + this.processedCompletionEvents.set(sessionId, processed); + return true; + } + + private releaseCompletionEvent(sessionId: string, eventId: string): void { + const processed = this.processedCompletionEvents.get(sessionId); + if (!processed) return; + const remaining = processed.filter((candidate) => candidate !== eventId); + if (remaining.length === 0) + this.processedCompletionEvents.delete(sessionId); + else this.processedCompletionEvents.set(sessionId, remaining); + } + + async register( + session: HarnessSession, + context: ProjectBootstrapRegistrationContext, + runtimeEpoch: string | null, + ): Promise { + if (this.closed || !session.projectBootstrap) return; + if ( + context.mode !== "boot" && + (runtimeEpoch === null || + !this.isRuntimeEpochCurrent(session.id, runtimeEpoch)) + ) { + throw new ProjectBootstrapDispatchForbiddenError(); + } + const claimedProjectId = this.provisionalSessionClaims.get(session.id); + if ( + claimedProjectId && + this.provisionalProjectClaims.get(claimedProjectId) === session.id + ) { + this.provisionalProjectClaims.delete(claimedProjectId); + this.provisionalSessionClaims.delete(session.id); + } + let shouldStart = false; + let shouldRetry = false; + let shouldDrain = false; + await this.serialize(session.id, async () => { + const firstRegistration = !this.registeredSessions.has(session.id); + let terminalTransitionEmitted = false; + let recoveredAttemptId: string | undefined; + let state = await this.load(session, context.emptyProject); + this.mergeRegistration(state, session); + + const pendingFailure = this.pendingBootstrapFailureTransitions.get( + session.id, + ); + if (pendingFailure) { + state = await this.commitPendingBootstrapFailureTransition( + state, + pendingFailure, + ); + terminalTransitionEmitted = true; + } + const attemptFailure = this.bootstrapFailureTransitions.get(session.id); + if (attemptFailure) { + state = await this.commitBootstrapFailureTransition( + state, + attemptFailure, + ); + terminalTransitionEmitted = true; + } + + if (context.mode === "boot" && firstRegistration) { + // A fresh process has no live completion barrier. Normalize every + // accepted/submitted/dispatching FIFO prefix atomically before any + // drain can write PTY bytes, preserving arrival order and tombstones. + state = await this.normalizeBootInputState(state); + } + + if (this.terminalPreemptionObligations.has(session.id)) { + state = await this.commitTerminalPreemption(state); + terminalTransitionEmitted = true; + } + + // Only a process-boot load proves an in-flight dispatch was abandoned. + // Live re-registration is idempotent and must not fail its active turn. + if ( + context.mode === "boot" && + firstRegistration && + state.metadata.bootstrap.status === "generating" + ) { + const attemptId = state.metadata.bootstrap.attemptId; + recoveredAttemptId = attemptId; + const attempt = state.attempts.find( + (candidate) => candidate.attemptId === attemptId, + ); + const definitelyUnsubmitted = + attempt?.phase === "claimed" || attempt?.phase === "not-submitted"; + this.markAttempt(state, attemptId, "retired"); + state.metadata.bootstrap = state.inputs.length + ? { status: "skipped", reason: "user-proceeded" } + : definitelyUnsubmitted && state.retryCount < MAX_RETRIES + ? { + status: "failed", + retryable: true, + errorCode: "injection_failed", + } + : { + status: "failed", + retryable: false, + errorCode: "delivery_timeout", + }; + shouldRetry = + definitelyUnsubmitted && + state.inputs.length === 0 && + state.retryCount < MAX_RETRIES && + session.ready && + session.status === "running"; + } + if ( + context.mode === "resumed" && + state.metadata.bootstrap.status === "failed" && + state.metadata.bootstrap.retryable && + state.metadata.bootstrap.errorCode === "injection_failed" && + state.retryCount < MAX_RETRIES && + state.attempts.at(-1)?.status === "retired" && + (state.attempts.at(-1)?.phase === "claimed" || + state.attempts.at(-1)?.phase === "not-submitted") && + session.ready && + session.status === "running" + ) { + shouldRetry = true; + } + await this.persist(session.id, state); + // Recovery is observable only after its classification is durable. A + // failed commit publishes its own persistence failure instead. + if (recoveredAttemptId) { + const bootstrap = state.metadata.bootstrap; + if (bootstrap.status === "skipped") { + terminalTransitionEmitted = true; + this.emit({ + name: "project_bootstrap.skipped", + projectId: state.metadata.projectId, + sessionId: session.id, + attemptId: recoveredAttemptId, + reason: bootstrap.reason, + queueDepth: state.inputs.length, + }); + } else if (bootstrap.status === "failed" && !shouldRetry) { + terminalTransitionEmitted = true; + this.emit({ + name: "project_bootstrap.failed", + projectId: state.metadata.projectId, + sessionId: session.id, + attemptId: recoveredAttemptId, + errorCode: bootstrap.errorCode, + retryable: bootstrap.retryable, + queueDepth: state.inputs.length, + }); + } + } + if (firstRegistration && context.mode === "created") { + this.emit({ + name: "project_bootstrap.scheduled", + projectId: state.metadata.projectId, + sessionId: session.id, + }); + } else if (firstRegistration && context.mode === "boot") { + this.emit({ + name: "project_bootstrap.recovered", + projectId: state.metadata.projectId, + sessionId: session.id, + }); + } + this.registeredSessions.add(session.id); + if ( + firstRegistration && + !terminalTransitionEmitted && + state.metadata.bootstrap.status === "skipped" + ) { + this.emit({ + name: "project_bootstrap.skipped", + projectId: state.metadata.projectId, + sessionId: session.id, + reason: state.metadata.bootstrap.reason, + queueDepth: state.inputs.length, + }); + } + if (state.metadata.bootstrap.status === "pending") { + if (session.status === "exited") { + await this.setFailure(state, "pending", "session_exited", false); + } else { + const allowed = await this.canDispatch(session); + if (session.ready && session.status === "running" && allowed) + shouldStart = true; + else if (allowed && runtimeEpoch !== null) + this.armTimer(session.id, "pending", runtimeEpoch); + else + await this.setFailure(state, "pending", "scope_unavailable", false); + } + } else if (isTerminal(state.metadata)) { + shouldDrain = + session.ready && + session.status === "running" && + (await this.canDispatch(session)); + } + }); + if (runtimeEpoch === null) return; + if (shouldRetry) await this.startGreeting(session.id, true, runtimeEpoch); + else if (shouldStart) + await this.startGreeting(session.id, false, runtimeEpoch); + else if (shouldDrain) + await this.drainSessionWithRecovery(session.id, runtimeEpoch); + } + + private async terminalizeRuntimeOwnership( + session: HarnessSession, + ): Promise { + let state = await this.load(session); + const pendingFailure = this.pendingBootstrapFailureTransitions.get( + session.id, + ); + if (pendingFailure) { + state = await this.commitPendingBootstrapFailureTransition( + state, + pendingFailure, + false, + ); + } + const attemptFailure = this.bootstrapFailureTransitions.get(session.id); + if (attemptFailure) { + state = await this.commitBootstrapFailureTransition( + state, + attemptFailure, + ); + } + if (this.terminalPreemptionObligations.has(session.id)) { + state = await this.commitTerminalPreemption(state); + } + + const timer = this.timers.get(session.id); + const activeTurn = this.activeTurns.get(session.id); + if (activeTurn?.kind === "user") { + if (this.userNotSubmittedTransitions.get(session.id) === activeTurn.id) { + state = await this.commitUserNotSubmittedTransition( + state, + activeTurn.id, + ); + } else if ( + state.dispatchingInputId === activeTurn.id && + state.inputs[0]?.id === activeTurn.id + ) { + // An ambiguous Enter may still own the replayable FIFO head. Remove + // that exact row and persist its tombstone before admitting a new PTY. + state = await this.resolveUncertainDispatch(state); + } else { + const receipt = this.receiptForInput(state, activeTurn.id); + if (receipt && receipt.status !== "completed") { + const exitedState = { + ...structuredClone(state), + receipts: structuredClone(state.receipts), + }; + this.updateReceiptStatus(exitedState, activeTurn.id, "uncertain"); + await this.persist(session.id, exitedState); + state = exitedState; + } + } + } + const expectedKey = + activeTurn?.kind === "bootstrap" + ? activeTurn.id + : (timer?.key ?? + (state.metadata.bootstrap.status === "generating" + ? state.metadata.bootstrap.attemptId + : "pending")); + if (activeTurn?.kind === "bootstrap") { + if ( + state.metadata.bootstrap.status === "generating" && + state.metadata.bootstrap.attemptId === activeTurn.id + ) { + const transition: BootstrapFailureTransitionObligation = { + attemptId: activeTurn.id, + errorCode: "session_exited", + retryable: false, + correlationRelease: "tombstone", + }; + this.bootstrapFailureTransitions.set(session.id, transition); + state = await this.commitBootstrapFailureTransition(state, transition); + } else { + this.markAttempt(state, activeTurn.id, "retired"); + await this.persist(session.id, state); + } + } else { + await this.setFailure(state, expectedKey, "session_exited", false, false); + } + // All fallible state transitions have committed. Volatile correlation is + // cleared by the caller only after this returns successfully. + const remainingActive = this.activeTurns.get(session.id); + if ( + activeTurn && + remainingActive?.kind === activeTurn.kind && + remainingActive.id === activeTurn.id + ) { + this.tombstoneMissingPromptBarrier(session.id, activeTurn); + this.clearActiveTurn(session.id, activeTurn.kind, activeTurn.id); + } + } + + /** + * Move coordinator ownership between server-issued PTY generations. This is + * awaited by SessionManager before publishing a replacement handle. + */ + transitionRuntimeEpoch( + session: HarnessSession, + nextRuntimeEpoch: string | null, + ): Promise { + return this.serialize(session.id, async () => { + this.assertOpen(); + const currentRuntimeEpoch = this.runtimeEpochs.get(session.id); + if (currentRuntimeEpoch === nextRuntimeEpoch) return; + if (currentRuntimeEpoch !== undefined) { + if (session.projectBootstrap) { + await this.terminalizeRuntimeOwnership({ + ...session, + status: "exited", + ready: false, + }); + } + this.clearRuntimeEpochState(session.id, currentRuntimeEpoch); + } + if (nextRuntimeEpoch !== null) { + if (!nextRuntimeEpoch || nextRuntimeEpoch.length > 128) { + throw new ProjectBootstrapDispatchForbiddenError(); + } + this.runtimeEpochs.set(session.id, nextRuntimeEpoch); + } + }); + } + + async onSessionStatus( + session: HarnessSession, + runtimeEpoch: string | null, + ): Promise { + if ( + this.closed || + !session.projectBootstrap || + runtimeEpoch === null || + !this.isRuntimeEpochCurrent(session.id, runtimeEpoch) + ) + return; + let action: "start" | "retry" | "drain" | null = null; + await this.serialize(session.id, async () => { + if ( + !this.isRuntimeEpochCurrent(session.id, runtimeEpoch) || + !this.registeredSessions.has(session.id) + ) { + return; + } + if (session.status === "exited") { + await this.terminalizeRuntimeOwnership(session); + this.clearRuntimeEpochState(session.id, runtimeEpoch); + return; + } + let state = await this.load(session); + const pendingFailure = this.pendingBootstrapFailureTransitions.get( + session.id, + ); + if (pendingFailure) { + state = await this.commitPendingBootstrapFailureTransition( + state, + pendingFailure, + ); + } + const attemptFailure = this.bootstrapFailureTransitions.get(session.id); + if (attemptFailure) { + state = await this.commitBootstrapFailureTransition( + state, + attemptFailure, + ); + } + if (this.terminalPreemptionObligations.has(session.id)) { + state = await this.commitTerminalPreemption(state); + } + if ( + session.ready && + session.status === "running" && + (await this.canDispatch(session)) + ) { + if (!this.isRuntimeEpochCurrent(session.id, runtimeEpoch)) return; + const blockingRedrain = this.blockingInputRedrainTimers.get(session.id); + if (blockingRedrain) { + clearTimeout(blockingRedrain); + this.blockingInputRedrainTimers.delete(session.id); + } + const recoverableClaim = + state.metadata.bootstrap.status === "failed" && + state.metadata.bootstrap.retryable && + (state.metadata.bootstrap.errorCode === "session_not_ready" || + state.metadata.bootstrap.errorCode === "injection_failed") && + hasSafeBootstrapRetryEvidence(state) && + state.retryCount < MAX_RETRIES && + state.inputs.length === 0; + action = recoverableClaim + ? "retry" + : isTerminal(state.metadata) + ? "drain" + : "start"; + } + }); + if (!this.isRuntimeEpochCurrent(session.id, runtimeEpoch)) return; + if (action === "drain") + await this.drainSessionWithRecovery(session.id, runtimeEpoch); + else if (action === "retry") + await this.startGreeting(session.id, true, runtimeEpoch); + else if (action === "start") + await this.startGreeting(session.id, false, runtimeEpoch); + } + + private async startGreeting( + sessionId: string, + retry: boolean, + runtimeEpoch: string, + ): Promise { + await this.serialize(sessionId, async () => { + if (!this.isRuntimeEpochCurrent(sessionId, runtimeEpoch)) return; + const session = this.options.sessionManager.get(sessionId); + if (!session?.projectBootstrap) return; + let state = await this.load(session); + if (!this.isRuntimeEpochCurrent(sessionId, runtimeEpoch)) return; + const pendingFailure = + this.pendingBootstrapFailureTransitions.get(sessionId); + if (pendingFailure) { + await this.commitPendingBootstrapFailureTransition( + state, + pendingFailure, + ); + return; + } + const attemptFailure = this.bootstrapFailureTransitions.get(sessionId); + if (attemptFailure) { + await this.commitBootstrapFailureTransition(state, attemptFailure); + return; + } + if (this.activeTurns.has(sessionId)) { + if (retry) throw new ProjectBootstrapRetryUnavailableError(); + return; + } + if (this.terminalPreemptions.has(sessionId)) { + this.terminalPreemptionObligations.add(sessionId); + state = await this.commitTerminalPreemption(state); + return; + } + if (!(await this.canDispatch(session))) { + if (retry) throw new ProjectBootstrapDispatchForbiddenError(); + if (state.metadata.bootstrap.status === "pending") { + await this.setFailure( + state, + "pending", + session.status === "exited" + ? "session_exited" + : "scope_unavailable", + false, + ); + } + return; + } + let mapStillEmpty: boolean; + try { + mapStillEmpty = + (await this.options.isMeaningfullyEmpty?.( + state.metadata.projectId, + )) ?? state.emptyProject; + } catch { + const expected = + state.metadata.bootstrap.status === "generating" + ? state.metadata.bootstrap.attemptId + : "pending"; + await this.setFailure(state, expected, "persistence_failed", true); + return; + } + if (!mapStillEmpty) { + state.emptyProject = false; + state.metadata.bootstrap = { + status: "skipped", + reason: "map-not-empty", + }; + await this.persist(sessionId, state); + this.emit({ + name: "project_bootstrap.skipped", + projectId: state.metadata.projectId, + sessionId, + reason: "map-not-empty", + queueDepth: state.inputs.length, + }); + return; + } + if (retry) { + if ( + state.metadata.bootstrap.status !== "failed" || + !state.metadata.bootstrap.retryable || + !hasSafeBootstrapRetryEvidence(state) || + state.inputs.length > 0 || + this.activeTurns.has(sessionId) || + state.retryCount >= MAX_RETRIES + ) { + throw new ProjectBootstrapRetryUnavailableError(); + } + state.retryCount += 1; + } else if (state.metadata.bootstrap.status !== "pending") { + return; + } + const attemptId = this.generateId(); + const claimed = structuredClone(state); + claimed.metadata.bootstrap = { status: "generating", attemptId }; + claimed.attempts.push({ + attemptId, + retryOrdinal: claimed.retryCount, + status: "active", + phase: "claimed", + }); + claimed.attempts = claimed.attempts.slice(-8); + try { + await this.persist(sessionId, claimed); + } catch { + // A ready session can enter this path without a readiness timer. Keep + // one persistence-only owner after a total primary+fallback failure so + // the last durable `pending` state cannot become ownerless. This owner + // never retries the PTY claim or allocates another attempt; it commits + // the bounded failure classification under the original pending CAS. + const transition: PendingBootstrapFailureTransitionObligation = { + errorCode: "persistence_failed", + retryable: true, + }; + this.pendingBootstrapFailureTransitions.set(sessionId, transition); + this.armTimer(sessionId, "pending", runtimeEpoch); + return; + } + state = claimed; + this.clearTimer(sessionId, "pending"); + this.emit({ + name: retry + ? "project_bootstrap.retried" + : "project_bootstrap.attempted", + projectId: state.metadata.projectId, + sessionId, + attemptId, + retryOrdinal: state.retryCount, + queueDepth: state.inputs.length, + }); + const prompt = projectBootstrapPrompt(state.retryCount, attemptId); + if (!this.isRuntimeEpochCurrent(sessionId, runtimeEpoch)) return; + if (!(await this.canDispatch(session))) { + await this.setFailure( + state, + attemptId, + session.status === "exited" ? "session_exited" : "scope_unavailable", + false, + ); + if (retry) throw new ProjectBootstrapDispatchForbiddenError(); + return; + } + if (this.terminalPreemptions.has(sessionId)) { + this.markAttempt(state, attemptId, "retired"); + state.metadata.bootstrap = { + status: "skipped", + reason: "user-proceeded", + }; + await this.persist(sessionId, state); + if (!this.reportedTerminalPreemptions.has(sessionId)) { + this.reportedTerminalPreemptions.add(sessionId); + this.emit({ + name: "project_bootstrap.preempted", + projectId: state.metadata.projectId, + sessionId, + attemptId, + reason: "user-proceeded", + queueDepth: state.inputs.length, + }); + } + return; + } + if (this.hasPendingApiInput(sessionId, runtimeEpoch)) { + await this.yieldStagedBootstrapToApiInput(state, attemptId); + return; + } + // Register before crossing the PTY boundary. A prompt hook may arrive + // immediately after the write, before submitInput's delayed Enter has + // resolved; registering afterward loses the only safe correlation. + const queue = this.expected.get(sessionId) ?? []; + queue.push({ + kind: "bootstrap", + id: attemptId, + text: prompt, + retired: false, + }); + this.expected.set(sessionId, queue); + this.activeTurns.set(sessionId, { + kind: "bootstrap", + id: attemptId, + }); + const admissionGeneration = this.admissionGeneration; + let crossedEnter = false; + let durableNotSubmitted = false; + try { + const accepted = await this.options.sessionManager.submitInput( + sessionId, + prompt, + true, + async () => + !this.closed && + this.isRuntimeEpochCurrent(sessionId, runtimeEpoch) && + !this.hasPendingApiInput(sessionId, runtimeEpoch) && + (await this.canDispatch(session)), + true, + { + beforeFirstWrite: async () => { + if ( + !this.isAdmissionCurrent(admissionGeneration) || + !this.isRuntimeEpochCurrent(sessionId, runtimeEpoch) || + this.hasPendingApiInput(sessionId, runtimeEpoch) + ) { + throw new SessionInputGuardRejectedError(false); + } + this.markAttemptPhase(state, attemptId, "dispatching"); + await this.persist(sessionId, state); + }, + canWriteNow: () => + this.isAdmissionCurrent(admissionGeneration) && + this.isRuntimeEpochCurrent(sessionId, runtimeEpoch) && + !this.hasPendingApiInput(sessionId, runtimeEpoch), + onNotSubmitted: async () => { + if (!this.isRuntimeEpochCurrent(sessionId, runtimeEpoch)) return; + this.markAttemptPhase(state, attemptId, "not-submitted"); + this.markAttempt(state, attemptId, "retired"); + if ( + state.metadata.bootstrap.status === "failed" && + state.metadata.bootstrap.errorCode === "persistence_failed" + ) { + state.metadata.bootstrap = { + ...state.metadata.bootstrap, + retryable: true, + }; + } + await this.persist(sessionId, state); + durableNotSubmitted = true; + }, + }, + ); + if (!accepted) { + // A false return proves the prompt did not cross the PTY boundary. + this.markAttemptPhase(state, attemptId, "not-submitted"); + this.removeExpectedGreeting(sessionId, attemptId); + const transition: BootstrapFailureTransitionObligation = { + attemptId, + errorCode: + session.status === "exited" + ? "session_exited" + : "scope_unavailable", + retryable: false, + correlationRelease: "remove", + }; + this.bootstrapFailureTransitions.set(sessionId, transition); + try { + await this.commitBootstrapFailureTransition(state, transition); + } catch { + this.armActiveTurnTimer( + sessionId, + { kind: "bootstrap", id: attemptId }, + runtimeEpoch, + ); + return; + } + return; + } + crossedEnter = true; + this.markAttemptPhase(state, attemptId, "submitted"); + // Enter owns a live model turn now. Arm its bound before any later + // state write so a storage rejection cannot release queued user input + // or make this submitted attempt eligible for blind replay. + if ( + this.isAdmissionCurrent(admissionGeneration) && + this.isRuntimeEpochCurrent(sessionId, runtimeEpoch) + ) { + this.armActiveTurnTimer( + sessionId, + { kind: "bootstrap", id: attemptId }, + runtimeEpoch, + ); + } + if ( + !this.isAdmissionCurrent(admissionGeneration) || + !this.isRuntimeEpochCurrent(sessionId, runtimeEpoch) + ) + return; + await this.persist(sessionId, state); + } catch (error) { + if ( + !this.isAdmissionCurrent(admissionGeneration) || + !this.isRuntimeEpochCurrent(sessionId, runtimeEpoch) + ) { + this.clearActiveTurn(sessionId, "bootstrap", attemptId); + this.removeExpectedGreeting(sessionId, attemptId); + return; + } + if (crossedEnter) { + // persist() already retains the bounded content-free failure wherever + // either durable store remains. The live turn and its timer continue + // to own the CLI until completion or this attempt's own deadline. + return; + } + const positivelyBeforeEnter = + durableNotSubmitted || + error instanceof SessionNotReadyError || + (error instanceof SessionInputGuardRejectedError && !error.staged) || + (error instanceof SessionBackgroundInputPreemptedError && + !error.staged); + if (!positivelyBeforeEnter) { + // A text or Enter write can report failure after accepting bytes, + // and a failed composer cleanup is equally ambiguous. Preserve the + // submitted-turn owner and give it the same bounded terminalization + // path as a successful submit. User input accepted concurrently can + // still preempt metadata, but cannot enter this composer until the + // ambiguity is durably retired. + this.armActiveTurnTimer( + sessionId, + { kind: "bootstrap", id: attemptId }, + runtimeEpoch, + ); + return; + } + if ( + this.hasPendingApiInput(sessionId, runtimeEpoch) && + (error instanceof SessionBackgroundInputPreemptedError || + error instanceof SessionInputGuardRejectedError) + ) { + try { + await this.yieldStagedBootstrapToApiInput(state, attemptId); + } catch { + this.armActiveTurnTimer( + sessionId, + { kind: "bootstrap", id: attemptId }, + runtimeEpoch, + ); + } + return; + } + if (error instanceof SessionBackgroundInputPreemptedError) { + if (durableNotSubmitted || !error.staged) + this.removeExpectedGreeting(sessionId, attemptId); + try { + this.terminalPreemptionObligations.add(sessionId); + state = await this.commitTerminalPreemption(state); + } catch { + this.armActiveTurnTimer( + sessionId, + { kind: "bootstrap", id: attemptId }, + runtimeEpoch, + ); + return; + } + this.clearActiveTurn(sessionId, "bootstrap", attemptId); + this.removeExpectedGreeting(sessionId, attemptId); + this.clearTimer(sessionId, attemptId); + if (state.inputs.length > 0) + await this.drainWithRecovery(state, runtimeEpoch); + return; + } + const removeUnsubmittedBarrier = + error instanceof SessionNotReadyError || + (error instanceof SessionInputGuardRejectedError && !error.staged) || + durableNotSubmitted; + const transition: BootstrapFailureTransitionObligation = { + attemptId, + errorCode: + error instanceof SessionNotReadyError + ? "session_not_ready" + : error instanceof SessionInputGuardRejectedError + ? session.status === "exited" + ? "session_exited" + : "scope_unavailable" + : "injection_failed", + retryable: + error instanceof SessionNotReadyError || + (durableNotSubmitted && + !(error instanceof SessionInputGuardRejectedError)), + correlationRelease: removeUnsubmittedBarrier ? "remove" : "tombstone", + }; + if (removeUnsubmittedBarrier) + this.removeExpectedGreeting(sessionId, attemptId); + this.bootstrapFailureTransitions.set(sessionId, transition); + try { + await this.commitBootstrapFailureTransition(state, transition); + } catch { + this.armActiveTurnTimer( + sessionId, + { kind: "bootstrap", id: attemptId }, + runtimeEpoch, + ); + return; + } + if (retry && error instanceof SessionInputGuardRejectedError) { + throw new ProjectBootstrapDispatchForbiddenError(); + } + } + }); + } + + private async setFailure( + state: PersistedProjectBootstrapState, + expectedKey: "pending" | string, + errorCode: ProjectBootstrapErrorCode, + retryable: boolean, + drainAfterCommit = true, + ): Promise { + if (expectedKey === "pending") { + const existing = this.pendingBootstrapFailureTransitions.get( + state.metadata.targetSessionId, + ); + if (!existing && state.metadata.bootstrap.status !== "pending") return; + const transition = existing ?? { errorCode, retryable }; + this.pendingBootstrapFailureTransitions.set( + state.metadata.targetSessionId, + transition, + ); + try { + await this.commitPendingBootstrapFailureTransition( + state, + transition, + drainAfterCommit, + ); + } catch (error) { + if (!this.closed) + this.armTimer(state.metadata.targetSessionId, "pending"); + throw error; + } + return; + } + const greeting = state.metadata.bootstrap; + const sessionId = state.metadata.targetSessionId; + const existing = this.bootstrapFailureTransitions.get(sessionId); + if ( + greeting.status !== "generating" && + existing?.attemptId !== expectedKey + ) { + return; + } + if ( + greeting.status === "generating" && + greeting.attemptId !== expectedKey + ) { + return; + } + const attempt = state.attempts.find( + (candidate) => candidate.attemptId === expectedKey, + ); + const positivelyNotSubmitted = + attempt?.phase === "claimed" || attempt?.phase === "not-submitted"; + const transition = existing ?? { + attemptId: expectedKey, + errorCode, + retryable, + correlationRelease: positivelyNotSubmitted + ? ("remove" as const) + : ("tombstone" as const), + }; + if (transition.correlationRelease === "remove") + this.removeExpectedGreeting(sessionId, expectedKey); + this.bootstrapFailureTransitions.set(sessionId, transition); + try { + const committed = await this.commitBootstrapFailureTransition( + state, + transition, + ); + Object.assign(state, structuredClone(committed)); + if (!drainAfterCommit) this.inputRedrainNeeded.delete(sessionId); + } catch (error) { + if (!this.closed && !this.activeTurns.has(sessionId)) + this.armTimer(sessionId, expectedKey); + throw error; + } + } + + private async fail( + sessionId: string, + expectedKey: "pending" | string, + errorCode: ProjectBootstrapErrorCode, + retryable: boolean, + runtimeEpoch: string, + ): Promise { + await this.serialize(sessionId, async () => { + if (!this.isRuntimeEpochCurrent(sessionId, runtimeEpoch)) return; + const session = this.options.sessionManager.get(sessionId); + if (!session?.projectBootstrap) return; + const state = await this.load(session); + if (this.terminalPreemptionObligations.has(sessionId)) { + await this.commitTerminalPreemption(state); + } else if (expectedKey === "pending") { + const existing = this.pendingBootstrapFailureTransitions.get(sessionId); + if (!existing && state.metadata.bootstrap.status !== "pending") return; + const transition = existing ?? { errorCode, retryable }; + this.pendingBootstrapFailureTransitions.set(sessionId, transition); + await this.commitPendingBootstrapFailureTransition(state, transition); + } else if ( + this.bootstrapFailureTransitions.get(sessionId)?.attemptId === + expectedKey + ) { + await this.commitBootstrapFailureTransition( + state, + this.bootstrapFailureTransitions.get(sessionId)!, + ); + } else { + await this.setFailure(state, expectedKey, errorCode, retryable); + } + }); + } + + retry(sessionId: string): Promise { + if (this.closed) + return Promise.reject(new ProjectBootstrapCoordinatorClosedError()); + const runtimeEpoch = this.currentRuntimeEpoch(sessionId); + if (runtimeEpoch === null) { + return Promise.reject(new ProjectBootstrapDispatchForbiddenError()); + } + return this.startGreeting(sessionId, true, runtimeEpoch); + } + + enqueue(sessionId: string, text: string): Promise { + return this.enqueueWithReceipt(sessionId, text).then( + (result) => result.metadata, + ); + } + + async enqueueWithReceipt( + sessionId: string, + text: string, + requestId?: string, + ): Promise<{ + metadata: ProjectBootstrapMetadata; + receipt: ProjectBootstrapInputReceipt; + }> { + if (this.closed) throw new ProjectBootstrapCoordinatorClosedError(); + const known = this.options.sessionManager.get(sessionId); + if (!known?.projectBootstrap) { + throw new Error("project bootstrap session not found"); + } + // Scope/principal/CWD ownership is the read authorization boundary too. + // Validate it before looking up a durable request receipt so a foreign or + // rebound caller cannot use idempotency as an existence oracle. + if (!(await this.canDispatch(known))) + throw new ProjectBootstrapDispatchForbiddenError(); + const runtimeEpoch = this.currentRuntimeEpoch(sessionId); + const payloadDigest = this.inputPayloadDigest(text); + const cachedState = this.states.get(sessionId); + const cachedReceipt = requestId + ? cachedState?.receipts.find((receipt) => receipt.requestId === requestId) + : undefined; + let pendingInputInstalled = false; + + // This signal is intentionally installed before waiting for the coordinator + // lock: startGreeting may currently be between its background text write and + // delayed Enter. Cancelling that staging window gives durable user input + // priority without ever splicing the two prompts together. + // A known idempotency receipt is a pure lookup. It must not preempt a live + // composer or install admission state merely because the response is being + // retried. If the state has not been loaded yet, defer this decision to the + // serialized durable lookup below. + if ( + runtimeEpoch !== null && + (!requestId || (cachedState && !cachedReceipt)) + ) { + this.notePendingApiInput(sessionId, runtimeEpoch); + pendingInputInstalled = true; + try { + this.options.sessionManager.preemptBackgroundInput(sessionId); + } catch (error) { + this.clearPendingApiInput(sessionId, runtimeEpoch); + throw error; + } + } + + const operation = this.serialize(sessionId, async () => { + this.assertOpen(); + const session = this.options.sessionManager.get(sessionId); + if (!session?.projectBootstrap) + throw new Error("project bootstrap session not found"); + if (!(await this.canDispatch(session))) + throw new ProjectBootstrapDispatchForbiddenError(); + const state = await this.load(session); + if (!(await this.canDispatch(session))) + throw new ProjectBootstrapDispatchForbiddenError(); + const existingReceipt = requestId + ? state.receipts.find((receipt) => receipt.requestId === requestId) + : undefined; + if (existingReceipt) { + if (existingReceipt.payloadDigest !== payloadDigest) { + throw new ProjectBootstrapRequestIdConflictError(); + } + if ( + existingReceipt.status === "queued" && + runtimeEpoch !== null && + this.isRuntimeEpochCurrent(sessionId, runtimeEpoch) && + isTerminal(state.metadata) && + !this.hasInputHold(sessionId) && + session.ready && + session.status === "running" && + (await this.canDispatch(session)) + ) { + // A prior attempt may have positively proved that Enter was never + // attempted and durably rolled this exact row back to queued. A + // response-loss retry with the same key is the next admission event: + // redrive the existing row through the one FIFO authority, never + // create a replacement receipt. Transient redrive failures leave and + // return the durable queued/submitted classification. + if (pendingInputInstalled) { + this.clearPendingApiInput(sessionId, runtimeEpoch); + pendingInputInstalled = false; + } + await this.drainWithRecovery(state, runtimeEpoch).catch(() => {}); + } + const latest = this.states.get(sessionId) ?? state; + const latestReceipt = this.receiptForInput( + latest, + existingReceipt.inputId, + ); + return { + metadata: structuredClone(latest.metadata), + receipt: this.publicReceipt(latestReceipt ?? existingReceipt), + }; + } + if ( + runtimeEpoch === null || + !this.isRuntimeEpochCurrent(sessionId, runtimeEpoch) || + this.options.sessionManager.getRuntimeEpoch(sessionId) !== runtimeEpoch + ) { + throw new ProjectBootstrapDispatchForbiddenError(); + } + if (!pendingInputInstalled) { + this.notePendingApiInput(sessionId, runtimeEpoch); + pendingInputInstalled = true; + this.options.sessionManager.preemptBackgroundInput(sessionId); + } + if (session.status === "exited" || !(await this.canDispatch(session))) { + throw new ProjectBootstrapDispatchForbiddenError(); + } + state.receipts = compactInputReceipts(state.receipts, 1); + const input: ProjectBootstrapQueuedInput = { + id: this.generateId(), + sessionId, + text, + acceptedAt: this.now(), + }; + state.inputs.push(input); + state.metadata.queuedInputIds.push(input.id); + state.receipts.push({ + requestId: requestId ?? null, + inputId: input.id, + status: "queued", + acceptedAt: input.acceptedAt, + payloadDigest, + }); + const bootstrap = state.metadata.bootstrap; + const preempt = + bootstrap.status === "pending" || + bootstrap.status === "generating" || + bootstrap.status === "failed"; + const attemptId = + bootstrap.status === "generating" ? bootstrap.attemptId : undefined; + if (preempt) { + state.metadata.bootstrap = { + status: "skipped", + reason: "user-proceeded", + }; + if (attemptId) this.markAttempt(state, attemptId, "retired"); + } + await this.persist(sessionId, state); + if (preempt) { + this.clearTimer(sessionId); + if (attemptId) { + this.retireAttemptCorrelation(sessionId, attemptId); + const active = this.activeTurns.get(sessionId); + const attempt = state.attempts.find( + (candidate) => candidate.attemptId === attemptId, + ); + if ( + active?.kind === "bootstrap" && + active.id === attemptId && + (attempt?.phase === "claimed" || attempt?.phase === "not-submitted") + ) { + this.removeExpectedGreeting(sessionId, attemptId); + this.bootstrapFailureTransitions.delete(sessionId); + this.clearActiveTurn(sessionId, "bootstrap", attemptId); + } + } + this.emit({ + name: "project_bootstrap.preempted", + projectId: state.metadata.projectId, + sessionId, + ...(attemptId ? { attemptId } : {}), + reason: "user-proceeded", + queueDepth: state.inputs.length, + }); + } + // Durable acceptance is the API acknowledgement boundary. A project or + // session rebind after this commit may pause dispatch, but must not turn + // the accepted request into a client-visible failure that invites a + // duplicate retry. + if (isTerminal(state.metadata)) { + if (pendingInputInstalled) { + this.clearPendingApiInput(sessionId, runtimeEpoch); + pendingInputInstalled = false; + } + await this.drainWithRecovery(state, runtimeEpoch); + } + // drain() advances through immutable queue-state clones. Return the + // latest authoritative projection rather than the pre-drain object so a + // 202 response never reports IDs that were already durably dequeued. + const latest = this.states.get(sessionId) ?? state; + const receipt = this.receiptForInput(latest, input.id); + if (!receipt) { + throw new Error("project bootstrap input receipt unavailable"); + } + return { + metadata: structuredClone(latest.metadata), + receipt: this.publicReceipt(receipt), + }; + }); + return operation.finally(() => { + if (pendingInputInstalled && runtimeEpoch !== null) + this.clearPendingApiInput(sessionId, runtimeEpoch); + }); + } + + private scheduleTerminalPreemptionRetry( + sessionId: string, + runtimeEpoch = this.currentRuntimeEpoch(sessionId), + ): void { + if ( + runtimeEpoch === null || + !this.isRuntimeEpochCurrent(sessionId, runtimeEpoch) || + !this.terminalPreemptionObligations.has(sessionId) || + this.terminalPreemptionRetryTimers.has(sessionId) + ) { + return; + } + const handle = setTimeout(() => { + if (this.terminalPreemptionRetryTimers.get(sessionId) !== handle) return; + this.terminalPreemptionRetryTimers.delete(sessionId); + void this.serialize(sessionId, async () => { + if ( + !this.isRuntimeEpochCurrent(sessionId, runtimeEpoch) || + !this.terminalPreemptionObligations.has(sessionId) + ) + return; + const session = this.options.sessionManager.get(sessionId); + // Registration is the authoritative publication wakeup for a session + // that does not yet expose bootstrap metadata. Keep the obligation, + // but avoid polling an unpublished or removed session. + if (!session?.projectBootstrap) return; + const state = await this.load(session); + await this.commitTerminalPreemption(state); + }).catch(() => + this.scheduleTerminalPreemptionRetry(sessionId, runtimeEpoch), + ); + }, this.readinessTimeoutMs); + handle.unref?.(); + this.terminalPreemptionRetryTimers.set(sessionId, handle); + } + + private async commitTerminalPreemption( + current: PersistedProjectBootstrapState, + ): Promise { + const sessionId = current.metadata.targetSessionId; + const prior = current.metadata.bootstrap; + let state = current; + let attemptId: string | undefined; + if (!isTerminal(current.metadata)) { + state = structuredClone(current); + attemptId = prior.status === "generating" ? prior.attemptId : undefined; + if (attemptId) this.markAttempt(state, attemptId, "retired"); + state.metadata.bootstrap = { + status: "skipped", + reason: "user-proceeded", + }; + // Copy-on-write is intentional: none of the live owner/timer/correlation + // state moves until this terminal transition is durable. + try { + await this.persist(sessionId, state); + } catch (error) { + this.scheduleTerminalPreemptionRetry(sessionId); + throw error; + } + this.clearTimer(sessionId); + if (attemptId) this.retireAttemptCorrelation(sessionId, attemptId); + if (!this.reportedTerminalPreemptions.has(sessionId)) { + this.reportedTerminalPreemptions.add(sessionId); + this.emit({ + name: "project_bootstrap.preempted", + projectId: state.metadata.projectId, + sessionId, + ...(attemptId ? { attemptId } : {}), + reason: "user-proceeded", + queueDepth: state.inputs.length, + }); + } + } + + const active = this.activeTurns.get(sessionId); + if (active?.kind === "bootstrap") { + const attempt = state.attempts.find( + (candidate) => candidate.attemptId === active.id, + ); + if (attempt?.phase === "claimed" || attempt?.phase === "not-submitted") { + this.removeExpectedGreeting(sessionId, active.id); + this.clearActiveTurn(sessionId, "bootstrap", active.id); + } + } + + const retryTimer = this.terminalPreemptionRetryTimers.get(sessionId); + if (retryTimer) { + clearTimeout(retryTimer); + this.terminalPreemptionRetryTimers.delete(sessionId); + } + this.terminalPreemptionObligations.delete(sessionId); + if (this.blockingTerminalPreemptions.delete(sessionId)) { + // Setup/trust bytes may not emit a model completion. Their raw hold is + // released only after user-proceeded is durable, then one bounded FIFO + // wakeup is requested. + this.terminalPreemptions.delete(sessionId); + this.requestInputRedrain(sessionId); + } else if (!this.terminalPreemptions.has(sessionId)) { + // An ordinary raw model turn may have completed while skip persistence + // was retrying. Its ownership is gone, but its durable obligation still + // had to commit before queued API work could progress. + this.requestInputRedrain(sessionId); + } + return state; + } + + /** + * Raw PTY input is already owned by the user and is never copied into this + * store. Install its synchronous hold first, then commit the durable + * user-proceeded transition before releasing any lifecycle owner. + */ + onTerminalInput( + sessionId: string, + context: { runtimeEpoch: string; blockingPrompt: boolean }, + ): void { + const { runtimeEpoch } = context; + if (!this.isRuntimeEpochCurrent(sessionId, runtimeEpoch)) return; + this.terminalPreemptionObligations.add(sessionId); + this.terminalPreemptions.add(sessionId); + if (context.blockingPrompt) this.blockingTerminalPreemptions.add(sessionId); + else this.blockingTerminalPreemptions.delete(sessionId); + void this.serialize(sessionId, async () => { + if ( + !this.isRuntimeEpochCurrent(sessionId, runtimeEpoch) || + !this.terminalPreemptionObligations.has(sessionId) + ) + return; + const session = this.options.sessionManager.get(sessionId); + if (!session?.projectBootstrap) return; + const state = await this.load(session); + await this.commitTerminalPreemption(state); + }).catch(() => + this.scheduleTerminalPreemptionRetry(sessionId, runtimeEpoch), + ); + } + + private async drainSession( + sessionId: string, + runtimeEpoch: string, + ): Promise { + return this.serialize(sessionId, async () => { + if (!this.isRuntimeEpochCurrent(sessionId, runtimeEpoch)) + return "not-runnable"; + const session = this.options.sessionManager.get(sessionId); + if (!session?.projectBootstrap) return "not-runnable"; + const state = await this.load(session); + if (!this.isRuntimeEpochCurrent(sessionId, runtimeEpoch)) + return "not-runnable"; + if (!isTerminal(state.metadata)) return "owned"; + return this.drain(state, runtimeEpoch); + }); + } + + private async drain( + initialState: PersistedProjectBootstrapState, + runtimeEpoch: string, + ): Promise { + const sessionId = initialState.metadata.targetSessionId; + if (!this.isRuntimeEpochCurrent(sessionId, runtimeEpoch)) + return "not-runnable"; + if (this.hasInputHold(sessionId)) return "owned"; + let state: PersistedProjectBootstrapState; + try { + // An accepted-input ledger is the commit/ack boundary. If the prior + // process reached the PTY but failed to rewrite the FIFO, finish that + // dequeue without submitting the prompt again. + state = await this.reconcileAcceptedInputs(initialState); + } catch { + return "transient-failure"; + } + const input = state.inputs[0]; + if (!input) return "empty"; + if (this.hasInputHold(sessionId)) return "owned"; + const session = this.options.sessionManager.get(input.sessionId); + if (!session?.ready || session.status !== "running") return "not-runnable"; + const admissionGeneration = this.admissionGeneration; + if (!this.isRuntimeEpochCurrent(sessionId, runtimeEpoch)) + return "not-runnable"; + if (!(await this.canDispatch(session, admissionGeneration))) + return "authorization-denied"; + + // A durable intent without a durable acceptance acknowledgement is + // irreducibly ambiguous across a crash. Never guess by replaying it: a + // later accepted ledger can complete the dequeue, while automatic replay + // could duplicate a user message already written to the PTY. + if (state.dispatchingInputId !== null) { + try { + state = await this.resolveUncertainDispatch(state); + } catch { + return "transient-failure"; + } + if (state.dispatchingInputId !== null) return "owned"; + const next = state.inputs[0]; + if (!next) return "progressed"; + if (this.hasInputHold(sessionId)) return "owned"; + return this.drain(state, runtimeEpoch); + } + + const prepared: PersistedProjectBootstrapState = { + ...structuredClone(state), + dispatchingInputId: input.id, + }; + try { + await this.persist(input.sessionId, prepared); + } catch { + // No external side effect occurred before the intent commit. + return "transient-failure"; + } + state = prepared; + if (!(await this.canDispatch(session, admissionGeneration))) { + const rollback: PersistedProjectBootstrapState = { + ...structuredClone(state), + dispatchingInputId: null, + }; + try { + await this.persist(input.sessionId, rollback); + } catch { + return "transient-failure"; + } + return "authorization-denied"; + } + + // Register correlation and the live turn gate before crossing the PTY + // boundary. Prompt hooks may run before submitInput resolves. + const queue = this.expected.get(input.sessionId) ?? []; + queue.push({ kind: "user", id: input.id, text: input.text }); + this.expected.set(input.sessionId, queue); + this.activeTurns.set(input.sessionId, { kind: "user", id: input.id }); + let accepted = false; + let durableNotSubmitted: PersistedProjectBootstrapState | null = null; + const persistNotSubmitted = async (): Promise => { + if ( + !this.isAdmissionCurrent(admissionGeneration) || + !this.isRuntimeEpochCurrent(input.sessionId, runtimeEpoch) + ) + return; + const rollback: PersistedProjectBootstrapState = { + ...structuredClone(state), + dispatchingInputId: null, + }; + await this.persist(input.sessionId, rollback); + // Set this only after the rollback is durable. A callback invocation by + // itself is not enough evidence to make the FIFO row replayable after a + // process loss. + durableNotSubmitted = rollback; + }; + try { + accepted = await this.options.sessionManager.submitInput( + input.sessionId, + input.text, + true, + async () => + this.isRuntimeEpochCurrent(input.sessionId, runtimeEpoch) && + (await this.canDispatch(session, admissionGeneration)), + false, + { + canWriteNow: () => + this.isAdmissionCurrent(admissionGeneration) && + this.isRuntimeEpochCurrent(input.sessionId, runtimeEpoch), + onNotSubmitted: persistNotSubmitted, + }, + ); + } catch (error) { + // Readiness and authorization failures are raised before any PTY byte. + // Persist the same positive rollback proof SessionManager supplies for a + // pre-Enter text-write rejection. If that write cannot commit, retain + // the conservative dispatch intent instead of making A replayable. + if ( + durableNotSubmitted === null && + (error instanceof SessionNotReadyError || + (error instanceof SessionInputGuardRejectedError && !error.staged) || + (error instanceof SessionBackgroundInputPreemptedError && + !error.staged)) + ) { + await persistNotSubmitted().catch(() => {}); + } + if (durableNotSubmitted !== null) { + this.clearActiveTurn(input.sessionId, "user", input.id); + this.removeExpectedPrompt(input.sessionId, "user", input.id); + if (error instanceof SessionNotReadyError) return "not-runnable"; + if (error instanceof SessionInputGuardRejectedError) + return "authorization-denied"; + if (error instanceof SessionBackgroundInputPreemptedError) + return "owned"; + return "transient-failure"; + } + if ( + !this.isAdmissionCurrent(admissionGeneration) || + !this.isRuntimeEpochCurrent(input.sessionId, runtimeEpoch) + ) + return "not-runnable"; + + // A generic PTY rejection can come from the Enter write itself. There is + // no safe way to distinguish "provider did not receive it" from "Enter + // crossed and the local write reported failure". Keep correlation and + // active ownership, classify the receipt as submitted, and let this + // turn's own deadline retire it as uncertain before admitting B. + const ambiguous: PersistedProjectBootstrapState = { + ...structuredClone(state), + receipts: structuredClone(state.receipts), + }; + this.updateReceiptStatus(ambiguous, input.id, "submitted"); + this.armActiveTurnTimer(input.sessionId, { + kind: "user", + id: input.id, + }); + await this.persist(input.sessionId, ambiguous).catch(() => {}); + return "owned"; + } + if (!accepted) { + if (!this.isAdmissionCurrent(admissionGeneration)) return "not-runnable"; + this.userNotSubmittedTransitions.set(input.sessionId, input.id); + try { + await this.commitUserNotSubmittedTransition(state, input.id); + } catch { + this.armActiveTurnTimer(input.sessionId, { + kind: "user", + id: input.id, + }); + return "owned"; + } + return "not-runnable"; + } + // Enter has crossed the PTY boundary, so this logical turn owns its own + // bounded deadline immediately. Keep that deadline even if any subsequent + // submitted-state, acknowledgement, or dequeue persistence step fails. + if ( + this.isAdmissionCurrent(admissionGeneration) && + this.isRuntimeEpochCurrent(input.sessionId, runtimeEpoch) + ) { + this.armActiveTurnTimer(input.sessionId, { + kind: "user", + id: input.id, + }); + } + // Enter has crossed the PTY boundary. Persist that fact before any + // acknowledgement/dequeue work so shutdown or response loss cannot make a + // submitted logical turn appear safely replayable. + const submittedState: PersistedProjectBootstrapState = { + ...structuredClone(state), + receipts: structuredClone(state.receipts), + }; + this.updateReceiptStatus(submittedState, input.id, "submitted"); + try { + await this.persist(input.sessionId, submittedState); + } catch { + return "owned"; + } + state = submittedState; + if ( + !this.isAdmissionCurrent(admissionGeneration) || + !this.isRuntimeEpochCurrent(input.sessionId, runtimeEpoch) + ) + return "owned"; + try { + await this.recordAcceptedInput(state, input.id); + } catch { + // The durable intent remains unresolved and will not be replayed after + // restart. True PTY exactly-once is impossible without this ack. + return "owned"; + } + + if ( + !this.isAdmissionCurrent(admissionGeneration) || + !this.isRuntimeEpochCurrent(input.sessionId, runtimeEpoch) + ) + return "owned"; + + const committed: PersistedProjectBootstrapState = { + ...structuredClone(state), + inputs: state.inputs.slice(1), + dispatchingInputId: null, + metadata: { + ...structuredClone(state.metadata), + queuedInputIds: state.metadata.queuedInputIds.slice(1), + }, + uncertainInputIds: state.uncertainInputIds.filter( + (inputId) => inputId !== input.id, + ), + receipts: structuredClone(state.receipts), + }; + this.updateReceiptStatus(committed, input.id, "submitted"); + if ( + !this.isAdmissionCurrent(admissionGeneration) || + !this.isRuntimeEpochCurrent(input.sessionId, runtimeEpoch) + ) + return "owned"; + try { + await this.persist(input.sessionId, committed); + } catch { + // The accepted ledger is durable. A restart will finish this exact + // dequeue without submitting the input twice. + return "owned"; + } + await this.writeAcceptedInputIds(input.sessionId, []).catch(() => {}); + if ( + !this.isAdmissionCurrent(admissionGeneration) || + !this.isRuntimeEpochCurrent(input.sessionId, runtimeEpoch) + ) + return "owned"; + // The next FIFO entry is intentionally not submitted here. Its dispatch + // is released only by this turn's correlated completion. + return "owned"; + } + + /** Add local-only correlation without removing transcript content. */ + decorateLocalEvent( + event: AnalyticsEvent, + runtimeEpoch: string, + ): AnalyticsEvent { + if (!this.isRuntimeEpochCurrent(event.harnessSessionId, runtimeEpoch)) + return event; + const session = this.options.sessionManager.get(event.harnessSessionId); + const preRegistrationRawHold = this.terminalPreemptions.has( + event.harnessSessionId, + ); + if ( + (!session?.projectBootstrap && !preRegistrationRawHold) || + event.type !== "prompt.submitted" + ) + return event; + if ( + this.correlationOverflow.has(event.harnessSessionId) || + this.completionDedupeOverflow.has(event.harnessSessionId) + ) { + // Overflow is a fail-closed correlation epoch. No later prompt may + // establish a trusted barrier until process/session recovery resets the + // in-memory epoch; otherwise an old completion could claim the new turn. + return event; + } + const prompt = + typeof event.payload.prompt === "string" ? event.payload.prompt : ""; + const queue = this.expected.get(event.harnessSessionId) ?? []; + const index = queue.findIndex((entry) => entry.text === prompt); + const [match] = index < 0 ? [] : queue.splice(index, 1); + if (queue.length === 0) this.expected.delete(event.harnessSessionId); + const observed = this.observedAttempts.get(event.harnessSessionId) ?? []; + // One barrier per observed prompt. turn.completed has no attempt token, so + // skipping ordinary/user barriers would let their completion release or + // satisfy a later coordinator-owned turn. + if (observed.length >= MAX_CORRELATION_BARRIERS) { + this.clearCorrelation(event.harnessSessionId); + this.correlationOverflow.add(event.harnessSessionId); + } else { + observed.push( + match?.kind === "bootstrap" + ? { + kind: "bootstrap", + id: match.id, + retired: match.retired === true, + } + : match?.kind === "user" + ? { kind: "user", id: match.id } + : { kind: "external" }, + ); + this.observedAttempts.set(event.harnessSessionId, observed); + } + if (!match) return event; + return { + ...event, + payload: { + ...event.payload, + projectBootstrapOrigin: + match.kind === "bootstrap" ? "infrastructure" : "user", + ...(match.kind === "bootstrap" + ? { projectBootstrapAttemptId: match.id } + : { projectBootstrapInputId: match.id }), + }, + }; + } + + /** Bootstrap telemetry receives no prompt, path, or provider content. */ + redactForTelemetry(event: AnalyticsEvent): AnalyticsEvent { + const session = this.options.sessionManager.get(event.harnessSessionId); + const nextObserved = this.observedAttempts.get(event.harnessSessionId)?.[0]; + const correlatedAttempt = + typeof event.payload.projectBootstrapAttemptId === "string" || + nextObserved?.kind === "bootstrap"; + const activeBootstrapLifecycle = + session?.projectBootstrap !== undefined && + !isTerminal(session.projectBootstrap); + const coordinatedUserInput = + typeof event.payload.projectBootstrapInputId === "string"; + return correlatedAttempt || activeBootstrapLifecycle || coordinatedUserInput + ? { + ...event, + // The normalized hook envelope is attacker-controlled too: every + // hook can supply `payload.session_id`. The harness session ID is the + // server-owned project bootstrap correlation key, so provider identity is not + // needed in remote planner telemetry at all. + agentSessionId: null, + payload: telemetryPayload(event), + } + : event; + } + + async onEventPersisted( + event: AnalyticsEvent, + runtimeEpoch: string, + ): Promise { + if ( + event.type !== "turn.completed" || + !this.isRuntimeEpochCurrent(event.harnessSessionId, runtimeEpoch) + ) + return; + await this.serialize(event.harnessSessionId, async () => { + if (!this.isRuntimeEpochCurrent(event.harnessSessionId, runtimeEpoch)) + return; + if (!this.claimCompletionEvent(event.harnessSessionId, event.eventId)) { + return; + } + let completionCommitted = false; + try { + // Overflow invalidates the whole in-memory correlation epoch. Check it + // before peeking or mutating anything: an old completion must never + // consume a post-overflow prompt or release its active owner. + if (this.correlationOverflow.has(event.harnessSessionId)) return; + + const observed = this.observedAttempts.get(event.harnessSessionId); + const completedTurn = observed?.[0]; + if (completedTurn?.kind === "external") { + // Consuming the exact barrier is an event-attributable effect. Keep + // this completion ID retired even if a later load/redrive fails. + this.consumeObservedTurn(event.harnessSessionId, completedTurn); + completionCommitted = true; + this.terminalPreemptions.delete(event.harnessSessionId); + this.reportedTerminalPreemptions.delete(event.harnessSessionId); + } + + const session = this.options.sessionManager.get(event.harnessSessionId); + if (!session?.projectBootstrap) return; + let state = await this.load(session); + if (this.terminalPreemptionObligations.has(event.harnessSessionId)) { + state = await this.commitTerminalPreemption(state); + } + + if (completedTurn?.kind === "user") { + const receipt = this.receiptForInput(state, completedTurn.id); + if (receipt && receipt.status !== "uncertain") { + if ( + state.dispatchingInputId === completedTurn.id && + state.inputs[0]?.id === completedTurn.id + ) { + const completed: PersistedProjectBootstrapState = { + ...structuredClone(state), + inputs: state.inputs.slice(1), + dispatchingInputId: null, + metadata: { + ...structuredClone(state.metadata), + queuedInputIds: state.metadata.queuedInputIds.slice(1), + }, + uncertainInputIds: state.uncertainInputIds.filter( + (inputId) => inputId !== completedTurn.id, + ), + uncertainInputs: state.uncertainInputs.filter( + (input) => input.id !== completedTurn.id, + ), + receipts: structuredClone(state.receipts), + }; + this.updateReceiptStatus( + completed, + completedTurn.id, + "completed", + ); + await this.persist(event.harnessSessionId, completed); + state = completed; + await this.writeAcceptedInputIds( + event.harnessSessionId, + [], + ).catch(() => {}); + } else { + const completed = structuredClone(state); + this.updateReceiptStatus( + completed, + completedTurn.id, + "completed", + ); + await this.persist(event.harnessSessionId, completed); + state = completed; + } + completionCommitted = true; + this.clearActiveTurn( + event.harnessSessionId, + "user", + completedTurn.id, + ); + } + // A previously terminal uncertain receipt is monotonic, but its + // exact late completion must still consume only its own tombstone. + completionCommitted = true; + this.consumeObservedTurn(event.harnessSessionId, completedTurn); + } + + if (state.metadata.bootstrap.status !== "generating") { + const activeBootstrap = this.activeTurns.get(event.harnessSessionId); + if ( + completedTurn?.kind === "bootstrap" && + !completedTurn.retired && + activeBootstrap?.kind === "bootstrap" && + activeBootstrap.id === completedTurn.id && + state.metadata.bootstrap.status === "failed" && + state.metadata.bootstrap.errorCode === "persistence_failed" + ) { + const assistantText = event.payload.assistantText; + if ( + typeof assistantText !== "string" || + assistantText.trim() === "" + ) { + const transition = this.bootstrapFailureTransitions.get( + event.harnessSessionId, + ) ?? { + attemptId: completedTurn.id, + errorCode: "model_turn_failed" as const, + retryable: true, + correlationRelease: "consume-observed-or-tombstone" as const, + }; + this.bootstrapFailureTransitions.set( + event.harnessSessionId, + transition, + ); + state = await this.commitBootstrapFailureTransition( + state, + transition, + ); + completionCommitted = true; + this.consumeObservedTurn(event.harnessSessionId, completedTurn); + return; + } + const delivered = structuredClone(state); + delivered.metadata.bootstrap = { + status: "delivered", + messageId: event.eventId, + }; + this.markAttempt(delivered, completedTurn.id, "completed"); + await this.persist(event.harnessSessionId, delivered); + state = delivered; + completionCommitted = true; + this.bootstrapFailureTransitions.delete(event.harnessSessionId); + this.consumeObservedTurn(event.harnessSessionId, completedTurn); + this.clearActiveTurn( + event.harnessSessionId, + "bootstrap", + completedTurn.id, + ); + this.emit({ + name: "project_bootstrap.delivered", + projectId: state.metadata.projectId, + sessionId: event.harnessSessionId, + attemptId: completedTurn.id, + queueDepth: state.inputs.length, + }); + if (state.inputs.length > 0) await this.drainWithRecovery(state); + return; + } + if ( + completedTurn?.kind === "bootstrap" && + state.metadata.bootstrap.status === "failed" && + state.metadata.bootstrap.errorCode === "delivery_timeout" && + !state.metadata.bootstrap.retryable && + state.inputs.length === 0 && + state.retryCount < MAX_RETRIES + ) { + const retryable = structuredClone(state); + retryable.metadata.bootstrap = { + status: "failed", + retryable: true, + errorCode: "delivery_timeout", + }; + await this.persist(event.harnessSessionId, retryable); + state = retryable; + completionCommitted = true; + } + if (completedTurn?.kind === "bootstrap") { + completionCommitted = true; + this.consumeObservedTurn(event.harnessSessionId, completedTurn); + this.clearActiveTurn( + event.harnessSessionId, + "bootstrap", + completedTurn.id, + ); + } + if (isTerminal(state.metadata) && state.inputs.length > 0) + await this.drainWithRecovery(state); + return; + } + + const attemptId = state.metadata.bootstrap.attemptId; + if (completedTurn?.kind !== "bootstrap") return; + if (completedTurn.retired || completedTurn.id !== attemptId) { + completionCommitted = true; + this.consumeObservedTurn(event.harnessSessionId, completedTurn); + return; + } + const text = event.payload.assistantText; + if (typeof text !== "string" || text.trim() === "") { + const transition: BootstrapFailureTransitionObligation = { + attemptId, + errorCode: "model_turn_failed", + retryable: true, + correlationRelease: "consume-observed-or-tombstone", + }; + this.bootstrapFailureTransitions.set( + event.harnessSessionId, + transition, + ); + state = await this.commitBootstrapFailureTransition( + state, + transition, + ); + completionCommitted = true; + this.consumeObservedTurn(event.harnessSessionId, completedTurn); + return; + } + + const delivered = structuredClone(state); + delivered.metadata.bootstrap = { + status: "delivered", + messageId: event.eventId, + }; + this.markAttempt(delivered, attemptId, "completed"); + await this.persist(event.harnessSessionId, delivered); + state = delivered; + completionCommitted = true; + this.bootstrapFailureTransitions.delete(event.harnessSessionId); + this.consumeObservedTurn(event.harnessSessionId, completedTurn); + this.clearActiveTurn(event.harnessSessionId, "bootstrap", attemptId); + this.clearTimer(event.harnessSessionId, attemptId); + this.emit({ + name: "project_bootstrap.delivered", + projectId: state.metadata.projectId, + sessionId: event.harnessSessionId, + attemptId, + queueDepth: state.inputs.length, + }); + await this.drainWithRecovery(state); + } catch (error) { + if (!completionCommitted) { + // Failure preceded every durable/correlation effect, so an exact + // retry may safely attempt this same commit. Once any effect lands, + // retirement is monotonic even if successor redrive fails. + this.releaseCompletionEvent(event.harnessSessionId, event.eventId); + } else { + this.inputRedrainNeeded.add(event.harnessSessionId); + this.scheduleInputRedrain(event.harnessSessionId, runtimeEpoch); + } + throw error; + } + }); + } +}