From b65119ead9b7823b0764765b0452a33d662aa531 Mon Sep 17 00:00:00 2001 From: Yash Date: Fri, 4 Sep 2026 18:27:45 +0000 Subject: [PATCH 01/19] feat(harness): define neutral subsession delegation contracts Refs: SAP-3151 --- packages/harness/src/index.ts | 40 +++ .../subsession-delegation-codec.test.ts | 186 ++++++++++ .../src/shared/subsession-delegation-codec.ts | 327 ++++++++++++++++++ .../src/shared/subsession-delegation.ts | 225 ++++++++++++ 4 files changed, 778 insertions(+) create mode 100644 packages/harness/src/shared/subsession-delegation-codec.test.ts create mode 100644 packages/harness/src/shared/subsession-delegation-codec.ts create mode 100644 packages/harness/src/shared/subsession-delegation.ts diff --git a/packages/harness/src/index.ts b/packages/harness/src/index.ts index 429f714a..de812f0c 100644 --- a/packages/harness/src/index.ts +++ b/packages/harness/src/index.ts @@ -57,6 +57,46 @@ export { agentMapVersionRefsEqual, projectBuildPlanVersionRefsEqual, } from "./shared/build-plan.js"; +export { + PROJECT_SUBSESSION_CLAIM_TTL_MS, + PROJECT_SUBSESSION_DELEGATION_LIMIT, + PROJECT_SUBSESSION_KEY_BYTES, + PROJECT_SUBSESSION_KICKOFF_CONTEXT_BYTES, + PROJECT_SUBSESSION_OUTCOME_BYTES, + PROJECT_SUBSESSION_REQUEST_BYTES, + PROJECT_SUBSESSION_SCHEMA_VERSION, + SUBSESSION_COORDINATOR_STORAGE_SCHEMA_VERSION, +} from "./shared/subsession-delegation.js"; +export type { + CanonicalDelegationBindingDigest, + CanonicalDelegationRequestDigest, + DelegatedContextState, + DelegatedKickoffState, + DelegatedSessionState, + DelegationError, + DelegationErrorCode, + DelegationFocusRef, + DelegationItemOutcome, + DelegationItemResult, + DelegationRecovery, + ProjectSubsessionDelegation, + ProjectSubsessionRequest, + ProjectSubsessionResult, + SubsessionBindingId, + SubsessionBindingRecord, + SubsessionClaim, + SubsessionContextDigest, + SubsessionKickoffDelivery, + SubsessionProjectionDigest, + SubsessionRuntimeBinding, +} from "./shared/subsession-delegation.js"; +export { + computeCanonicalDelegationBindingDigest, + computeCanonicalDelegationRequestDigest, + computeSubsessionContextDigest, + parseProjectSubsessionRequest, + SubsessionDelegationValidationError, +} from "./shared/subsession-delegation-codec.js"; export { canonicalWorkstreamScopes, canonicalizeAgentBriefFocusScope, diff --git a/packages/harness/src/shared/subsession-delegation-codec.test.ts b/packages/harness/src/shared/subsession-delegation-codec.test.ts new file mode 100644 index 00000000..bc89721d --- /dev/null +++ b/packages/harness/src/shared/subsession-delegation-codec.test.ts @@ -0,0 +1,186 @@ +import { describe, expect, it } from "vitest"; + +import { + computeCanonicalDelegationBindingDigest, + computeCanonicalDelegationRequestDigest, + parseProjectSubsessionRequest, + SubsessionDelegationValidationError, +} from "./subsession-delegation-codec.js"; + +const projectId = "project_00000000-0000-4000-8000-000000000001"; +const map = { + projectId, + versionId: "mapv_018f0000-0000-7000-8000-000000000001", + contentDigest: `sha256:${"1".repeat(64)}`, +}; +const plan = { + projectId, + planId: "plan_018f0000-0000-7000-8000-000000000002", + versionId: "planv_018f0000-0000-7000-8000-000000000003", + semanticDigest: `sha256:${"2".repeat(64)}`, +}; + +const request = (delegations: unknown[]) => ({ + schemaVersion: 1, + requestKey: "request-1", + operation: { kind: "delegate", delegations }, +}); + +describe("subsession delegation codec", () => { + it("normalizes text and canonicalizes batch order before hashing", () => { + const left = parseProjectSubsessionRequest( + request([ + { + delegationKey: "publisher", + outcome: "Publish e\u0301vidence\r\nwithout changing scope", + focus: { + kind: "assignment", + map, + plan, + assignmentId: "work_018f0000-0000-7000-8000-000000000004", + }, + }, + { delegationKey: "research", outcome: "Collect evidence" }, + ]), + projectId, + ); + const right = parseProjectSubsessionRequest( + request([ + { delegationKey: "research", outcome: "Collect evidence" }, + { + delegationKey: "publisher", + outcome: "Publish évidence\nwithout changing scope", + focus: { + kind: "assignment", + map, + plan, + assignmentId: "work_018f0000-0000-7000-8000-000000000004", + }, + }, + ]), + projectId, + ); + + expect(left).toEqual(right); + expect(computeCanonicalDelegationRequestDigest(left)).toBe( + computeCanonicalDelegationRequestDigest(right), + ); + expect(left.operation.kind).toBe("delegate"); + if (left.operation.kind === "delegate") { + expect(left.operation.delegations.map((entry) => entry.delegationKey)).toEqual([ + "publisher", + "research", + ]); + } + }); + + it("separates request identity from immutable binding content", () => { + const first = parseProjectSubsessionRequest( + request([{ delegationKey: "research", outcome: "Collect evidence" }]), + projectId, + ); + const second = parseProjectSubsessionRequest( + { + ...request([ + { delegationKey: "research", outcome: "Collect evidence" }, + ]), + requestKey: "request-2", + }, + projectId, + ); + expect(computeCanonicalDelegationRequestDigest(first)).not.toBe( + computeCanonicalDelegationRequestDigest(second), + ); + if (first.operation.kind !== "delegate" || second.operation.kind !== "delegate") + throw new Error("unexpected operation"); + expect( + computeCanonicalDelegationBindingDigest(first.operation.delegations[0]!), + ).toBe( + computeCanonicalDelegationBindingDigest(second.operation.delegations[0]!), + ); + }); + + it.each([ + ["empty batch", request([]), "capacity_exceeded"], + [ + "duplicate keys", + request([ + { delegationKey: "same", outcome: "First" }, + { delegationKey: "same", outcome: "Second" }, + ]), + "invalid_request", + ], + [ + "separator in key", + request([{ delegationKey: "parent/child", outcome: "Do work" }]), + "invalid_request", + ], + [ + "oversized outcome", + request([{ delegationKey: "large", outcome: "x".repeat(4_097) }]), + "invalid_request", + ], + [ + "unsupported schema", + { ...request([{ delegationKey: "one", outcome: "Do work" }]), schemaVersion: 2 }, + "unsupported_schema", + ], + ])("rejects %s before side effects", (_name, input, code) => { + expect(() => parseProjectSubsessionRequest(input, projectId)).toThrowError( + expect.objectContaining({ code }), + ); + }); + + it("rejects exact focus from another project", () => { + let error: unknown; + try { + parseProjectSubsessionRequest( + request([ + { + delegationKey: "foreign", + outcome: "Do work", + focus: { + kind: "map-node", + map: { ...map, projectId: "project_foreign" }, + plan: null, + nodeId: "node_018f0000-0000-7000-8000-000000000005", + }, + }, + ]), + projectId, + ); + } catch (failure) { + error = failure; + } + expect(error).toBeInstanceOf(SubsessionDelegationValidationError); + expect(error).toMatchObject({ + code: "invalid_request", + issues: [{ code: "invalid_or_cross_project_focus" }], + }); + }); + + it("accepts an exact self refresh without granting arbitrary session selection", () => { + expect( + parseProjectSubsessionRequest( + { + schemaVersion: 1, + requestKey: "refresh-1", + operation: { + kind: "refresh-focused-context", + target: { kind: "self" }, + expectedContextEpoch: 2, + expectedContextDigest: `sha256:${"3".repeat(64)}`, + focus: null, + }, + }, + projectId, + ), + ).toMatchObject({ + operation: { + target: { kind: "self" }, + expectedContextEpoch: 2, + }, + }); + }); +}); + diff --git a/packages/harness/src/shared/subsession-delegation-codec.ts b/packages/harness/src/shared/subsession-delegation-codec.ts new file mode 100644 index 00000000..d198bc4f --- /dev/null +++ b/packages/harness/src/shared/subsession-delegation-codec.ts @@ -0,0 +1,327 @@ +import { Buffer } from "node:buffer"; + +import { + AGENT_MAP_UUID_V7_PATTERN, + hasAgentMapControlCharacter, +} from "./agent-map-codec.js"; +import { canonicalDigest, canonicalJson } from "./agent-map-canonical.js"; +import { + parseAgentBriefVersionRef, + parseAgentMapVersionRef, + parseProjectBuildPlanVersionRef, +} from "./build-plan-codec.js"; +import { + PROJECT_SUBSESSION_DELEGATION_LIMIT, + PROJECT_SUBSESSION_KEY_BYTES, + PROJECT_SUBSESSION_KICKOFF_CONTEXT_BYTES, + PROJECT_SUBSESSION_OUTCOME_BYTES, + PROJECT_SUBSESSION_REQUEST_BYTES, + PROJECT_SUBSESSION_SCHEMA_VERSION, + type CanonicalDelegationBindingDigest, + type CanonicalDelegationRequestDigest, + type DelegationFocusRef, + type ProjectSubsessionDelegation, + type ProjectSubsessionRequest, + type SubsessionContextDigest, +} from "./subsession-delegation.js"; + +export interface SubsessionDelegationValidationIssue { + path: string; + code: string; +} + +export class SubsessionDelegationValidationError extends Error { + readonly code: "invalid_request" | "unsupported_schema" | "capacity_exceeded"; + readonly issues: readonly SubsessionDelegationValidationIssue[]; + + constructor( + code: "invalid_request" | "unsupported_schema" | "capacity_exceeded", + issues: readonly SubsessionDelegationValidationIssue[], + ) { + super("Project subsession request is invalid"); + this.name = "SubsessionDelegationValidationError"; + this.code = code; + this.issues = issues.slice(0, 32).map(({ path, code: issueCode }) => ({ + path: path.slice(0, 256), + code: issueCode.slice(0, 128), + })); + } +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +const hasExactKeys = ( + value: Record, + required: readonly string[], + optional: readonly string[] = [], +): boolean => { + const keys = Object.keys(value); + const allowed = new Set([...required, ...optional]); + return required.every((key) => keys.includes(key)) && + keys.every((key) => allowed.has(key)); +}; + +const normalizeText = (value: string): string => + value.normalize("NFC").replace(/\r\n?/gu, "\n"); + +const byteLength = (value: string): number => Buffer.byteLength(value, "utf8"); + +const isKey = (value: unknown): value is string => + typeof value === "string" && + byteLength(value) >= 1 && + byteLength(value) <= PROJECT_SUBSESSION_KEY_BYTES && + /^[A-Za-z0-9._-]+$/u.test(value); + +const isPromptText = ( + value: unknown, + maximumBytes: number, +): value is string => + typeof value === "string" && + value.trim().length > 0 && + byteLength(normalizeText(value)) <= maximumBytes && + ![...value].some((character) => { + const point = character.codePointAt(0) ?? 0; + return hasAgentMapControlCharacter(character) && + point !== 0x09 && point !== 0x0a && point !== 0x0d; + }); + +const id = (value: unknown, prefix: string): value is string => + typeof value === "string" && + new RegExp(`^${prefix}_${AGENT_MAP_UUID_V7_PATTERN}$`, "u").test(value); + +const digest = (value: unknown): value is SubsessionContextDigest => + typeof value === "string" && /^sha256:[0-9a-f]{64}$/u.test(value); + +function invalid(path: string, code: string): never { + throw new SubsessionDelegationValidationError("invalid_request", [ + { path, code }, + ]); +} + +function parseFocus( + value: unknown, + expectedProjectId: string, + path: string, +): DelegationFocusRef { + if (!isRecord(value) || typeof value.kind !== "string") + return invalid(path, "invalid_focus"); + try { + if ( + value.kind === "assignment" && + hasExactKeys(value, ["kind", "map", "plan", "assignmentId"]) && + id(value.assignmentId, "work") + ) { + return { + kind: "assignment", + map: parseAgentMapVersionRef(value.map, expectedProjectId), + plan: parseProjectBuildPlanVersionRef(value.plan, expectedProjectId), + assignmentId: value.assignmentId, + } as DelegationFocusRef; + } + if ( + value.kind === "map-node" && + hasExactKeys(value, ["kind", "map", "plan", "nodeId"]) && + id(value.nodeId, "node") + ) { + return { + kind: "map-node", + map: parseAgentMapVersionRef(value.map, expectedProjectId), + plan: + value.plan === null + ? null + : parseProjectBuildPlanVersionRef(value.plan, expectedProjectId), + nodeId: value.nodeId, + } as DelegationFocusRef; + } + if ( + value.kind === "brief" && + hasExactKeys(value, ["kind", "brief"]) + ) { + return { + kind: "brief", + brief: parseAgentBriefVersionRef(value.brief, expectedProjectId), + }; + } + } catch { + // Collapse codec detail into the bounded public issue below. + } + return invalid(path, "invalid_or_cross_project_focus"); +} + +function parseDelegation( + value: unknown, + expectedProjectId: string, + index: number, +): ProjectSubsessionDelegation { + const path = `operation.delegations[${index}]`; + if ( + !isRecord(value) || + !hasExactKeys(value, ["delegationKey", "outcome"], [ + "kickoffContext", + "focus", + ]) || + !isKey(value.delegationKey) || + !isPromptText(value.outcome, PROJECT_SUBSESSION_OUTCOME_BYTES) || + (value.kickoffContext !== undefined && + !isPromptText( + value.kickoffContext, + PROJECT_SUBSESSION_KICKOFF_CONTEXT_BYTES, + )) + ) { + return invalid(path, "invalid_delegation"); + } + return { + delegationKey: normalizeText(value.delegationKey), + outcome: normalizeText(value.outcome), + ...(value.kickoffContext === undefined + ? {} + : { kickoffContext: normalizeText(value.kickoffContext) }), + ...(value.focus === undefined + ? {} + : { focus: parseFocus(value.focus, expectedProjectId, `${path}.focus`) }), + }; +} + +export function parseProjectSubsessionRequest( + value: unknown, + expectedProjectId: string, +): ProjectSubsessionRequest { + if (!isRecord(value)) invalid("$", "expected_object"); + if (value.schemaVersion !== PROJECT_SUBSESSION_SCHEMA_VERSION) { + throw new SubsessionDelegationValidationError("unsupported_schema", [ + { path: "schemaVersion", code: "unsupported_schema" }, + ]); + } + if ( + !hasExactKeys(value, ["schemaVersion", "requestKey", "operation"]) || + !isKey(value.requestKey) || + !isRecord(value.operation) || + typeof value.operation.kind !== "string" + ) { + return invalid("$", "invalid_envelope"); + } + + let operation: ProjectSubsessionRequest["operation"]; + if ( + value.operation.kind === "delegate" && + hasExactKeys(value.operation, ["kind", "delegations"]) && + Array.isArray(value.operation.delegations) + ) { + if ( + value.operation.delegations.length < 1 || + value.operation.delegations.length > PROJECT_SUBSESSION_DELEGATION_LIMIT + ) { + throw new SubsessionDelegationValidationError("capacity_exceeded", [ + { path: "operation.delegations", code: "delegation_count" }, + ]); + } + const delegations = value.operation.delegations + .map((entry, index) => parseDelegation(entry, expectedProjectId, index)) + .sort((left, right) => + left.delegationKey < right.delegationKey + ? -1 + : left.delegationKey > right.delegationKey + ? 1 + : 0, + ); + if ( + new Set(delegations.map(({ delegationKey }) => delegationKey)).size !== + delegations.length + ) { + return invalid("operation.delegations", "duplicate_delegation_key"); + } + operation = { kind: "delegate", delegations }; + } else if ( + value.operation.kind === "refresh-focused-context" && + hasExactKeys(value.operation, [ + "kind", + "target", + "expectedContextEpoch", + "expectedContextDigest", + "focus", + ]) && + isRecord(value.operation.target) && + Number.isSafeInteger(value.operation.expectedContextEpoch) && + (value.operation.expectedContextEpoch as number) > 0 && + digest(value.operation.expectedContextDigest) + ) { + let target: Extract< + ProjectSubsessionRequest["operation"], + { kind: "refresh-focused-context" } + >["target"]; + if ( + value.operation.target.kind === "self" && + hasExactKeys(value.operation.target, ["kind"]) + ) { + target = { kind: "self" }; + } else if ( + value.operation.target.kind === "child" && + hasExactKeys(value.operation.target, ["kind", "delegationKey"]) && + isKey(value.operation.target.delegationKey) + ) { + target = { + kind: "child", + delegationKey: normalizeText(value.operation.target.delegationKey), + }; + } else { + return invalid("operation.target", "invalid_target"); + } + operation = { + kind: "refresh-focused-context", + target, + expectedContextEpoch: value.operation.expectedContextEpoch as number, + expectedContextDigest: value.operation.expectedContextDigest, + focus: + value.operation.focus === null + ? null + : parseFocus( + value.operation.focus, + expectedProjectId, + "operation.focus", + ), + }; + } else { + return invalid("operation", "invalid_operation"); + } + + const parsed: ProjectSubsessionRequest = { + schemaVersion: PROJECT_SUBSESSION_SCHEMA_VERSION, + requestKey: normalizeText(value.requestKey), + operation, + }; + if (byteLength(canonicalJson(parsed)) > PROJECT_SUBSESSION_REQUEST_BYTES) { + throw new SubsessionDelegationValidationError("capacity_exceeded", [ + { path: "$", code: "request_bytes" }, + ]); + } + return parsed; +} + +export function computeCanonicalDelegationRequestDigest( + request: ProjectSubsessionRequest, +): CanonicalDelegationRequestDigest { + return canonicalDigest( + "sapiom.project-subsession.request.v1", + request, + ) as CanonicalDelegationRequestDigest; +} + +export function computeCanonicalDelegationBindingDigest( + delegation: ProjectSubsessionDelegation, +): CanonicalDelegationBindingDigest { + return canonicalDigest( + "sapiom.project-subsession.binding.v1", + delegation, + ) as CanonicalDelegationBindingDigest; +} + +export function computeSubsessionContextDigest( + focus: DelegationFocusRef | null, +): SubsessionContextDigest { + return canonicalDigest( + "sapiom.project-subsession.context.v1", + focus, + ) as SubsessionContextDigest; +} + diff --git a/packages/harness/src/shared/subsession-delegation.ts b/packages/harness/src/shared/subsession-delegation.ts new file mode 100644 index 00000000..444f2ce5 --- /dev/null +++ b/packages/harness/src/shared/subsession-delegation.ts @@ -0,0 +1,225 @@ +import type { + AgentMapVersionRef, + PlanNodeId, + StudioProjectId, +} from "./agent-map.js"; +import type { + AgentBriefVersionRef, + PlanningAssignmentId, + ProjectBuildPlanVersionRef, +} from "./build-plan.js"; + +export const PROJECT_SUBSESSION_SCHEMA_VERSION = 1 as const; +export const SUBSESSION_COORDINATOR_STORAGE_SCHEMA_VERSION = 1 as const; + +export const PROJECT_SUBSESSION_DELEGATION_LIMIT = 16; +export const PROJECT_SUBSESSION_KEY_BYTES = 128; +export const PROJECT_SUBSESSION_OUTCOME_BYTES = 4 * 1_024; +export const PROJECT_SUBSESSION_KICKOFF_CONTEXT_BYTES = 16 * 1_024; +export const PROJECT_SUBSESSION_REQUEST_BYTES = 64 * 1_024; +export const PROJECT_SUBSESSION_CLAIM_TTL_MS = 120_000; + +type Brand = string & { readonly __brand: TBrand }; + +export type CanonicalDelegationRequestDigest = + Brand<"CanonicalDelegationRequestDigest">; +export type CanonicalDelegationBindingDigest = + Brand<"CanonicalDelegationBindingDigest">; +export type SubsessionBindingId = Brand<"SubsessionBindingId">; +export type SubsessionContextDigest = Brand<"SubsessionContextDigest">; +export type SubsessionProjectionDigest = Brand<"SubsessionProjectionDigest">; + +export type DelegationFocusRef = + | Readonly<{ + kind: "assignment"; + map: AgentMapVersionRef; + plan: ProjectBuildPlanVersionRef; + assignmentId: PlanningAssignmentId; + }> + | Readonly<{ + kind: "map-node"; + map: AgentMapVersionRef; + plan: ProjectBuildPlanVersionRef | null; + nodeId: PlanNodeId; + }> + | Readonly<{ + kind: "brief"; + brief: AgentBriefVersionRef; + }>; + +export type ProjectSubsessionDelegation = Readonly<{ + delegationKey: string; + outcome: string; + kickoffContext?: string; + focus?: DelegationFocusRef; +}>; + +export type ProjectSubsessionRequest = Readonly<{ + schemaVersion: typeof PROJECT_SUBSESSION_SCHEMA_VERSION; + requestKey: string; + operation: + | Readonly<{ + kind: "delegate"; + delegations: readonly ProjectSubsessionDelegation[]; + }> + | Readonly<{ + kind: "refresh-focused-context"; + target: + | Readonly<{ kind: "self" }> + | Readonly<{ kind: "child"; delegationKey: string }>; + expectedContextEpoch: number; + expectedContextDigest: SubsessionContextDigest; + focus: DelegationFocusRef | null; + }>; +}>; + +export type DelegationErrorCode = + | "invalid_capability" + | "expired_capability" + | "revoked_capability" + | "capability_scope_mismatch" + | "invalid_request" + | "unsupported_schema" + | "capacity_exceeded" + | "request_key_reused" + | "storage_unavailable" + | "internal_error" + | "delegation_key_reused" + | "context_not_found" + | "context_stale" + | "context_refresh_conflict" + | "binding_session_mismatch" + | "session_incompatible" + | "session_unreachable" + | "session_closed" + | "adapter_unavailable" + | "adapter_identity_ambiguous" + | "session_create_failed" + | "session_restart_failed" + | "readiness_timeout" + | "kickoff_failed"; + +export type DelegationRecovery = + | "none" + | "correct" + | "retry" + | "reread" + | "refresh_context" + | "inspect_session" + | "new_request_key" + | "new_delegation_key" + | "reduce_request"; + +export type DelegationError = Readonly<{ + code: DelegationErrorCode; + retryable: boolean; + recovery: DelegationRecovery; + issues?: readonly Readonly<{ path: string; code: string }>[]; +}>; + +export type DelegatedSessionState = + | "reserved" + | "spawn-claimed" + | "starting" + | "awaiting-ready" + | "ready" + | "exited" + | "failed" + | "closed"; + +export type DelegatedContextState = + | "none" + | "current" + | "stale" + | "refreshing"; + +export type DelegatedKickoffState = + | "pending" + | "claimed" + | "submitted-unacknowledged" + | "acknowledged" + | "uncertain"; + +export type DelegationItemOutcome = + | "created" + | "reused" + | "already-running" + | "failed"; + +export type DelegationItemResult = Readonly<{ + delegationKey: string; + bindingId: SubsessionBindingId | null; + sessionId: string | null; + outcome: DelegationItemOutcome; + sessionState: DelegatedSessionState; + contextState: DelegatedContextState; + kickoffState: DelegatedKickoffState; + error?: DelegationError; +}>; + +export type ProjectSubsessionResult = Readonly<{ + schemaVersion: typeof PROJECT_SUBSESSION_SCHEMA_VERSION; + requestKey: string; + requestDigest: CanonicalDelegationRequestDigest; + replayed: boolean; + results: readonly DelegationItemResult[]; +}>; + +export type SubsessionClaim = Readonly<{ + claimId: string; + ownerId: string; + claimedAt: string; + expiresAt: string; +}>; + +export type SubsessionRuntimeBinding = Readonly<{ + runtimeToken: string; + incarnation: number; + spawnEpoch: number; +}>; + +export type SubsessionKickoffDelivery = Readonly<{ + contextEpoch: number; + deliveryId: string; + inputId: string; + eventWatermark: string | null; + state: DelegatedKickoffState; + attempt: number; + claim: SubsessionClaim | null; + submittedAt: string | null; + acknowledgedAt: string | null; +}>; + +/** + * Durable coordinator ownership. The private SessionManager marker added by + * the runtime slice must match project, parent, binding, session, and + * incarnation before this record authorizes any session mutation. + */ +export type SubsessionBindingRecord = Readonly<{ + bindingId: SubsessionBindingId; + projectId: StudioProjectId; + parentSessionId: string; + delegationKey: string; + bindingDigest: CanonicalDelegationBindingDigest; + outcome: string; + kickoffContext: string | null; + initialFocus: DelegationFocusRef | null; + sessionId: string; + harness: "claude-code" | "codex"; + projectRoot: string; + lifecycleEpoch: number; + spawnEpoch: number; + contextEpoch: number; + contextDigest: SubsessionContextDigest; + contextState: DelegatedContextState; + currentFocus: DelegationFocusRef | null; + projectionDigest: SubsessionProjectionDigest | null; + sessionState: DelegatedSessionState; + spawnClaim: SubsessionClaim | null; + runtime: SubsessionRuntimeBinding | null; + deliveries: readonly SubsessionKickoffDelivery[]; + lastError: DelegationError | null; + createdAt: string; + updatedAt: string; +}>; + From 185b63d1f5d4973b8f8c386ad385115e86408795 Mon Sep 17 00:00:00 2001 From: Yash Date: Fri, 4 Sep 2026 18:30:43 +0000 Subject: [PATCH 02/19] feat(harness): persist fenced subsession claims Refs: SAP-3151 --- .../core/subsession-coordinator-store.test.ts | 390 +++++ .../src/core/subsession-coordinator-store.ts | 1317 +++++++++++++++++ 2 files changed, 1707 insertions(+) create mode 100644 packages/harness/src/core/subsession-coordinator-store.test.ts create mode 100644 packages/harness/src/core/subsession-coordinator-store.ts diff --git a/packages/harness/src/core/subsession-coordinator-store.test.ts b/packages/harness/src/core/subsession-coordinator-store.test.ts new file mode 100644 index 00000000..c663e6d1 --- /dev/null +++ b/packages/harness/src/core/subsession-coordinator-store.test.ts @@ -0,0 +1,390 @@ +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { ProjectAgentSession } from "../shared/agent-map.js"; +import type { SubsessionBindingId } from "../shared/subsession-delegation.js"; +import { + SubsessionCoordinatorStore, + SubsessionCoordinatorStoreError, +} from "./subsession-coordinator-store.js"; + +const projectId = "project_00000000-0000-4000-8000-000000000001"; +const identity: ProjectAgentSession = { + projectId, + userId: "user-1", + sessionId: "parent-session-1", +}; +const target = { harness: "codex" as const, projectRoot: "/project/root" }; + +const delegate = ( + requestKey = "request-1", + delegations: Array<{ + delegationKey: string; + outcome: string; + kickoffContext?: string; + }> = [{ delegationKey: "research", outcome: "Collect evidence" }], +) => ({ + schemaVersion: 1, + requestKey, + operation: { kind: "delegate", delegations }, +}); +describe("SubsessionCoordinatorStore", () => { + const roots: string[] = []; + + afterEach(async () => { + await Promise.all( + roots.splice(0).map((root) => + fs.rm(root, { recursive: true, force: true }), + ), + ); + }); + + async function fixture() { + const root = await fs.mkdtemp( + path.join(os.tmpdir(), "subsession-coordinator-store-"), + ); + roots.push(root); + return root; + } + + it("converges concurrent instances on one receipt, binding, and reserved real session id", async () => { + const root = await fixture(); + const firstEvent = vi.fn(); + const secondEvent = vi.fn(); + const first = new SubsessionCoordinatorStore(root, { + onEvent: firstEvent, + }); + const second = new SubsessionCoordinatorStore(root, { + onEvent: secondEvent, + }); + + const [left, right] = await Promise.all([ + first.reserveDelegations(identity, delegate(), target), + second.reserveDelegations(identity, delegate(), target), + ]); + const restarted = await new SubsessionCoordinatorStore(root).read(projectId); + + expect(left.bindings).toEqual(right.bindings); + expect([left.replayed, right.replayed].sort()).toEqual([false, true]); + expect(restarted.requestReceipts).toHaveLength(1); + expect(restarted.bindings).toHaveLength(1); + expect(restarted.bindings[0]!.sessionId).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u, + ); + const file = path.join(root, "projects", projectId, "subsessions.json"); + expect((await fs.stat(file)).mode & 0o777).toBe(0o600); + expect( + firstEvent.mock.calls.filter( + ([event]) => event.name === "subsession.binding_reserved", + ).length + + secondEvent.mock.calls.filter( + ([event]) => event.name === "subsession.binding_reserved", + ).length, + ).toBe(1); + }); + + it("rejects changed request and binding keys without changing the original", async () => { + const root = await fixture(); + const store = new SubsessionCoordinatorStore(root); + const original = await store.reserveDelegations( + identity, + delegate(), + target, + ); + + await expect( + store.reserveDelegations( + identity, + delegate("request-1", [ + { delegationKey: "research", outcome: "Different task" }, + ]), + target, + ), + ).rejects.toMatchObject({ code: "request_key_reused" }); + await expect( + store.reserveDelegations( + identity, + delegate("request-2", [ + { delegationKey: "research", outcome: "Different task" }, + ]), + target, + ), + ).rejects.toMatchObject({ code: "delegation_key_reused" }); + + const aggregate = await store.read(projectId); + expect(aggregate.requestReceipts).toHaveLength(1); + expect(aggregate.bindings).toEqual(original.bindings); + }); + + it("reuses a compatible binding across request keys and reserves a batch atomically", async () => { + const root = await fixture(); + const store = new SubsessionCoordinatorStore(root); + const first = await store.reserveDelegations(identity, delegate(), target); + const second = await store.reserveDelegations( + identity, + delegate("request-2"), + target, + ); + expect(second.replayed).toBe(false); + expect(second.bindings[0]!.bindingId).toBe(first.bindings[0]!.bindingId); + + await expect( + store.reserveDelegations( + identity, + delegate("request-3", [ + { delegationKey: "publisher", outcome: "Publish evidence" }, + { delegationKey: "research", outcome: "Changed task" }, + ]), + target, + ), + ).rejects.toMatchObject({ code: "delegation_key_reused" }); + const aggregate = await store.read(projectId); + expect(aggregate.bindings.map(({ delegationKey }) => delegationKey)).toEqual([ + "research", + ]); + expect(aggregate.requestReceipts).toHaveLength(2); + }); + + it.each(["write", "file-sync", "rename", "directory-sync"] as const)( + "exposes only complete state when %s fails", + async (failedStep) => { + const root = await fixture(); + let fail = false; + const store = new SubsessionCoordinatorStore(root, { + beforePersistStep: (step) => { + if (fail && step === failedStep) throw new Error("injected failure"); + }, + }); + await store.read(projectId); + fail = true; + await expect( + store.reserveDelegations(identity, delegate(), target), + ).rejects.toMatchObject({ code: "storage_unavailable" }); + + const restarted = await new SubsessionCoordinatorStore(root).read( + projectId, + ); + expect(restarted.bindings.length).toBe( + failedStep === "directory-sync" ? 1 : 0, + ); + expect(restarted.requestReceipts.length).toBe(restarted.bindings.length); + }, + ); + + it("allows one spawn claimant and requires inspection before expired takeover", async () => { + const root = await fixture(); + let now = new Date("2026-09-04T12:00:00.000Z"); + const options = { + now: () => now, + claimTtlMs: 1_000, + }; + const first = new SubsessionCoordinatorStore(root, options); + const second = new SubsessionCoordinatorStore(root, options); + const reserved = await first.reserveDelegations( + identity, + delegate(), + target, + ); + const binding = reserved.bindings[0]!; + + const [left, right] = await Promise.all([ + first.claimSpawn(identity, binding.bindingId, { + ownerId: "coordinator-1", + expectedLifecycleEpoch: 1, + expectedSpawnEpoch: 0, + }), + second.claimSpawn(identity, binding.bindingId, { + ownerId: "coordinator-2", + expectedLifecycleEpoch: 1, + expectedSpawnEpoch: 0, + }), + ]); + const winner = [left, right].find((result) => result.claimed)!; + const loser = [left, right].find((result) => !result.claimed)!; + expect(loser).toMatchObject({ claimed: false, reason: "active" }); + + now = new Date("2026-09-04T12:00:02.000Z"); + const observed = await second.claimSpawn(identity, binding.bindingId, { + ownerId: "coordinator-2", + expectedLifecycleEpoch: winner.binding.lifecycleEpoch, + expectedSpawnEpoch: winner.binding.spawnEpoch, + }); + expect(observed).toMatchObject({ + claimed: false, + reason: "expired-requires-inspection", + }); + if (!winner.claimed || !winner.binding.spawnClaim) + throw new Error("missing winning claim"); + const takeover = await second.takeoverExpiredSpawnClaim( + identity, + binding.bindingId, + { + ownerId: "coordinator-2", + expiredClaimId: winner.binding.spawnClaim.claimId, + expectedLifecycleEpoch: winner.binding.lifecycleEpoch, + expectedSpawnEpoch: winner.binding.spawnEpoch, + }, + ); + expect(takeover.claimed).toBe(true); + expect(takeover.binding.spawnEpoch).toBe(2); + }); + + it("fences stale spawn callbacks and only releases a claim with zero-process proof", async () => { + const root = await fixture(); + const store = new SubsessionCoordinatorStore(root); + const binding = ( + await store.reserveDelegations(identity, delegate(), target) + ).bindings[0]!; + const claim = await store.claimSpawn(identity, binding.bindingId, { + ownerId: "coordinator-1", + expectedLifecycleEpoch: 1, + expectedSpawnEpoch: 0, + }); + if (!claim.claimed || !claim.binding.spawnClaim) + throw new Error("claim was not acquired"); + + await expect( + store.attachSpawnedRuntime(identity, binding.bindingId, { + claimId: "claim_stale", + spawnEpoch: claim.binding.spawnEpoch, + runtimeToken: "runtime-stale", + incarnation: 1, + }), + ).rejects.toMatchObject({ code: "claim_conflict" }); + const released = await store.releaseUnspawnedClaim( + identity, + binding.bindingId, + { + claimId: claim.binding.spawnClaim.claimId, + spawnEpoch: claim.binding.spawnEpoch, + proof: "no-process-created", + }, + ); + expect(released).toMatchObject({ + sessionState: "reserved", + spawnClaim: null, + runtime: null, + }); + }); + + it("persists one kickoff sender and never blindly retries uncertain delivery", async () => { + const root = await fixture(); + const store = new SubsessionCoordinatorStore(root); + const binding = ( + await store.reserveDelegations(identity, delegate(), target) + ).bindings[0]!; + const spawn = await store.claimSpawn(identity, binding.bindingId, { + ownerId: "coordinator-1", + expectedLifecycleEpoch: 1, + expectedSpawnEpoch: 0, + }); + if (!spawn.claimed || !spawn.binding.spawnClaim) + throw new Error("spawn claim was not acquired"); + const starting = await store.attachSpawnedRuntime( + identity, + binding.bindingId, + { + claimId: spawn.binding.spawnClaim.claimId, + spawnEpoch: spawn.binding.spawnEpoch, + runtimeToken: "runtime-1", + incarnation: 1, + }, + ); + const ready = await store.transitionSession(identity, binding.bindingId, { + expectedLifecycleEpoch: starting.lifecycleEpoch, + expectedSpawnEpoch: starting.spawnEpoch, + expectedRuntimeToken: "runtime-1", + state: "ready", + }); + + const [left, right] = await Promise.all([ + store.claimKickoff(identity, binding.bindingId, { + ownerId: "sender-1", + expectedLifecycleEpoch: ready.lifecycleEpoch, + expectedSpawnEpoch: ready.spawnEpoch, + expectedContextEpoch: ready.contextEpoch, + eventWatermark: "event-10", + }), + new SubsessionCoordinatorStore(root).claimKickoff( + identity, + binding.bindingId, + { + ownerId: "sender-2", + expectedLifecycleEpoch: ready.lifecycleEpoch, + expectedSpawnEpoch: ready.spawnEpoch, + expectedContextEpoch: ready.contextEpoch, + eventWatermark: "event-10", + }, + ), + ]); + const winner = [left, right].find((result) => result.claimed)!; + expect([left, right].filter((result) => result.claimed)).toHaveLength(1); + if (!winner.claimed) throw new Error("kickoff claim was not acquired"); + const delivery = winner.binding.deliveries[0]!; + if (!delivery.claim) throw new Error("kickoff claim was not persisted"); + + const uncertain = await store.recordKickoffWrite( + identity, + binding.bindingId, + { + contextEpoch: delivery.contextEpoch, + deliveryId: delivery.deliveryId, + inputId: delivery.inputId, + claimId: delivery.claim.claimId, + phase: "text-staged", + }, + ); + expect(uncertain.deliveries[0]!.state).toBe("uncertain"); + await expect( + store.claimKickoff(identity, binding.bindingId, { + ownerId: "sender-3", + expectedLifecycleEpoch: uncertain.lifecycleEpoch, + expectedSpawnEpoch: uncertain.spawnEpoch, + expectedContextEpoch: uncertain.contextEpoch, + eventWatermark: "event-10", + }), + ).resolves.toMatchObject({ claimed: false, reason: "terminal" }); + + const acknowledged = await store.acknowledgeKickoff( + identity, + binding.bindingId, + { + contextEpoch: delivery.contextEpoch, + deliveryId: delivery.deliveryId, + inputId: delivery.inputId, + eventWatermark: "event-10", + }, + ); + expect(acknowledged.deliveries[0]!.state).toBe("acknowledged"); + await expect( + store.acknowledgeKickoff(identity, binding.bindingId, { + contextEpoch: delivery.contextEpoch, + deliveryId: delivery.deliveryId, + inputId: "input_foreign", + eventWatermark: "event-10", + }), + ).rejects.toBeInstanceOf(SubsessionCoordinatorStoreError); + }); + + it("scopes mutations to the trusted parent and never adopts a foreign binding", async () => { + const root = await fixture(); + const store = new SubsessionCoordinatorStore(root); + const binding = ( + await store.reserveDelegations(identity, delegate(), target) + ).bindings[0]!; + const foreign: ProjectAgentSession = { + ...identity, + sessionId: "manual-session-with-no-binding", + }; + await expect( + store.claimSpawn(foreign, binding.bindingId as SubsessionBindingId, { + ownerId: "coordinator-foreign", + expectedLifecycleEpoch: 1, + expectedSpawnEpoch: 0, + }), + ).rejects.toMatchObject({ code: "binding_scope_mismatch" }); + expect((await store.read(projectId)).bindings[0]).toEqual(binding); + }); +}); diff --git a/packages/harness/src/core/subsession-coordinator-store.ts b/packages/harness/src/core/subsession-coordinator-store.ts new file mode 100644 index 00000000..ac225e71 --- /dev/null +++ b/packages/harness/src/core/subsession-coordinator-store.ts @@ -0,0 +1,1317 @@ +import { randomUUID } from "node:crypto"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; + +import type { ProjectAgentSession, StudioProjectId } from "../shared/agent-map.js"; +import { canonicalDigest } from "../shared/agent-map-canonical.js"; +import { + hasAgentMapControlCharacter, + parseProjectAgentActorRef, +} from "../shared/agent-map-codec.js"; +import type { HarnessKind } from "../shared/types.js"; +import { + computeCanonicalDelegationBindingDigest, + computeCanonicalDelegationRequestDigest, + computeSubsessionContextDigest, + parseProjectSubsessionRequest, +} from "../shared/subsession-delegation-codec.js"; +import { + PROJECT_SUBSESSION_CLAIM_TTL_MS, + PROJECT_SUBSESSION_SCHEMA_VERSION, + SUBSESSION_COORDINATOR_STORAGE_SCHEMA_VERSION, + type CanonicalDelegationRequestDigest, + type DelegatedSessionState, + type DelegationError, + type ProjectSubsessionRequest, + type SubsessionBindingId, + type SubsessionBindingRecord, + type SubsessionClaim, + type SubsessionKickoffDelivery, +} from "../shared/subsession-delegation.js"; +import { DurableFileLock } from "./durable-file-lock.js"; +import { isStudioProjectId } from "./studio-project-catalog.js"; + +export const SUBSESSION_COORDINATOR_BINDING_LIMIT = 8_192; +export const SUBSESSION_COORDINATOR_RECEIPT_LIMIT = 8_192; +export const SUBSESSION_COORDINATOR_DELIVERY_LIMIT = 64; + +export type SubsessionCoordinatorStoreErrorCode = + | "malformed_state" + | "unsupported_schema" + | "storage_unavailable" + | "capacity_exceeded" + | "request_key_reused" + | "delegation_key_reused" + | "binding_not_found" + | "binding_scope_mismatch" + | "lifecycle_conflict" + | "claim_conflict" + | "session_closed"; + +export class SubsessionCoordinatorStoreError extends Error { + constructor( + readonly code: SubsessionCoordinatorStoreErrorCode, + readonly schemaVersion?: number, + ) { + super( + code === "storage_unavailable" + ? "Subsession coordinator storage is unavailable" + : code === "unsupported_schema" + ? "Subsession coordinator state uses an unsupported schema" + : "Subsession coordinator operation was rejected", + ); + this.name = "SubsessionCoordinatorStoreError"; + } +} + +export type SubsessionCoordinatorRequestReceipt = Readonly<{ + parentSessionId: string; + requestKey: string; + requestDigest: CanonicalDelegationRequestDigest; + operation: ProjectSubsessionRequest["operation"]["kind"]; + bindingIds: readonly SubsessionBindingId[]; + createdAt: string; +}>; + +export type SubsessionCoordinatorRequestTombstone = + SubsessionCoordinatorRequestReceipt; + +export type SubsessionCoordinatorAggregate = Readonly<{ + schemaVersion: typeof SUBSESSION_COORDINATOR_STORAGE_SCHEMA_VERSION; + recordVersion: number; + projectId: StudioProjectId; + requestReceipts: readonly SubsessionCoordinatorRequestReceipt[]; + requestTombstones: readonly SubsessionCoordinatorRequestTombstone[]; + bindings: readonly SubsessionBindingRecord[]; + createdAt: string; + updatedAt: string; + aggregateDigest: string; +}>; + +export interface SubsessionCoordinatorStoreEvent { + name: + | "subsession.store_initialized" + | "subsession.binding_reserved" + | "subsession.duplicate_prevented" + | "subsession.spawn_claimed" + | "subsession.kickoff_claimed" + | "subsession.kickoff_uncertain"; + projectId: StudioProjectId; + count?: number; +} + +export interface ReservedDelegations { + replayed: boolean; + requestDigest: CanonicalDelegationRequestDigest; + bindings: readonly SubsessionBindingRecord[]; +} + +export type SpawnClaimResult = + | Readonly<{ claimed: true; binding: SubsessionBindingRecord }> + | Readonly<{ + claimed: false; + reason: "active" | "expired-requires-inspection"; + binding: SubsessionBindingRecord; + }>; + +export type KickoffClaimResult = + | Readonly<{ claimed: true; binding: SubsessionBindingRecord }> + | Readonly<{ + claimed: false; + reason: "already-claimed" | "expired-requires-reconciliation" | "terminal"; + binding: SubsessionBindingRecord; + }>; + +type ShallowMutable = { -readonly [K in keyof T]: T[K] }; +type MutableClaim = ShallowMutable; +type MutableDelivery = Omit< + ShallowMutable, + "claim" +> & { claim: MutableClaim | null }; +type MutableRuntime = ShallowMutable< + NonNullable +>; +type MutableBinding = Omit< + ShallowMutable, + "spawnClaim" | "runtime" | "deliveries" | "lastError" +> & { + spawnClaim: MutableClaim | null; + runtime: MutableRuntime | null; + deliveries: MutableDelivery[]; + lastError: DelegationError | null; +}; +type MutableAggregate = Omit< + ShallowMutable, + "requestReceipts" | "requestTombstones" | "bindings" +> & { + requestReceipts: SubsessionCoordinatorRequestReceipt[]; + requestTombstones: SubsessionCoordinatorRequestTombstone[]; + bindings: MutableBinding[]; +}; + +const storageError = () => + new SubsessionCoordinatorStoreError("storage_unavailable"); + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +const exact = (value: Record, keys: readonly string[]) => { + const actual = Object.keys(value).sort(); + const expected = [...keys].sort(); + return ( + actual.length === expected.length && + actual.every((key, index) => key === expected[index]) + ); +}; + +const timestamp = (value: unknown): value is string => { + if (typeof value !== "string") return false; + try { + return new Date(value).toISOString() === value; + } catch { + return false; + } +}; + +const digest = (value: unknown): value is string => + typeof value === "string" && /^sha256:[0-9a-f]{64}$/u.test(value); + +const identifier = (value: unknown, prefix?: string): value is string => + typeof value === "string" && + value.length > 0 && + value.length <= 256 && + !hasAgentMapControlCharacter(value) && + (prefix === undefined || value.startsWith(`${prefix}_`)); + +const parseClaim = (value: unknown): SubsessionClaim | null => { + if (value === null) return null; + if ( + !isRecord(value) || + !exact(value, ["claimId", "ownerId", "claimedAt", "expiresAt"]) || + !identifier(value.claimId) || + !identifier(value.ownerId) || + !timestamp(value.claimedAt) || + !timestamp(value.expiresAt) || + value.expiresAt <= value.claimedAt + ) { + throw new SubsessionCoordinatorStoreError("malformed_state"); + } + return structuredClone(value) as unknown as SubsessionClaim; +}; + +const parseDelivery = (value: unknown): SubsessionKickoffDelivery => { + if ( + !isRecord(value) || + !exact(value, [ + "contextEpoch", + "deliveryId", + "inputId", + "eventWatermark", + "state", + "attempt", + "claim", + "submittedAt", + "acknowledgedAt", + ]) || + !Number.isSafeInteger(value.contextEpoch) || + (value.contextEpoch as number) < 1 || + !identifier(value.deliveryId) || + !identifier(value.inputId) || + (value.eventWatermark !== null && !identifier(value.eventWatermark)) || + ![ + "pending", + "claimed", + "submitted-unacknowledged", + "acknowledged", + "uncertain", + ].includes(String(value.state)) || + !Number.isSafeInteger(value.attempt) || + (value.attempt as number) < 0 || + (value.submittedAt !== null && !timestamp(value.submittedAt)) || + (value.acknowledgedAt !== null && !timestamp(value.acknowledgedAt)) + ) { + throw new SubsessionCoordinatorStoreError("malformed_state"); + } + const claim = parseClaim(value.claim); + if ( + (value.state === "claimed") !== (claim !== null) || + (value.state === "acknowledged") !== (value.acknowledgedAt !== null) + ) { + throw new SubsessionCoordinatorStoreError("malformed_state"); + } + return { ...structuredClone(value), claim } as unknown as SubsessionKickoffDelivery; +}; + +const parseBoundedError = (value: unknown): DelegationError | null => { + if (value === null) return null; + if (!isRecord(value)) + throw new SubsessionCoordinatorStoreError("malformed_state"); + const allowed = ["code", "retryable", "recovery", "issues"]; + if ( + !Object.keys(value).every((key) => allowed.includes(key)) || + !exact( + Object.fromEntries( + Object.entries(value).filter(([, entry]) => entry !== undefined), + ), + value.issues === undefined + ? ["code", "retryable", "recovery"] + : allowed, + ) || + !identifier(value.code) || + typeof value.retryable !== "boolean" || + !identifier(value.recovery) || + (value.issues !== undefined && + (!Array.isArray(value.issues) || + value.issues.length > 32 || + !value.issues.every( + (issue) => + isRecord(issue) && + exact(issue, ["path", "code"]) && + identifier(issue.path) && + identifier(issue.code), + ))) + ) { + throw new SubsessionCoordinatorStoreError("malformed_state"); + } + return structuredClone(value) as unknown as DelegationError; +}; + +function parseBinding( + value: unknown, + projectId: StudioProjectId, +): SubsessionBindingRecord { + if ( + !isRecord(value) || + !exact(value, [ + "bindingId", + "projectId", + "parentSessionId", + "delegationKey", + "bindingDigest", + "outcome", + "kickoffContext", + "initialFocus", + "sessionId", + "harness", + "projectRoot", + "lifecycleEpoch", + "spawnEpoch", + "contextEpoch", + "contextDigest", + "contextState", + "currentFocus", + "projectionDigest", + "sessionState", + "spawnClaim", + "runtime", + "deliveries", + "lastError", + "createdAt", + "updatedAt", + ]) || + value.projectId !== projectId || + !identifier(value.bindingId, "binding") || + !identifier(value.parentSessionId) || + !identifier(value.sessionId) || + !["claude-code", "codex"].includes(String(value.harness)) || + typeof value.projectRoot !== "string" || + !path.isAbsolute(value.projectRoot) || + !Number.isSafeInteger(value.lifecycleEpoch) || + (value.lifecycleEpoch as number) < 1 || + !Number.isSafeInteger(value.spawnEpoch) || + (value.spawnEpoch as number) < 0 || + !Number.isSafeInteger(value.contextEpoch) || + (value.contextEpoch as number) < 1 || + !digest(value.bindingDigest) || + !digest(value.contextDigest) || + (value.projectionDigest !== null && !digest(value.projectionDigest)) || + !["none", "current", "stale", "refreshing"].includes( + String(value.contextState), + ) || + ![ + "reserved", + "spawn-claimed", + "starting", + "awaiting-ready", + "ready", + "exited", + "failed", + "closed", + ].includes(String(value.sessionState)) || + !timestamp(value.createdAt) || + !timestamp(value.updatedAt) || + !Array.isArray(value.deliveries) || + value.deliveries.length < 1 || + value.deliveries.length > SUBSESSION_COORDINATOR_DELIVERY_LIMIT + ) { + throw new SubsessionCoordinatorStoreError("malformed_state"); + } + + const parsedRequest = parseProjectSubsessionRequest( + { + schemaVersion: PROJECT_SUBSESSION_SCHEMA_VERSION, + requestKey: "persistence-check", + operation: { + kind: "delegate", + delegations: [ + { + delegationKey: value.delegationKey, + outcome: value.outcome, + ...(value.kickoffContext === null + ? {} + : { kickoffContext: value.kickoffContext }), + ...(value.initialFocus === null + ? {} + : { focus: value.initialFocus }), + }, + ], + }, + }, + projectId, + ); + if (parsedRequest.operation.kind !== "delegate") + throw new SubsessionCoordinatorStoreError("malformed_state"); + const delegation = parsedRequest.operation.delegations[0]!; + const currentFocus = + value.currentFocus === null + ? null + : (() => { + const parsed = parseProjectSubsessionRequest( + { + schemaVersion: PROJECT_SUBSESSION_SCHEMA_VERSION, + requestKey: "context-check", + operation: { + kind: "delegate", + delegations: [ + { + delegationKey: "context-check", + outcome: "Context integrity check", + focus: value.currentFocus, + }, + ], + }, + }, + projectId, + ); + if (parsed.operation.kind !== "delegate") + throw new SubsessionCoordinatorStoreError("malformed_state"); + return parsed.operation.delegations[0]!.focus!; + })(); + if ( + computeCanonicalDelegationBindingDigest(delegation) !== + value.bindingDigest || + computeSubsessionContextDigest(currentFocus) !== value.contextDigest + ) { + throw new SubsessionCoordinatorStoreError("malformed_state"); + } + const spawnClaim = parseClaim(value.spawnClaim); + if ((value.sessionState === "spawn-claimed") !== (spawnClaim !== null)) + throw new SubsessionCoordinatorStoreError("malformed_state"); + let runtime: SubsessionBindingRecord["runtime"] = null; + if (value.runtime !== null) { + if ( + !isRecord(value.runtime) || + !exact(value.runtime, ["runtimeToken", "incarnation", "spawnEpoch"]) || + !identifier(value.runtime.runtimeToken) || + !Number.isSafeInteger(value.runtime.incarnation) || + (value.runtime.incarnation as number) < 1 || + value.runtime.spawnEpoch !== value.spawnEpoch + ) { + throw new SubsessionCoordinatorStoreError("malformed_state"); + } + runtime = structuredClone(value.runtime) as SubsessionBindingRecord["runtime"]; + } + const deliveries = value.deliveries.map(parseDelivery); + if ( + new Set(deliveries.map(({ contextEpoch }) => contextEpoch)).size !== + deliveries.length || + deliveries.at(-1)?.contextEpoch !== value.contextEpoch + ) { + throw new SubsessionCoordinatorStoreError("malformed_state"); + } + return { + ...structuredClone(value), + initialFocus: delegation.focus ?? null, + currentFocus, + spawnClaim, + runtime, + deliveries, + lastError: parseBoundedError(value.lastError), + } as unknown as SubsessionBindingRecord; +} + +const aggregateDigest = ( + value: Omit | SubsessionCoordinatorAggregate, +) => + canonicalDigest( + "sapiom.project-subsession.aggregate.v1", + Object.fromEntries( + Object.entries(value).filter(([key]) => key !== "aggregateDigest"), + ), + ); + +function parseReceipt( + value: unknown, +): SubsessionCoordinatorRequestReceipt { + if ( + !isRecord(value) || + !exact(value, [ + "parentSessionId", + "requestKey", + "requestDigest", + "operation", + "bindingIds", + "createdAt", + ]) || + !identifier(value.parentSessionId) || + !identifier(value.requestKey) || + !digest(value.requestDigest) || + !["delegate", "refresh-focused-context"].includes(String(value.operation)) || + !Array.isArray(value.bindingIds) || + value.bindingIds.length > 16 || + !value.bindingIds.every((entry) => identifier(entry, "binding")) || + new Set(value.bindingIds).size !== value.bindingIds.length || + !timestamp(value.createdAt) + ) { + throw new SubsessionCoordinatorStoreError("malformed_state"); + } + return structuredClone(value) as unknown as SubsessionCoordinatorRequestReceipt; +} + +export function parseSubsessionCoordinatorAggregate( + value: unknown, + expectedProjectId: StudioProjectId, +): SubsessionCoordinatorAggregate { + if (!isRecord(value)) + throw new SubsessionCoordinatorStoreError("malformed_state"); + if ( + value.schemaVersion !== SUBSESSION_COORDINATOR_STORAGE_SCHEMA_VERSION + ) { + throw new SubsessionCoordinatorStoreError( + "unsupported_schema", + typeof value.schemaVersion === "number" ? value.schemaVersion : undefined, + ); + } + if ( + !exact(value, [ + "schemaVersion", + "recordVersion", + "projectId", + "requestReceipts", + "requestTombstones", + "bindings", + "createdAt", + "updatedAt", + "aggregateDigest", + ]) || + value.projectId !== expectedProjectId || + !Number.isSafeInteger(value.recordVersion) || + (value.recordVersion as number) < 1 || + !Array.isArray(value.requestReceipts) || + value.requestReceipts.length > SUBSESSION_COORDINATOR_RECEIPT_LIMIT || + !Array.isArray(value.requestTombstones) || + value.requestTombstones.length > SUBSESSION_COORDINATOR_RECEIPT_LIMIT || + !Array.isArray(value.bindings) || + value.bindings.length > SUBSESSION_COORDINATOR_BINDING_LIMIT || + !timestamp(value.createdAt) || + !timestamp(value.updatedAt) || + !digest(value.aggregateDigest) || + aggregateDigest(value as unknown as SubsessionCoordinatorAggregate) !== + value.aggregateDigest + ) { + throw new SubsessionCoordinatorStoreError("malformed_state"); + } + const requestReceipts = value.requestReceipts.map(parseReceipt); + const requestTombstones = value.requestTombstones.map(parseReceipt); + const bindings = value.bindings.map((entry) => + parseBinding(entry, expectedProjectId), + ); + const requestKeys = [...requestReceipts, ...requestTombstones].map( + ({ parentSessionId, requestKey }) => `${parentSessionId}\0${requestKey}`, + ); + const bindingKeys = bindings.map( + ({ parentSessionId, delegationKey }) => + `${parentSessionId}\0${delegationKey}`, + ); + if ( + new Set(requestKeys).size !== requestKeys.length || + new Set(bindingKeys).size !== bindingKeys.length || + new Set(bindings.map(({ bindingId }) => bindingId)).size !== + bindings.length || + new Set(bindings.map(({ sessionId }) => sessionId)).size !== bindings.length + ) { + throw new SubsessionCoordinatorStoreError("malformed_state"); + } + const bindingIds = new Set(bindings.map(({ bindingId }) => bindingId)); + if ( + [...requestReceipts, ...requestTombstones].some(({ bindingIds: ids }) => + ids.some((bindingId) => !bindingIds.has(bindingId)), + ) + ) { + throw new SubsessionCoordinatorStoreError("malformed_state"); + } + return { + ...structuredClone(value), + requestReceipts, + requestTombstones, + bindings, + } as unknown as SubsessionCoordinatorAggregate; +} + +const transitions: Readonly> = { + reserved: ["spawn-claimed", "failed", "closed"], + "spawn-claimed": ["reserved", "starting", "failed", "closed"], + starting: ["awaiting-ready", "ready", "exited", "failed", "closed"], + "awaiting-ready": ["ready", "exited", "failed", "closed"], + ready: ["exited", "failed", "closed"], + exited: ["spawn-claimed", "failed", "closed"], + failed: ["spawn-claimed", "closed"], + closed: [], +}; + +export class SubsessionCoordinatorStore { + private readonly queues = new Map>(); + + constructor( + private readonly agentMapRoot: string, + private readonly options: { + now?: () => Date; + generateId?: () => string; + generateSessionId?: () => string; + claimTtlMs?: number; + onEvent?: (event: SubsessionCoordinatorStoreEvent) => void | Promise; + beforePersistStep?: ( + step: "write" | "file-sync" | "rename" | "directory-sync", + ) => void | Promise; + } = {}, + ) {} + + private filePath(projectId: StudioProjectId): string { + return path.join( + this.agentMapRoot, + "projects", + projectId, + "subsessions.json", + ); + } + + private now(): string { + return (this.options.now?.() ?? new Date()).toISOString(); + } + + private id(): string { + return (this.options.generateId ?? randomUUID)(); + } + + private emit(event: SubsessionCoordinatorStoreEvent): void { + try { + void Promise.resolve(this.options.onEvent?.(event)).catch(() => {}); + } catch { + // Content-free observability cannot alter durable state. + } + } + + private initial(projectId: StudioProjectId): SubsessionCoordinatorAggregate { + const now = this.now(); + const initial = { + schemaVersion: SUBSESSION_COORDINATOR_STORAGE_SCHEMA_VERSION, + recordVersion: 1, + projectId, + requestReceipts: [], + requestTombstones: [], + bindings: [], + createdAt: now, + updatedAt: now, + } as const; + return { ...initial, aggregateDigest: aggregateDigest(initial) }; + } + + private async readDisk(projectId: StudioProjectId): Promise<{ + aggregate: SubsessionCoordinatorAggregate; + created: boolean; + }> { + try { + const decoded = JSON.parse( + await fs.readFile(this.filePath(projectId), "utf8"), + ) as unknown; + return { + aggregate: parseSubsessionCoordinatorAggregate(decoded, projectId), + created: false, + }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") + return { aggregate: this.initial(projectId), created: true }; + if ( + error instanceof SubsessionCoordinatorStoreError && + error.code !== "storage_unavailable" + ) { + throw error; + } + if (error instanceof SyntaxError) + throw new SubsessionCoordinatorStoreError("malformed_state"); + throw storageError(); + } + } + + private async persist( + projectId: StudioProjectId, + aggregate: SubsessionCoordinatorAggregate, + ): Promise { + const file = this.filePath(projectId); + const directory = path.dirname(file); + const temporary = `${file}.tmp-${process.pid}-${randomUUID()}`; + let handle: fs.FileHandle | undefined; + try { + await fs.mkdir(directory, { recursive: true, mode: 0o700 }); + handle = await fs.open(temporary, "wx", 0o600); + await this.options.beforePersistStep?.("write"); + await handle.writeFile(`${JSON.stringify(aggregate, null, 2)}\n`, "utf8"); + await this.options.beforePersistStep?.("file-sync"); + await handle.sync(); + await handle.close(); + handle = undefined; + await this.options.beforePersistStep?.("rename"); + await fs.rename(temporary, file); + await fs.chmod(file, 0o600); + const directoryHandle = await fs.open(directory, "r"); + try { + await this.options.beforePersistStep?.("directory-sync"); + await directoryHandle.sync(); + } finally { + await directoryHandle.close(); + } + } catch { + throw storageError(); + } finally { + await handle?.close().catch(() => {}); + await fs.rm(temporary, { force: true }).catch(() => {}); + } + } + + private enqueue( + projectId: StudioProjectId, + operation: () => Promise, + ): Promise { + const previous = this.queues.get(projectId) ?? Promise.resolve(); + const result = previous.then(operation, operation); + const tail = result.then( + () => undefined, + () => undefined, + ); + this.queues.set(projectId, tail); + void tail.finally(() => { + if (this.queues.get(projectId) === tail) this.queues.delete(projectId); + }); + return result; + } + + private async transact( + projectId: StudioProjectId, + operation: ( + aggregate: MutableAggregate, + ) => Promise<{ value: T; next?: MutableAggregate }>, + ): Promise { + if (!isStudioProjectId(projectId)) + throw new SubsessionCoordinatorStoreError("malformed_state"); + return this.enqueue(projectId, async () => { + const release = await new DurableFileLock(this.filePath(projectId), { + storageError, + }).acquire(); + try { + const loaded = await this.readDisk(projectId); + const outcome = await operation( + structuredClone(loaded.aggregate) as unknown as MutableAggregate, + ); + if (loaded.created || outcome.next) { + const candidate = outcome.next ?? + (structuredClone(loaded.aggregate) as unknown as MutableAggregate); + const sealed = parseSubsessionCoordinatorAggregate( + { + ...candidate, + aggregateDigest: aggregateDigest(candidate), + }, + projectId, + ); + await this.persist(projectId, sealed); + } + if (loaded.created) + this.emit({ name: "subsession.store_initialized", projectId }); + return structuredClone(outcome.value); + } finally { + await release(); + } + }); + } + + read(projectId: StudioProjectId): Promise { + return this.transact(projectId, async (aggregate) => ({ value: aggregate })); + } + + async reserveDelegations( + identity: ProjectAgentSession, + rawRequest: unknown, + target: Readonly<{ harness: HarnessKind; projectRoot: string }>, + ): Promise { + parseProjectAgentActorRef({ + userId: identity.userId, + sessionId: identity.sessionId, + }); + const request = parseProjectSubsessionRequest(rawRequest, identity.projectId); + if (request.operation.kind !== "delegate") + throw new SubsessionCoordinatorStoreError("malformed_state"); + if ( + !["claude-code", "codex"].includes(target.harness) || + !path.isAbsolute(target.projectRoot) || + target.projectRoot.includes("\0") + ) { + throw new SubsessionCoordinatorStoreError("malformed_state"); + } + const requestDigest = computeCanonicalDelegationRequestDigest(request); + const operation = request.operation; + return this.transact(identity.projectId, async (aggregate) => { + const sameRequest = ( + receipt: SubsessionCoordinatorRequestReceipt, + ): boolean => + receipt.parentSessionId === identity.sessionId && + receipt.requestKey === request.requestKey; + const previous = + aggregate.requestReceipts.find(sameRequest) ?? + aggregate.requestTombstones.find(sameRequest); + if (previous) { + if ( + previous.requestDigest !== requestDigest || + previous.operation !== "delegate" + ) { + throw new SubsessionCoordinatorStoreError("request_key_reused"); + } + const bindings = previous.bindingIds.map((bindingId) => { + const binding = aggregate.bindings.find( + (entry) => entry.bindingId === bindingId, + ); + if (!binding) + throw new SubsessionCoordinatorStoreError("malformed_state"); + return binding; + }); + this.emit({ + name: "subsession.duplicate_prevented", + projectId: identity.projectId, + count: bindings.length, + }); + return { + value: { replayed: true, requestDigest, bindings }, + }; + } + if ( + aggregate.requestReceipts.length >= + SUBSESSION_COORDINATOR_RECEIPT_LIMIT + ) { + throw new SubsessionCoordinatorStoreError("capacity_exceeded"); + } + + const now = this.now(); + const bindings: SubsessionBindingRecord[] = []; + let created = 0; + for (const delegation of operation.delegations) { + const bindingDigest = + computeCanonicalDelegationBindingDigest(delegation); + const existing = aggregate.bindings.find( + (entry) => + entry.parentSessionId === identity.sessionId && + entry.delegationKey === delegation.delegationKey, + ); + if (existing) { + if (existing.bindingDigest !== bindingDigest) + throw new SubsessionCoordinatorStoreError( + "delegation_key_reused", + ); + bindings.push(existing); + continue; + } + if ( + aggregate.bindings.length + created >= + SUBSESSION_COORDINATOR_BINDING_LIMIT + ) { + throw new SubsessionCoordinatorStoreError("capacity_exceeded"); + } + const contextFocus = delegation.focus ?? null; + const contextEpoch = 1; + const binding: SubsessionBindingRecord = { + bindingId: `binding_${this.id()}` as SubsessionBindingId, + projectId: identity.projectId, + parentSessionId: identity.sessionId, + delegationKey: delegation.delegationKey, + bindingDigest, + outcome: delegation.outcome, + kickoffContext: delegation.kickoffContext ?? null, + initialFocus: contextFocus, + sessionId: (this.options.generateSessionId ?? randomUUID)(), + harness: target.harness, + projectRoot: target.projectRoot, + lifecycleEpoch: 1, + spawnEpoch: 0, + contextEpoch, + contextDigest: computeSubsessionContextDigest(contextFocus), + contextState: contextFocus === null ? "none" : "current", + currentFocus: contextFocus, + projectionDigest: null, + sessionState: "reserved", + spawnClaim: null, + runtime: null, + deliveries: [ + { + contextEpoch, + deliveryId: `delivery_${this.id()}`, + inputId: `input_${this.id()}`, + eventWatermark: null, + state: "pending", + attempt: 0, + claim: null, + submittedAt: null, + acknowledgedAt: null, + }, + ], + lastError: null, + createdAt: now, + updatedAt: now, + }; + aggregate.bindings.push(binding as unknown as MutableBinding); + bindings.push(binding); + created += 1; + } + const receipt: SubsessionCoordinatorRequestReceipt = { + parentSessionId: identity.sessionId, + requestKey: request.requestKey, + requestDigest, + operation: "delegate", + bindingIds: bindings.map(({ bindingId }) => bindingId), + createdAt: now, + }; + aggregate.requestReceipts.push(receipt); + aggregate.recordVersion += 1; + aggregate.updatedAt = now; + this.emit({ + name: "subsession.binding_reserved", + projectId: identity.projectId, + count: created, + }); + return { + value: { replayed: false, requestDigest, bindings }, + next: aggregate, + }; + }); + } + + private scopedBinding( + aggregate: MutableAggregate, + identity: ProjectAgentSession, + bindingId: SubsessionBindingId, + ): MutableBinding { + const binding = aggregate.bindings.find( + (entry) => entry.bindingId === bindingId, + ); + if (!binding) + throw new SubsessionCoordinatorStoreError("binding_not_found"); + if ( + binding.projectId !== identity.projectId || + binding.parentSessionId !== identity.sessionId + ) { + throw new SubsessionCoordinatorStoreError("binding_scope_mismatch"); + } + return binding; + } + + private claim(now: string, ownerId: string): SubsessionClaim { + return { + claimId: `claim_${this.id()}`, + ownerId, + claimedAt: now, + expiresAt: new Date( + new Date(now).getTime() + + (this.options.claimTtlMs ?? PROJECT_SUBSESSION_CLAIM_TTL_MS), + ).toISOString(), + }; + } + + claimSpawn( + identity: ProjectAgentSession, + bindingId: SubsessionBindingId, + request: Readonly<{ + ownerId: string; + expectedLifecycleEpoch: number; + expectedSpawnEpoch: number; + }>, + ): Promise { + return this.transact(identity.projectId, async (aggregate) => { + const binding = this.scopedBinding(aggregate, identity, bindingId); + if (binding.sessionState === "closed") + throw new SubsessionCoordinatorStoreError("session_closed"); + if (binding.spawnClaim) { + return { + value: { + claimed: false, + reason: + binding.spawnClaim.expiresAt <= this.now() + ? "expired-requires-inspection" + : "active", + binding, + }, + }; + } + if ( + binding.lifecycleEpoch !== request.expectedLifecycleEpoch || + binding.spawnEpoch !== request.expectedSpawnEpoch || + !["reserved", "exited", "failed"].includes(binding.sessionState) || + binding.runtime !== null + ) { + throw new SubsessionCoordinatorStoreError("lifecycle_conflict"); + } + const now = this.now(); + binding.spawnEpoch += 1; + binding.lifecycleEpoch += 1; + binding.spawnClaim = this.claim(now, request.ownerId) as MutableClaim; + binding.sessionState = "spawn-claimed"; + binding.updatedAt = now; + aggregate.recordVersion += 1; + aggregate.updatedAt = now; + this.emit({ + name: "subsession.spawn_claimed", + projectId: identity.projectId, + }); + return { + value: { claimed: true, binding }, + next: aggregate, + }; + }); + } + + takeoverExpiredSpawnClaim( + identity: ProjectAgentSession, + bindingId: SubsessionBindingId, + request: Readonly<{ + ownerId: string; + expiredClaimId: string; + expectedLifecycleEpoch: number; + expectedSpawnEpoch: number; + }>, + ): Promise { + return this.transact(identity.projectId, async (aggregate) => { + const binding = this.scopedBinding(aggregate, identity, bindingId); + const now = this.now(); + if ( + !binding.spawnClaim || + binding.spawnClaim.claimId !== request.expiredClaimId || + binding.spawnClaim.expiresAt > now || + binding.lifecycleEpoch !== request.expectedLifecycleEpoch || + binding.spawnEpoch !== request.expectedSpawnEpoch || + binding.sessionState !== "spawn-claimed" || + binding.runtime !== null + ) { + throw new SubsessionCoordinatorStoreError("claim_conflict"); + } + binding.spawnEpoch += 1; + binding.lifecycleEpoch += 1; + binding.spawnClaim = this.claim(now, request.ownerId) as MutableClaim; + binding.updatedAt = now; + aggregate.recordVersion += 1; + aggregate.updatedAt = now; + return { + value: { claimed: true, binding }, + next: aggregate, + }; + }); + } + + releaseUnspawnedClaim( + identity: ProjectAgentSession, + bindingId: SubsessionBindingId, + request: Readonly<{ + claimId: string; + spawnEpoch: number; + proof: "no-process-created"; + }>, + ): Promise { + return this.transact(identity.projectId, async (aggregate) => { + const binding = this.scopedBinding(aggregate, identity, bindingId); + if ( + binding.spawnClaim?.claimId !== request.claimId || + binding.spawnEpoch !== request.spawnEpoch || + binding.sessionState !== "spawn-claimed" || + binding.runtime !== null + ) { + throw new SubsessionCoordinatorStoreError("claim_conflict"); + } + const now = this.now(); + binding.spawnClaim = null; + binding.sessionState = "reserved"; + binding.lifecycleEpoch += 1; + binding.updatedAt = now; + aggregate.recordVersion += 1; + aggregate.updatedAt = now; + return { value: binding, next: aggregate }; + }); + } + + attachSpawnedRuntime( + identity: ProjectAgentSession, + bindingId: SubsessionBindingId, + request: Readonly<{ + claimId: string; + spawnEpoch: number; + runtimeToken: string; + incarnation: number; + }>, + ): Promise { + return this.transact(identity.projectId, async (aggregate) => { + const binding = this.scopedBinding(aggregate, identity, bindingId); + if ( + binding.spawnClaim?.claimId !== request.claimId || + binding.spawnEpoch !== request.spawnEpoch || + binding.sessionState !== "spawn-claimed" || + binding.runtime !== null || + !identifier(request.runtimeToken) || + !Number.isSafeInteger(request.incarnation) || + request.incarnation < 1 + ) { + throw new SubsessionCoordinatorStoreError("claim_conflict"); + } + const now = this.now(); + binding.runtime = { + runtimeToken: request.runtimeToken, + incarnation: request.incarnation, + spawnEpoch: request.spawnEpoch, + }; + binding.spawnClaim = null; + binding.sessionState = "starting"; + binding.lifecycleEpoch += 1; + binding.updatedAt = now; + aggregate.recordVersion += 1; + aggregate.updatedAt = now; + return { value: binding, next: aggregate }; + }); + } + + transitionSession( + identity: ProjectAgentSession, + bindingId: SubsessionBindingId, + request: Readonly<{ + expectedLifecycleEpoch: number; + expectedSpawnEpoch: number; + expectedRuntimeToken: string | null; + state: DelegatedSessionState; + error?: DelegationError | null; + }>, + ): Promise { + return this.transact(identity.projectId, async (aggregate) => { + const binding = this.scopedBinding(aggregate, identity, bindingId); + if ( + binding.lifecycleEpoch !== request.expectedLifecycleEpoch || + binding.spawnEpoch !== request.expectedSpawnEpoch || + (binding.runtime?.runtimeToken ?? null) !== request.expectedRuntimeToken || + !transitions[binding.sessionState].includes(request.state) + ) { + throw new SubsessionCoordinatorStoreError("lifecycle_conflict"); + } + const now = this.now(); + binding.sessionState = request.state; + binding.lifecycleEpoch += 1; + if (["exited", "failed", "closed"].includes(request.state)) + binding.runtime = null; + binding.lastError = request.error ?? null; + binding.updatedAt = now; + aggregate.recordVersion += 1; + aggregate.updatedAt = now; + return { value: binding, next: aggregate }; + }); + } + + claimKickoff( + identity: ProjectAgentSession, + bindingId: SubsessionBindingId, + request: Readonly<{ + ownerId: string; + expectedLifecycleEpoch: number; + expectedSpawnEpoch: number; + expectedContextEpoch: number; + eventWatermark: string; + }>, + ): Promise { + return this.transact(identity.projectId, async (aggregate) => { + const binding = this.scopedBinding(aggregate, identity, bindingId); + const delivery = binding.deliveries.find( + (entry) => entry.contextEpoch === request.expectedContextEpoch, + ); + if (!delivery) + throw new SubsessionCoordinatorStoreError("lifecycle_conflict"); + if ( + binding.lifecycleEpoch !== request.expectedLifecycleEpoch || + binding.spawnEpoch !== request.expectedSpawnEpoch || + binding.contextEpoch !== request.expectedContextEpoch || + binding.sessionState !== "ready" || + !binding.runtime + ) { + throw new SubsessionCoordinatorStoreError("lifecycle_conflict"); + } + if (delivery.state !== "pending") { + const expired = + delivery.state === "claimed" && + delivery.claim !== null && + delivery.claim.expiresAt <= this.now(); + return { + value: { + claimed: false, + reason: expired + ? "expired-requires-reconciliation" + : delivery.state === "claimed" + ? "already-claimed" + : "terminal", + binding, + }, + }; + } + const now = this.now(); + delivery.state = "claimed"; + delivery.attempt += 1; + delivery.claim = this.claim(now, request.ownerId) as MutableClaim; + delivery.eventWatermark = request.eventWatermark; + binding.updatedAt = now; + aggregate.recordVersion += 1; + aggregate.updatedAt = now; + this.emit({ + name: "subsession.kickoff_claimed", + projectId: identity.projectId, + }); + return { + value: { claimed: true, binding }, + next: aggregate, + }; + }); + } + + recordKickoffWrite( + identity: ProjectAgentSession, + bindingId: SubsessionBindingId, + request: Readonly<{ + contextEpoch: number; + deliveryId: string; + inputId: string; + claimId: string; + phase: "not-written" | "text-staged" | "enter-written"; + }>, + ): Promise { + return this.transact(identity.projectId, async (aggregate) => { + const binding = this.scopedBinding(aggregate, identity, bindingId); + const delivery = binding.deliveries.find( + (entry) => entry.contextEpoch === request.contextEpoch, + ); + if ( + !delivery || + delivery.deliveryId !== request.deliveryId || + delivery.inputId !== request.inputId || + delivery.state !== "claimed" || + delivery.claim?.claimId !== request.claimId + ) { + throw new SubsessionCoordinatorStoreError("claim_conflict"); + } + const now = this.now(); + delivery.claim = null; + if (request.phase === "not-written") { + delivery.state = "pending"; + delivery.eventWatermark = null; + } else if (request.phase === "enter-written") { + delivery.state = "submitted-unacknowledged"; + delivery.submittedAt = now; + } else { + delivery.state = "uncertain"; + } + binding.updatedAt = now; + aggregate.recordVersion += 1; + aggregate.updatedAt = now; + if (request.phase === "text-staged") + this.emit({ + name: "subsession.kickoff_uncertain", + projectId: identity.projectId, + }); + return { value: binding, next: aggregate }; + }); + } + + markKickoffUncertain( + identity: ProjectAgentSession, + bindingId: SubsessionBindingId, + request: Readonly<{ + contextEpoch: number; + deliveryId: string; + inputId: string; + }>, + ): Promise { + return this.transact(identity.projectId, async (aggregate) => { + const binding = this.scopedBinding(aggregate, identity, bindingId); + const delivery = binding.deliveries.find( + (entry) => entry.contextEpoch === request.contextEpoch, + ); + if ( + !delivery || + delivery.deliveryId !== request.deliveryId || + delivery.inputId !== request.inputId || + !["claimed", "submitted-unacknowledged", "uncertain"].includes( + delivery.state, + ) + ) { + throw new SubsessionCoordinatorStoreError("claim_conflict"); + } + if (delivery.state === "uncertain") return { value: binding }; + const now = this.now(); + delivery.state = "uncertain"; + delivery.claim = null; + binding.updatedAt = now; + aggregate.recordVersion += 1; + aggregate.updatedAt = now; + this.emit({ + name: "subsession.kickoff_uncertain", + projectId: identity.projectId, + }); + return { value: binding, next: aggregate }; + }); + } + + acknowledgeKickoff( + identity: ProjectAgentSession, + bindingId: SubsessionBindingId, + request: Readonly<{ + contextEpoch: number; + deliveryId: string; + inputId: string; + eventWatermark: string; + }>, + ): Promise { + return this.transact(identity.projectId, async (aggregate) => { + const binding = this.scopedBinding(aggregate, identity, bindingId); + const delivery = binding.deliveries.find( + (entry) => entry.contextEpoch === request.contextEpoch, + ); + if ( + !delivery || + delivery.deliveryId !== request.deliveryId || + delivery.inputId !== request.inputId || + delivery.eventWatermark !== request.eventWatermark || + ![ + "claimed", + "submitted-unacknowledged", + "uncertain", + "acknowledged", + ].includes(delivery.state) + ) { + throw new SubsessionCoordinatorStoreError("claim_conflict"); + } + if (delivery.state === "acknowledged") return { value: binding }; + const now = this.now(); + delivery.state = "acknowledged"; + delivery.claim = null; + delivery.acknowledgedAt = now; + binding.updatedAt = now; + aggregate.recordVersion += 1; + aggregate.updatedAt = now; + return { value: binding, next: aggregate }; + }); + } +} From a8b0cc99ca5dc9285395a4c480ec40c7cb8db2da Mon Sep 17 00:00:00 2001 From: Yash Date: Fri, 4 Sep 2026 18:40:17 +0000 Subject: [PATCH 03/19] feat(harness): recover reserved writable sessions Refs: SAP-3151 --- packages/harness/src/core/errors.ts | 24 + .../harness/src/core/session-manager.test.ts | 213 ++++++++ packages/harness/src/core/session-manager.ts | 484 +++++++++++++++++- packages/harness/src/index.ts | 2 + packages/harness/src/server/rest.ts | 10 +- 5 files changed, 728 insertions(+), 5 deletions(-) diff --git a/packages/harness/src/core/errors.ts b/packages/harness/src/core/errors.ts index abfddbb6..69572152 100644 --- a/packages/harness/src/core/errors.ts +++ b/packages/harness/src/core/errors.ts @@ -96,6 +96,30 @@ export class AgentSessionIdentityReservedError extends HarnessError { } } +/** + * A server-owned reserved session ID did not carry the exact private + * coordinator marker. Manual sessions can never satisfy this check by + * matching cwd, title, assignment, or any other public field. + */ +export class SubsessionBindingMismatchError extends HarnessError { + constructor() { + super( + "SUBSESSION_BINDING_MISMATCH", + "The reserved subsession is not owned by this coordinator binding", + ); + } +} + +/** A same-ID fresh start lacked one of its required zero-turn proofs. */ +export class SubsessionFreshRestartForbiddenError extends HarnessError { + constructor() { + super( + "SUBSESSION_FRESH_RESTART_FORBIDDEN", + "The reserved subsession cannot be restarted as a fresh conversation", + ); + } +} + /** * Thrown when an operation requires a harness adapter that has not been * registered. Maps to HTTP 400. diff --git a/packages/harness/src/core/session-manager.test.ts b/packages/harness/src/core/session-manager.test.ts index 46d2f803..c499d9cd 100644 --- a/packages/harness/src/core/session-manager.test.ts +++ b/packages/harness/src/core/session-manager.test.ts @@ -20,9 +20,12 @@ import { SessionInputIsolationError, SessionManager, SessionManagerClosingError, + SubsessionBindingMismatchError, + SubsessionFreshRestartForbiddenError, sanitizeExitTail, type PtySpawnFn, type SessionManagerOptions, + type TrustedSubsessionBindingMarker, } from "./session-manager.js"; import { IngestCredentialRegistry } from "./ingest-credentials.js"; @@ -139,6 +142,7 @@ describe("SessionManager", () => { ingestCredentials?: SessionManagerOptions["ingestCredentials"]; writeSessionRegistry?: SessionManagerOptions["writeSessionRegistry"]; writeAgentSessionOwnerRegistry?: SessionManagerOptions["writeAgentSessionOwnerRegistry"]; + writeSubsessionBindingRegistry?: SessionManagerOptions["writeSubsessionBindingRegistry"]; /** Pid given to every fake pty this manager spawns — see createFakePty(). */ fakePid?: number; } = {}, @@ -179,6 +183,7 @@ describe("SessionManager", () => { platform: opts.platform, writeSessionRegistry: opts.writeSessionRegistry, writeAgentSessionOwnerRegistry: opts.writeAgentSessionOwnerRegistry, + writeSubsessionBindingRegistry: opts.writeSubsessionBindingRegistry, }); managers.push(manager); return { manager, adapter, spawns }; @@ -204,6 +209,214 @@ describe("SessionManager", () => { expect(manager.list()).toHaveLength(1); }); + const marker = ( + sessionId: string, + incarnation = 1, + spawnEpoch = 1, + ): TrustedSubsessionBindingMarker => ({ + projectId: "project_00000000-0000-4000-8000-000000000001", + parentSessionId: "parent-session-1", + bindingId: "binding-1", + sessionId, + incarnation, + spawnEpoch, + }); + + const delegatedCreate = (sessionId: string) => ({ + cwd: "/tmp/proj", + harness: "claude-code" as const, + trusted: { + agentMapIdentity: () => ({ + projectId: "project_00000000-0000-4000-8000-000000000001", + userId: "user-1", + sessionId, + }), + initialTitle: "Collect evidence", + }, + }); + + it("creates a reserved writable session only with its exact private binding", async () => { + const { manager, adapter } = makeManager(); + const sessionId = "00000000-0000-4000-8000-000000000111"; + const input = delegatedCreate(sessionId); + const session = await manager.createReserved( + sessionId, + { cwd: input.cwd, harness: input.harness }, + marker(sessionId), + input.trusted, + ); + + expect(session).toMatchObject({ + id: sessionId, + status: "running", + title: "Collect evidence", + ready: false, + agentMapIdentity: { + projectId: marker(sessionId).projectId, + sessionId, + }, + }); + expect(adapter.launch).toHaveBeenCalledTimes(1); + expect(manager.matchesSubsessionBinding(marker(sessionId))).toBe(true); + expect(await readFile(sessionsPath, "utf8")).not.toContain("binding-1"); + const sidecar = `${sessionsPath}.subsession-bindings.json`; + expect(JSON.parse(await readFile(sidecar, "utf8"))).toMatchObject({ + version: 1, + markers: { [sessionId]: marker(sessionId) }, + closedSessionIds: [], + }); + expect((await stat(sidecar)).mode & 0o777).toBe(0o600); + + const { manager: restartedManager } = makeManager(); + await restartedManager.init(); + expect(restartedManager.getSubsessionBinding(sessionId)).toEqual( + marker(sessionId), + ); + + await expect( + manager.createReserved( + sessionId, + { cwd: input.cwd, harness: input.harness }, + { ...marker(sessionId), bindingId: "foreign-binding" }, + input.trusted, + ), + ).rejects.toBeInstanceOf(SubsessionBindingMismatchError); + expect(adapter.launch).toHaveBeenCalledTimes(1); + }); + + it("never adopts a manual row merely because its reserved id matches", async () => { + const { manager } = makeManager(); + const manual = await manager.create({ + cwd: "/tmp/proj", + harness: "claude-code", + }); + const input = delegatedCreate(manual.id); + await expect( + manager.createReserved( + manual.id, + { cwd: input.cwd, harness: input.harness }, + marker(manual.id), + input.trusted, + ), + ).rejects.toBeInstanceOf(SubsessionBindingMismatchError); + expect(manager.getSubsessionBinding(manual.id)).toBeNull(); + }); + + it("fresh-restarts an exact zero-turn bound row under the same Harness id", async () => { + const { manager, adapter, spawns } = makeManager(); + const sessionId = "00000000-0000-4000-8000-000000000112"; + const input = delegatedCreate(sessionId); + await manager.createReserved( + sessionId, + { cwd: input.cwd, harness: input.harness }, + marker(sessionId), + input.trusted, + ); + spawns[0]!.emitExit(1); + await manager.flush(); + + const restarted = await manager.restartFreshBound( + sessionId, + marker(sessionId), + marker(sessionId, 2, 2), + input.trusted, + async () => false, + ); + expect(restarted).toMatchObject({ id: sessionId, status: "running" }); + expect(manager.list().filter(({ id }) => id === sessionId)).toHaveLength(1); + expect(manager.getSubsessionBinding(sessionId)).toEqual( + marker(sessionId, 2, 2), + ); + expect(adapter.launch).toHaveBeenCalledTimes(2); + }); + + it("refuses a fresh bound restart when any recorded turn exists", async () => { + const { manager, spawns } = makeManager(); + const sessionId = "00000000-0000-4000-8000-000000000113"; + const input = delegatedCreate(sessionId); + await manager.createReserved( + sessionId, + { cwd: input.cwd, harness: input.harness }, + marker(sessionId), + input.trusted, + ); + spawns[0]!.emitExit(1); + await manager.flush(); + await expect( + manager.restartFreshBound( + sessionId, + marker(sessionId), + marker(sessionId, 2, 2), + input.trusted, + async () => true, + ), + ).rejects.toBeInstanceOf(SubsessionFreshRestartForbiddenError); + expect(manager.getSubsessionBinding(sessionId)).toEqual(marker(sessionId)); + }); + + it("persists an explicit delegated-session close and forbids automatic resurrection", async () => { + const { manager, spawns } = makeManager(); + const sessionId = "00000000-0000-4000-8000-000000000114"; + const input = delegatedCreate(sessionId); + await manager.createReserved( + sessionId, + { cwd: input.cwd, harness: input.harness }, + marker(sessionId), + input.trusted, + ); + await manager.close(sessionId); + expect(manager.wasSubsessionClosedByUser(marker(sessionId))).toBe(true); + spawns[0]!.emitExit(0); + await manager.flush(); + await expect( + manager.restartFreshBound( + sessionId, + marker(sessionId), + marker(sessionId, 2, 2), + input.trusted, + async () => false, + ), + ).rejects.toBeInstanceOf(SubsessionFreshRestartForbiddenError); + expect( + JSON.parse( + await readFile(`${sessionsPath}.subsession-bindings.json`, "utf8"), + ), + ).toMatchObject({ closedSessionIds: [sessionId] }); + }); + + it("reports exact input write phases and kills only an exact runtime", async () => { + const { manager, spawns } = makeManager(); + const session = await manager.create({ + cwd: "/tmp/proj", + harness: "claude-code", + }); + const runtime = manager.getRuntimeEpoch(session.id)!; + manager.setReady(session.id, runtime); + + const submitted = await manager.submitInputTracked( + session.id, + "Implement the scoped task", + ); + expect(submitted).toEqual({ accepted: true, phase: "enter-written" }); + expect(await manager.killIfRuntime(session.id, "foreign-runtime")).toBe( + false, + ); + expect(spawns[0]!.pty.kill).not.toHaveBeenCalled(); + + spawns[0]!.pty.write.mockImplementationOnce(() => { + throw new Error("ambiguous write"); + }); + const ambiguous = await manager.submitInputTracked( + session.id, + "Retry-sensitive task", + ); + expect(ambiguous).toMatchObject({ + accepted: false, + phase: "text-staged", + error: expect.any(Error), + }); + }); + it("closes PTY admission before shutdown and rejects creates and resumes", async () => { let releaseLaunchOptions!: () => void; const launchOptionsReady = new Promise((resolve) => { diff --git a/packages/harness/src/core/session-manager.ts b/packages/harness/src/core/session-manager.ts index d3f673e7..1a14cdc7 100644 --- a/packages/harness/src/core/session-manager.ts +++ b/packages/harness/src/core/session-manager.ts @@ -7,7 +7,15 @@ import { createHash, randomUUID } from "node:crypto"; import { EventEmitter } from "node:events"; -import { mkdir, readFile, rename, writeFile, chmod } from "node:fs/promises"; +import { + chmod, + mkdir, + open, + readFile, + rename, + rm, + writeFile, +} from "node:fs/promises"; import { basename, dirname, join, resolve, sep } from "node:path"; import { createRequire } from "node:module"; @@ -44,6 +52,8 @@ import { SessionAlreadyLiveError, SessionNotReadyError, SessionNotResumeableError, + SubsessionBindingMismatchError, + SubsessionFreshRestartForbiddenError, UnknownSessionError, } from "./errors.js"; import { listHarnessAdapters } from "./adapters/registry.js"; @@ -59,6 +69,8 @@ export { SessionAlreadyLiveError, SessionNotReadyError, SessionNotResumeableError, + SubsessionBindingMismatchError, + SubsessionFreshRestartForbiddenError, UnknownSessionError, } from "./errors.js"; @@ -108,8 +120,31 @@ export interface SessionInputWriteLifecycle { /** Durable positive evidence that the writer returned before attempting * Enter. Errors at the Enter write are intentionally excluded. */ onNotSubmitted?: () => Promise; + /** Synchronous byte-boundary observation for durable delivery recovery. */ + onWritePhase?: (phase: SessionInputWritePhase) => void; } +export type SessionInputWritePhase = + | "not-written" + | "text-staged" + | "enter-written"; + +export type TrackedSessionInputResult = Readonly<{ + accepted: boolean; + phase: SessionInputWritePhase; + error?: unknown; +}>; + +/** Server-private half of a coordinator/session ownership proof. */ +export type TrustedSubsessionBindingMarker = Readonly<{ + projectId: string; + parentSessionId: string; + bindingId: string; + sessionId: string; + incarnation: number; + spawnEpoch: number; +}>; + export interface TerminalInputContext { /** Server-owned identity of the exact PTY receiving these bytes. */ runtimeEpoch: string; @@ -178,6 +213,49 @@ function sameProjectAgent( ); } +function parseTrustedSubsessionBindingMarker( + value: unknown, + expectedSessionId?: string, +): TrustedSubsessionBindingMarker | null { + if ( + !isRecord(value) || + Object.keys(value).sort().join(",") !== + "bindingId,incarnation,parentSessionId,projectId,sessionId,spawnEpoch" || + ![value.projectId, value.parentSessionId, value.bindingId, value.sessionId].every( + (entry) => + typeof entry === "string" && + entry.length > 0 && + entry.length <= 256 && + ![...entry].some((character) => { + const point = character.codePointAt(0) ?? 0; + return point <= 0x1f || point === 0x7f; + }), + ) || + (expectedSessionId !== undefined && value.sessionId !== expectedSessionId) || + !Number.isSafeInteger(value.incarnation) || + (value.incarnation as number) < 1 || + !Number.isSafeInteger(value.spawnEpoch) || + (value.spawnEpoch as number) < 1 + ) { + return null; + } + return structuredClone(value) as TrustedSubsessionBindingMarker; +} + +function sameSubsessionBinding( + left: TrustedSubsessionBindingMarker, + right: TrustedSubsessionBindingMarker, +): boolean { + return ( + left.projectId === right.projectId && + left.parentSessionId === right.parentSessionId && + left.bindingId === right.bindingId && + left.sessionId === right.sessionId && + left.incarnation === right.incarnation && + left.spawnEpoch === right.spawnEpoch + ); +} + function parseBootstrapState( value: unknown, ): ProjectBootstrapMetadata["bootstrap"] | null { @@ -482,6 +560,9 @@ const BRACKETED_PASTE_END = "\x1b[201~"; const AGENT_SESSION_OWNER_FILE_VERSION = 1; const AGENT_SESSION_OWNER_MAX_ENTRIES = 50_000; const AGENT_SESSION_OWNER_MAX_BYTES = 4 * 1024 * 1024; +const SUBSESSION_BINDING_FILE_VERSION = 1; +const SUBSESSION_BINDING_MAX_ENTRIES = 8_192; +const SUBSESSION_BINDING_MAX_BYTES = 2 * 1024 * 1024; /** See `recordActivity()`: minimum gap between two `onActivity` broadcasts * for the same session — pty.onData fires per chunk (often many times a * second for a busy TUI), but the SPA's busy indicator only needs "this @@ -652,6 +733,11 @@ export interface SessionManagerOptions { file: string, serialized: string, ) => Promise; + /** Fault-injection seam for the private coordinator ownership sidecar. */ + writeSubsessionBindingRegistry?: ( + file: string, + serialized: string, + ) => Promise; /** * Writes HARNESS_CONTEXT_FILE for a session — the caller (server/index.ts's * `writeSessionContext`) owns resolving the session's `boundWorkflowPath` @@ -814,6 +900,8 @@ export class SessionManager { * accepted by a HarnessSession. Keeping this outside sessions.json avoids * leaking historical aliases through the browser DTO. */ private readonly agentSessionOwnersPath: string; + /** Never projected through REST; public session fields are not ownership. */ + private readonly subsessionBindingsPath: string; private readonly spawnPty: PtySpawnFn | undefined; private readonly loadSpawnPty: () => Promise; private readonly buildLaunchOpts: LaunchOptsBuilder; @@ -835,6 +923,9 @@ export class SessionManager { private readonly writeAgentSessionOwnerRegistry: | ((file: string, serialized: string) => Promise) | undefined; + private readonly writeSubsessionBindingRegistry: + | ((file: string, serialized: string) => Promise) + | undefined; private readonly writeWorkspaceContext: ( session: HarnessSession, ) => Promise; @@ -885,6 +976,11 @@ export class SessionManager { * state only after the candidate was published or rejected. */ private sessionRegistryIdentityFence: Promise | null = null; private readonly agentSessionOwners = new Map(); + private readonly subsessionBindings = new Map< + string, + TrustedSubsessionBindingMarker + >(); + private readonly userClosedSubsessions = new Set(); /** Serializes the full authorize -> reserve -> pointer commit transition. * A file-level atomic rename alone is insufficient when two starts race the * in-memory ownership check before either write begins. */ @@ -893,6 +989,8 @@ export class SessionManager { * is also the one that owns the first-session lifecycle. */ private readonly projectCreateQueues = new Map>(); private agentSessionOwnerWriteSeq = 0; + private subsessionBindingWriteSeq = 0; + private subsessionBindingQueue: Promise = Promise.resolve(); private initialized = false; private closing = false; @@ -908,6 +1006,7 @@ export class SessionManager { options.sessionsPath ?? HARNESS_PATHS.sessions, ); this.agentSessionOwnersPath = `${this.sessionsPath}.agent-session-owners.json`; + this.subsessionBindingsPath = `${this.sessionsPath}.subsession-bindings.json`; this.spawnPty = options.spawnPty; this.loadSpawnPty = options.loadSpawnPty ?? loadDefaultSpawn; this.buildLaunchOpts = options.buildLaunchOpts ?? defaultBuildLaunchOpts; @@ -924,6 +1023,8 @@ export class SessionManager { this.writeSessionRegistry = options.writeSessionRegistry; this.writeAgentSessionOwnerRegistry = options.writeAgentSessionOwnerRegistry; + this.writeSubsessionBindingRegistry = + options.writeSubsessionBindingRegistry; this.writeWorkspaceContext = options.writeWorkspaceContext ?? (async () => {}); this.prepareWorkspaceContext = @@ -1005,6 +1106,7 @@ export class SessionManager { this.sessions.set(session.id, session); } dirty = (await this.loadAgentSessionOwners(persisted)) || dirty; + await this.loadSubsessionBindings(); if (dirty) await this.persist(); } @@ -1102,9 +1204,102 @@ export class SessionManager { async create( req: CreateSessionRequest, trusted: TrustedSessionCreateOptions = {}, + ): Promise { + return this.createWithId(this.generateId(), req, trusted); + } + + /** + * Server-only reserved-ID create. The private marker is committed before a + * session row or process can exist, closing the row-before-binding crash + * window while preserving the ordinary writable create path. + */ + async createReserved( + reservedSessionId: string, + req: CreateSessionRequest, + markerInput: TrustedSubsessionBindingMarker, + trusted: TrustedSessionCreateOptions, + ): Promise { + const marker = parseTrustedSubsessionBindingMarker( + markerInput, + reservedSessionId, + ); + if (!marker) throw new SubsessionBindingMismatchError(); + const operation = async (): Promise => { + const existingMarker = this.subsessionBindings.get(reservedSessionId); + const existingSession = this.sessions.get(reservedSessionId); + if (existingMarker) { + if (!sameSubsessionBinding(existingMarker, marker)) + throw new SubsessionBindingMismatchError(); + if (this.userClosedSubsessions.has(reservedSessionId)) + throw new SubsessionFreshRestartForbiddenError(); + if (existingSession) return existingSession; + } else { + if (existingSession) throw new SubsessionBindingMismatchError(); + this.subsessionBindings.set(reservedSessionId, marker); + try { + await this.persistSubsessionBindings(); + } catch (error) { + if (this.subsessionBindings.get(reservedSessionId) === marker) + this.subsessionBindings.delete(reservedSessionId); + throw error; + } + } + return this.createWithId(reservedSessionId, req, trusted, marker); + }; + const next = this.subsessionBindingQueue.catch(() => {}).then(operation); + this.subsessionBindingQueue = next.then( + () => undefined, + () => undefined, + ); + return next; + } + + getSubsessionBinding( + sessionId: string, + ): TrustedSubsessionBindingMarker | null { + const marker = this.subsessionBindings.get(sessionId); + return marker ? structuredClone(marker) : null; + } + + matchesSubsessionBinding( + expected: TrustedSubsessionBindingMarker, + ): boolean { + const parsed = parseTrustedSubsessionBindingMarker( + expected, + expected.sessionId, + ); + const current = parsed + ? this.subsessionBindings.get(parsed.sessionId) + : undefined; + return Boolean(parsed && current && sameSubsessionBinding(current, parsed)); + } + + wasSubsessionClosedByUser( + expected: TrustedSubsessionBindingMarker, + ): boolean { + return ( + this.matchesSubsessionBinding(expected) && + this.userClosedSubsessions.has(expected.sessionId) + ); + } + + private async createWithId( + id: string, + req: CreateSessionRequest, + trusted: TrustedSessionCreateOptions, + expectedSubsessionBinding?: TrustedSubsessionBindingMarker, ): Promise { if (this.closing) throw new SessionManagerClosingError(); - const id = this.generateId(); + const marker = this.subsessionBindings.get(id); + if ( + (marker !== undefined || expectedSubsessionBinding !== undefined) && + (!marker || + !expectedSubsessionBinding || + !sameSubsessionBinding(marker, expectedSubsessionBinding)) + ) { + throw new SubsessionBindingMismatchError(); + } + if (this.sessions.has(id)) throw new SubsessionBindingMismatchError(); const adapter = this.getAdapter(req.harness); const trustedIdentity = trusted.agentMapIdentity?.(id); const agentMapIdentity = this.resolveAgentMapIdentity @@ -1238,6 +1433,135 @@ export class SessionManager { : createResolved(); } + /** + * Narrow recovery for an exact coordinator-owned row that exited before its + * first turn and has no resumable vendor conversation. The Harness ID stays + * fixed; the private marker advances before a fresh PTY can be admitted. + */ + async restartFreshBound( + id: string, + expected: TrustedSubsessionBindingMarker, + nextInput: TrustedSubsessionBindingMarker, + trusted: TrustedSessionCreateOptions, + hasRecordedTurns: (sessionId: string) => Promise, + ): Promise { + if (this.closing) throw new SessionManagerClosingError(); + const currentExpected = parseTrustedSubsessionBindingMarker(expected, id); + const next = parseTrustedSubsessionBindingMarker(nextInput, id); + const current = this.subsessionBindings.get(id); + const session = this.sessions.get(id); + if ( + !currentExpected || + !next || + !current || + !session || + (current.projectId !== currentExpected.projectId || + current.parentSessionId !== currentExpected.parentSessionId || + current.bindingId !== currentExpected.bindingId || + current.sessionId !== currentExpected.sessionId) || + next.projectId !== currentExpected.projectId || + next.parentSessionId !== currentExpected.parentSessionId || + next.bindingId !== currentExpected.bindingId || + next.sessionId !== currentExpected.sessionId || + next.incarnation !== currentExpected.incarnation + 1 || + next.spawnEpoch <= currentExpected.spawnEpoch || + this.ptys.has(id) || + session.status !== "exited" + ) { + throw new SubsessionBindingMismatchError(); + } + if (this.userClosedSubsessions.has(id)) + throw new SubsessionFreshRestartForbiddenError(); + // A retry may observe the already-advanced marker after the sidecar write + // committed but before the fresh process existed. + if ( + !sameSubsessionBinding(current, currentExpected) && + !sameSubsessionBinding(current, next) + ) { + throw new SubsessionBindingMismatchError(); + } + const adapter = this.getAdapter(session.harness); + if ( + (session.agentSessionId !== null && + (await adapter.canResume(session.agentSessionId, session.cwd))) || + (await hasRecordedTurns(id)) + ) { + throw new SubsessionFreshRestartForbiddenError(); + } + + if (!sameSubsessionBinding(current, next)) { + this.subsessionBindings.set(id, next); + try { + await this.persistSubsessionBindings(); + } catch (error) { + this.subsessionBindings.set(id, current); + throw error; + } + } + + const trustedIdentity = trusted.agentMapIdentity?.(id); + const agentMapIdentity = this.resolveAgentMapIdentity + ? await this.resolveAgentMapIdentity(id, session.cwd, trustedIdentity) + : trustedIdentity; + if ( + !agentMapIdentity || + agentMapIdentity.projectId !== next.projectId || + agentMapIdentity.sessionId !== id + ) { + throw new ProjectSessionScopeUnavailableError(id); + } + + const lastActiveBeforeRestart = session.lastActiveAt; + session.status = "starting"; + session.exitCode = null; + session.exitTail = null; + session.agentSessionId = null; + session.agentMapIdentity = structuredClone(agentMapIdentity); + session.lastActiveAt = this.now(); + let spec: SpawnSpec; + try { + const promptAppendix = trusted.promptAppendix?.(id); + const focusedContext = trusted.focusedContext?.(id); + const sessionStartSystemMessage = + trusted.sessionStartSystemMessage?.(id); + const context = { + ...(promptAppendix ? { promptAppendix } : {}), + ...(focusedContext ? { focusedContext } : {}), + ...(sessionStartSystemMessage + ? { sessionStartSystemMessage } + : {}), + agentMapIdentity, + }; + const opts: LaunchOpts = { + harnessSessionId: id, + cwd: session.cwd, + ...(await this.buildLaunchOpts(id, session, context)), + }; + spec = adapter.launch(opts); + } catch (error) { + session.status = "exited"; + session.lastActiveAt = lastActiveBeforeRestart; + await Promise.resolve(this.onAgentMapSessionExit?.(id)).catch(() => {}); + throw error; + } + try { + await this.persist(); + this.emitStatus(session); + await this.writeWorkspaceContext(session); + await this.ensureCanvasTemplate(session.cwd); + await this.spawn(session, spec, () => + this.revalidateAgentMapIdentity(id, session.cwd, agentMapIdentity), + ); + return session; + } catch (error) { + session.lastActiveAt = lastActiveBeforeRestart; + await this.transitionExited(session, null, { + stampLastActive: false, + }).catch(() => {}); + throw error; + } + } + /** * Registers a purely historical (never-launched-by-this-harness) session so * it can subsequently be resumed via `resume()`. Called by @@ -1457,6 +1781,21 @@ export class SessionManager { * Returns false (resolved immediately) when the session has no live pty. * Returns true (resolved on actual death) when a pty was signalled. */ + async close(id: string): Promise { + if (this.subsessionBindings.has(id) && !this.userClosedSubsessions.has(id)) { + this.userClosedSubsessions.add(id); + try { + await this.persistSubsessionBindings(); + } catch (error) { + this.userClosedSubsessions.delete(id); + throw error; + } + } + const live = this.ptys.has(id); + void this.kill(id).catch(() => {}); + return live; + } + kill(id: string): Promise { const handle = this.ptys.get(id); if (!handle) { @@ -1507,6 +1846,13 @@ export class SessionManager { return handle.exited.then(() => true); } + /** Kill only the exact PTY generation a losing coordinator created. */ + killIfRuntime(id: string, runtimeEpoch: string): Promise { + if (this.ptys.get(id)?.runtimeEpoch !== runtimeEpoch) + return Promise.resolve(false); + return this.kill(id); + } + /** * Kills every currently-live pty and returns a Promise that resolves when * all of them have actually exited (real or synthesized). Bounded by the @@ -1786,6 +2132,7 @@ export class SessionManager { if (!submit) { try { + lifecycle?.onWritePhase?.("text-staged"); handle.pty.write(text); } catch (error) { // submit:false is public arbitrary draft text, not a single control @@ -1797,7 +2144,9 @@ export class SessionManager { this.observeTrustedSubmittedText(handle, text); } else if (text.length === 0) { try { + lifecycle?.onWritePhase?.("text-staged"); handle.pty.write("\r"); + lifecycle?.onWritePhase?.("enter-written"); } catch (error) { // Enter may have crossed or may have left an existing draft intact. // Either way, require a proven reset before another submission. @@ -1840,6 +2189,7 @@ export class SessionManager { let enterAttempted = false; try { try { + lifecycle?.onWritePhase?.("text-staged"); handle.pty.write(paste ? wrapPaste(text) : text); } catch (error) { // A PTY can report a text-write failure after staging a prefix. Clear @@ -1904,6 +2254,7 @@ export class SessionManager { } enterAttempted = true; handle.pty.write("\r"); + lifecycle?.onWritePhase?.("enter-written"); this.observeTrustedTerminalInput(handle, "\r"); } catch (error) { if (enterAttempted) { @@ -1927,6 +2278,41 @@ export class SessionManager { return true; } + /** + * Internal tracked variant for retry-safe coordinator delivery. It never + * turns an ambiguous write exception into zero-byte proof: callers receive + * the furthest phase observed at the exact PTY boundary. + */ + async submitInputTracked( + id: string, + text: string, + options: Readonly<{ + canWrite?: () => boolean | Promise; + lifecycle?: Omit; + background?: boolean; + }> = {}, + ): Promise { + let phase: SessionInputWritePhase = "not-written"; + try { + const accepted = await this.submitInput( + id, + text, + true, + options.canWrite, + options.background ?? true, + { + ...options.lifecycle, + onWritePhase: (next) => { + phase = next; + }, + }, + ); + return { accepted, phase }; + } catch (error) { + return { accepted: false, phase, error }; + } + } + resize(id: string, cols: number, rows: number): boolean { const handle = this.ptys.get(id); if (!handle) return false; @@ -2581,6 +2967,7 @@ export class SessionManager { await Promise.all([...this.projectCreateQueues.values()]); } await this.agentSessionIdentityQueue; + await this.subsessionBindingQueue; await this.writeQueue; } @@ -3067,6 +3454,99 @@ export class SessionManager { await rename(tmpPath, this.agentSessionOwnersPath); } + private async loadSubsessionBindings(): Promise { + let decoded: unknown; + try { + const raw = await readFile(this.subsessionBindingsPath, "utf8"); + if (Buffer.byteLength(raw, "utf8") > SUBSESSION_BINDING_MAX_BYTES) + throw new Error("subsession binding registry exceeds its size limit"); + decoded = JSON.parse(raw) as unknown; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } + if ( + !isRecord(decoded) || + Object.keys(decoded).sort().join(",") !== + "closedSessionIds,markers,version" || + decoded.version !== SUBSESSION_BINDING_FILE_VERSION || + !isRecord(decoded.markers) || + !Array.isArray(decoded.closedSessionIds) || + decoded.closedSessionIds.length > SUBSESSION_BINDING_MAX_ENTRIES || + !decoded.closedSessionIds.every( + (sessionId) => typeof sessionId === "string", + ) + ) { + throw new Error("subsession binding registry is malformed"); + } + const entries = Object.entries(decoded.markers); + if (entries.length > SUBSESSION_BINDING_MAX_ENTRIES) + throw new Error("subsession binding registry exceeds its entry limit"); + const bindingIds = new Set(); + for (const [sessionId, value] of entries) { + const marker = parseTrustedSubsessionBindingMarker(value, sessionId); + if (!marker || bindingIds.has(marker.bindingId)) + throw new Error("subsession binding registry is malformed"); + bindingIds.add(marker.bindingId); + this.subsessionBindings.set(sessionId, marker); + } + for (const sessionId of decoded.closedSessionIds) { + if (!this.subsessionBindings.has(sessionId)) + throw new Error("subsession binding registry is malformed"); + this.userClosedSubsessions.add(sessionId); + } + } + + private async persistSubsessionBindings(): Promise { + const markers = Object.fromEntries( + [...this.subsessionBindings.entries()] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([sessionId, marker]) => [sessionId, marker]), + ); + const serialized = `${JSON.stringify( + { + version: SUBSESSION_BINDING_FILE_VERSION, + markers, + closedSessionIds: [...this.userClosedSubsessions].sort(), + }, + null, + 2, + )}\n`; + if (Buffer.byteLength(serialized, "utf8") > SUBSESSION_BINDING_MAX_BYTES) + throw new Error("subsession binding registry exceeds its size limit"); + if (this.writeSubsessionBindingRegistry) { + await this.writeSubsessionBindingRegistry( + this.subsessionBindingsPath, + serialized, + ); + return; + } + const directory = dirname(this.subsessionBindingsPath); + await mkdir(directory, { recursive: true }); + const temporary = `${this.subsessionBindingsPath}.tmp-${process.pid}-${ + this.subsessionBindingWriteSeq++ + }`; + let handle: Awaited> | undefined; + try { + handle = await open(temporary, "wx", 0o600); + await handle.writeFile(serialized, "utf8"); + await handle.sync(); + await handle.close(); + handle = undefined; + await rename(temporary, this.subsessionBindingsPath); + await chmod(this.subsessionBindingsPath, 0o600); + const directoryHandle = await open(directory, "r"); + try { + await directoryHandle.sync(); + } finally { + await directoryHandle.close(); + } + } finally { + await handle?.close().catch(() => {}); + await rm(temporary, { force: true }).catch(() => {}); + } + } + private persistIdentityCandidate(candidate: HarnessSession): Promise { const current = this.list(); const index = current.findIndex((session) => session.id === candidate.id); diff --git a/packages/harness/src/index.ts b/packages/harness/src/index.ts index de812f0c..99a56d14 100644 --- a/packages/harness/src/index.ts +++ b/packages/harness/src/index.ts @@ -222,6 +222,8 @@ export { SessionNotReadyError, SessionNotResumeableError, SessionAlreadyLiveError, + SubsessionBindingMismatchError, + SubsessionFreshRestartForbiddenError, AdapterNotFoundError, ExternalHarnessError, SpawnTargetError, diff --git a/packages/harness/src/server/rest.ts b/packages/harness/src/server/rest.ts index 30f24024..97578c0e 100644 --- a/packages/harness/src/server/rest.ts +++ b/packages/harness/src/server/rest.ts @@ -907,14 +907,18 @@ export function createRestRouter(options: RestRouterOptions): Router { } }); - router.delete("/sessions/:id", (req, res) => { + router.delete("/sessions/:id", async (req, res, next) => { const existed = sessionManager.get(req.params.id) !== undefined; if (!existed) { res.status(404).json({ error: "session not found" }); return; } - void sessionManager.kill(req.params.id); - res.json({ ok: true }); + try { + await sessionManager.close(req.params.id); + res.json({ ok: true }); + } catch (error) { + next(error); + } }); router.post("/sessions/:id/input", async (req, res, next) => { From 6a7f609178dec93758312931809487a2196e4c9c Mon Sep 17 00:00:00 2001 From: Yash Date: Fri, 4 Sep 2026 18:51:57 +0000 Subject: [PATCH 04/19] feat(harness): coordinate writable project subsessions Refs: SAP-3151 --- .../core/subsession-coordinator-store.test.ts | 37 + .../src/core/subsession-coordinator-store.ts | 176 +++ .../src/core/subsession-coordinator.test.ts | 250 ++++ .../src/core/subsession-coordinator.ts | 1037 +++++++++++++++++ 4 files changed, 1500 insertions(+) create mode 100644 packages/harness/src/core/subsession-coordinator.test.ts create mode 100644 packages/harness/src/core/subsession-coordinator.ts diff --git a/packages/harness/src/core/subsession-coordinator-store.test.ts b/packages/harness/src/core/subsession-coordinator-store.test.ts index c663e6d1..c4a99b49 100644 --- a/packages/harness/src/core/subsession-coordinator-store.test.ts +++ b/packages/harness/src/core/subsession-coordinator-store.test.ts @@ -118,6 +118,43 @@ describe("SubsessionCoordinatorStore", () => { expect(aggregate.bindings).toEqual(original.bindings); }); + it("refreshes child context with an idempotent receipt and a new delivery epoch", async () => { + const root = await fixture(); + const store = new SubsessionCoordinatorStore(root); + const reserved = await store.reserveDelegations( + identity, + delegate(), + target, + ); + const binding = reserved.bindings[0]!; + const request = { + schemaVersion: 1, + requestKey: "refresh-1", + operation: { + kind: "refresh-focused-context", + target: { kind: "child", delegationKey: "research" }, + expectedContextEpoch: binding.contextEpoch, + expectedContextDigest: binding.contextDigest, + focus: null, + }, + } as const; + + const first = await store.refreshFocusedContext(identity, request); + const replay = await store.refreshFocusedContext(identity, request); + + expect(first.replayed).toBe(false); + expect(replay.replayed).toBe(true); + expect(first.binding.contextEpoch).toBe(2); + expect(first.binding.deliveries).toHaveLength(2); + expect(replay.binding).toEqual(first.binding); + await expect( + store.refreshFocusedContext(identity, { + ...request, + operation: { ...request.operation, expectedContextEpoch: 7 }, + }), + ).rejects.toMatchObject({ code: "request_key_reused" }); + }); + it("reuses a compatible binding across request keys and reserves a batch atomically", async () => { const root = await fixture(); const store = new SubsessionCoordinatorStore(root); diff --git a/packages/harness/src/core/subsession-coordinator-store.ts b/packages/harness/src/core/subsession-coordinator-store.ts index ac225e71..de61676d 100644 --- a/packages/harness/src/core/subsession-coordinator-store.ts +++ b/packages/harness/src/core/subsession-coordinator-store.ts @@ -27,6 +27,7 @@ import { type SubsessionBindingRecord, type SubsessionClaim, type SubsessionKickoffDelivery, + type SubsessionProjectionDigest, } from "../shared/subsession-delegation.js"; import { DurableFileLock } from "./durable-file-lock.js"; import { isStudioProjectId } from "./studio-project-catalog.js"; @@ -122,6 +123,12 @@ export type KickoffClaimResult = binding: SubsessionBindingRecord; }>; +export type FocusedContextRefreshResult = Readonly<{ + replayed: boolean; + requestDigest: CanonicalDelegationRequestDigest; + binding: SubsessionBindingRecord; +}>; + type ShallowMutable = { -readonly [K in keyof T]: T[K] }; type MutableClaim = ShallowMutable; type MutableDelivery = Omit< @@ -747,6 +754,175 @@ export class SubsessionCoordinatorStore { return this.transact(projectId, async (aggregate) => ({ value: aggregate })); } + readBinding( + identity: ProjectAgentSession, + selector: Readonly< + | { kind: "binding-id"; bindingId: SubsessionBindingId } + | { kind: "child"; delegationKey: string } + | { kind: "self" } + >, + ): Promise { + return this.transact(identity.projectId, async (aggregate) => { + const binding = + selector.kind === "binding-id" + ? aggregate.bindings.find( + (entry) => + entry.bindingId === selector.bindingId && + entry.parentSessionId === identity.sessionId, + ) + : selector.kind === "child" + ? aggregate.bindings.find( + (entry) => + entry.parentSessionId === identity.sessionId && + entry.delegationKey === selector.delegationKey, + ) + : aggregate.bindings.find( + (entry) => entry.sessionId === identity.sessionId, + ); + if (!binding) + throw new SubsessionCoordinatorStoreError("binding_not_found"); + if (binding.projectId !== identity.projectId) + throw new SubsessionCoordinatorStoreError("binding_scope_mismatch"); + return { value: binding }; + }); + } + + setFocusedContextState( + identity: ProjectAgentSession, + bindingId: SubsessionBindingId, + request: Readonly<{ + expectedContextEpoch: number; + expectedContextDigest: string; + state: "none" | "current" | "stale"; + projectionDigest: SubsessionProjectionDigest | null; + }>, + ): Promise { + return this.transact(identity.projectId, async (aggregate) => { + const binding = this.scopedBinding(aggregate, identity, bindingId); + if ( + binding.contextEpoch !== request.expectedContextEpoch || + binding.contextDigest !== request.expectedContextDigest + ) { + throw new SubsessionCoordinatorStoreError("lifecycle_conflict"); + } + if ( + binding.contextState === request.state && + binding.projectionDigest === request.projectionDigest + ) { + return { value: binding }; + } + const now = this.now(); + binding.contextState = request.state; + binding.projectionDigest = request.projectionDigest; + binding.updatedAt = now; + aggregate.recordVersion += 1; + aggregate.updatedAt = now; + return { value: binding, next: aggregate }; + }); + } + + refreshFocusedContext( + identity: ProjectAgentSession, + rawRequest: unknown, + ): Promise { + const request = parseProjectSubsessionRequest(rawRequest, identity.projectId); + if (request.operation.kind !== "refresh-focused-context") + throw new SubsessionCoordinatorStoreError("malformed_state"); + const operation = request.operation; + const targetSelector = operation.target; + const requestDigest = computeCanonicalDelegationRequestDigest(request); + return this.transact(identity.projectId, async (aggregate) => { + const sameRequest = (receipt: SubsessionCoordinatorRequestReceipt) => + receipt.parentSessionId === identity.sessionId && + receipt.requestKey === request.requestKey; + const previous = + aggregate.requestReceipts.find(sameRequest) ?? + aggregate.requestTombstones.find(sameRequest); + if (previous) { + if ( + previous.requestDigest !== requestDigest || + previous.operation !== "refresh-focused-context" || + previous.bindingIds.length !== 1 + ) { + throw new SubsessionCoordinatorStoreError("request_key_reused"); + } + const binding = aggregate.bindings.find( + ({ bindingId }) => bindingId === previous.bindingIds[0], + ); + if (!binding) + throw new SubsessionCoordinatorStoreError("malformed_state"); + return { value: { replayed: true, requestDigest, binding } }; + } + if ( + aggregate.requestReceipts.length >= + SUBSESSION_COORDINATOR_RECEIPT_LIMIT + ) { + throw new SubsessionCoordinatorStoreError("capacity_exceeded"); + } + const target = + targetSelector.kind === "self" + ? aggregate.bindings.find( + ({ sessionId }) => sessionId === identity.sessionId, + ) + : aggregate.bindings.find( + ({ parentSessionId, delegationKey }) => + parentSessionId === identity.sessionId && + delegationKey === targetSelector.delegationKey, + ); + if (!target) + throw new SubsessionCoordinatorStoreError("binding_not_found"); + if ( + target.contextEpoch !== operation.expectedContextEpoch || + target.contextDigest !== operation.expectedContextDigest + ) { + throw new SubsessionCoordinatorStoreError("lifecycle_conflict"); + } + const currentDelivery = target.deliveries.find( + ({ contextEpoch }) => contextEpoch === target.contextEpoch, + ); + if (currentDelivery?.state === "uncertain") + throw new SubsessionCoordinatorStoreError("claim_conflict"); + if (target.deliveries.length >= SUBSESSION_COORDINATOR_DELIVERY_LIMIT) + throw new SubsessionCoordinatorStoreError("capacity_exceeded"); + + const now = this.now(); + target.contextEpoch += 1; + target.contextDigest = computeSubsessionContextDigest( + operation.focus, + ); + target.contextState = + operation.focus === null ? "none" : "refreshing"; + target.currentFocus = operation.focus; + target.projectionDigest = null; + target.deliveries.push({ + contextEpoch: target.contextEpoch, + deliveryId: `delivery_${this.id()}`, + inputId: `input_${this.id()}`, + eventWatermark: null, + state: "pending", + attempt: 0, + claim: null, + submittedAt: null, + acknowledgedAt: null, + }); + target.updatedAt = now; + aggregate.requestReceipts.push({ + parentSessionId: identity.sessionId, + requestKey: request.requestKey, + requestDigest, + operation: "refresh-focused-context", + bindingIds: [target.bindingId], + createdAt: now, + }); + aggregate.recordVersion += 1; + aggregate.updatedAt = now; + return { + value: { replayed: false, requestDigest, binding: target }, + next: aggregate, + }; + }); + } + async reserveDelegations( identity: ProjectAgentSession, rawRequest: unknown, diff --git a/packages/harness/src/core/subsession-coordinator.test.ts b/packages/harness/src/core/subsession-coordinator.test.ts new file mode 100644 index 00000000..1b272b02 --- /dev/null +++ b/packages/harness/src/core/subsession-coordinator.test.ts @@ -0,0 +1,250 @@ +import * as fs from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { ProjectAgentSession } from "../shared/agent-map.js"; +import type { + AnalyticsEvent, + HarnessAdapter, + SpawnSpec, +} from "../shared/types.js"; +import type { BuildPlanStore } from "./build-plan-store.js"; +import type { EventReader } from "./collector/store.js"; +import { IngestCredentialRegistry } from "./ingest-credentials.js"; +import { SessionManager, type PtySpawnFn } from "./session-manager.js"; +import { + SubsessionCoordinator, + SubsessionCoordinatorError, +} from "./subsession-coordinator.js"; +import { SubsessionCoordinatorStore } from "./subsession-coordinator-store.js"; + +const projectId = "project_00000000-0000-4000-8000-000000000001"; +const parentId = "parent-session-1"; +const identity: ProjectAgentSession = { + projectId, + userId: "user-1", + sessionId: parentId, +}; + +function adapter(): HarnessAdapter { + const spec = (cwd: string): SpawnSpec => ({ + command: "fake-claude", + args: [], + env: {}, + cwd, + }); + return { + id: "claude-code", + eventSource: "hooks", + launch: ({ cwd }) => spec(cwd), + resume: (_id, { cwd }) => spec(cwd), + doctor: async () => [], + listPastSessions: async () => [], + canResume: async () => false, + }; +} + +function fakePty() { + const data: Array<(chunk: string) => void> = []; + const exits: Array<(event: { exitCode: number }) => void> = []; + const writes: string[] = []; + return { + pty: { + write: vi.fn((value: string) => writes.push(String(value))), + resize: vi.fn(), + kill: vi.fn(), + onData: (listener: (chunk: string) => void) => { + data.push(listener); + return { dispose: () => {} }; + }, + onExit: (listener: (event: { exitCode: number }) => void) => { + exits.push(listener); + return { dispose: () => {} }; + }, + } as unknown as ReturnType, + writes, + }; +} + +describe("SubsessionCoordinator", () => { + const roots: string[] = []; + + afterEach(async () => { + vi.useRealTimers(); + await Promise.all( + roots.splice(0).map((root) => + fs.rm(root, { recursive: true, force: true }), + ), + ); + }); + + async function fixture() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "subsession-service-")); + roots.push(root); + const spawned: ReturnType[] = []; + const spawnPty = vi.fn(() => { + const spawnedPty = fakePty(); + spawned.push(spawnedPty); + return spawnedPty.pty; + }); + const manager = new SessionManager({ + adapters: { "claude-code": adapter() }, + ingestUrl: "http://127.0.0.1:4100/ingest", + ingestCredentials: new IngestCredentialRegistry(), + sessionsPath: path.join(root, "sessions.json"), + spawnPty, + resolveAgentMapIdentity: async (_sessionId, _cwd, persisted) => persisted, + }); + await manager.init(); + await manager.create( + { cwd: root, harness: "claude-code" }, + { + agentMapIdentity: (sessionId) => ({ ...identity, sessionId }), + }, + ); + const parent = manager.list()[0]!; + expect(parent.id).not.toBe(parentId); + // The caller capability identity is server-derived from its real Harness + // session ID, so use that exact value for the fixture. + const caller = { ...identity, sessionId: parent.id }; + manager.setReady(parent.id, manager.getRuntimeEpoch(parent.id)!); + const unsubscribe = manager.onStatusChange((session, context) => { + if ( + session.id !== parent.id && + session.status === "running" && + !session.ready && + context.runtimeEpoch + ) { + manager.setReady(session.id, context.runtimeEpoch); + } + }); + const events: AnalyticsEvent[] = []; + const eventReader: EventReader = { + async *read(filter) { + const ids = filter?.harnessSessionId; + const accepted = new Set( + typeof ids === "string" ? [ids] : ids ?? [], + ); + for (const event of events) { + if (accepted.size > 0 && !accepted.has(event.harnessSessionId)) + continue; + if (filter?.types && !filter.types.includes(event.type)) continue; + yield event; + } + }, + index: async () => ({ bySession: new Map(), byAgentSession: new Map() }), + }; + const store = new SubsessionCoordinatorStore( + path.join(root, "agent-map"), + ); + const planningStore = { + read: vi.fn(async () => { + throw new Error("no focused context expected"); + }), + } as unknown as BuildPlanStore; + const coordinator = new SubsessionCoordinator({ + store, + sessionManager: manager, + planningStore, + eventReader, + readinessTimeoutMs: 500, + }); + return { + root, + manager, + caller, + store, + coordinator, + events, + spawnPty, + spawned, + unsubscribe, + }; + } + + const request = { + schemaVersion: 1, + requestKey: "request-1", + operation: { + kind: "delegate", + delegations: [ + { + delegationKey: "research", + outcome: "Implement the research slice", + kickoffContext: "Run the focused tests.", + }, + ], + }, + } as const; + + it("creates one ordinary writable child and reuses it on retry", async () => { + const { coordinator, caller, manager, spawnPty, unsubscribe } = + await fixture(); + const first = await coordinator.execute(caller, request); + const replay = await coordinator.execute(caller, request); + unsubscribe(); + + expect(first.results[0]).toMatchObject({ + outcome: "created", + sessionState: "ready", + contextState: "none", + kickoffState: "submitted-unacknowledged", + }); + expect(replay.results[0]).toMatchObject({ + outcome: "reused", + sessionId: first.results[0]!.sessionId, + }); + expect(manager.list()).toHaveLength(2); + expect(spawnPty).toHaveBeenCalledTimes(2); + const child = manager.get(first.results[0]!.sessionId!); + expect(child?.agentMapIdentity).toEqual({ + projectId, + userId: caller.userId, + sessionId: first.results[0]!.sessionId, + }); + }); + + it("acknowledges only the exact persisted kickoff marker", async () => { + const { coordinator, caller, manager, store, spawned, unsubscribe } = + await fixture(); + const result = await coordinator.execute(caller, request); + const sessionId = result.results[0]!.sessionId!; + const prompt = spawned[1]!.writes + .find((value) => value.includes("sapiom-project-delegation"))!; + const event: AnalyticsEvent = { + eventId: "event-1", + seq: 1, + ts: new Date().toISOString(), + userId: caller.userId, + tenantId: null, + machineId: "machine-1", + harnessSessionId: sessionId, + agentSessionId: null, + harness: "claude-code", + type: "prompt.submitted", + payload: { prompt }, + }; + await coordinator.onEventPersisted( + event, + manager.getRuntimeEpoch(sessionId)!, + ); + const aggregate = await store.read(projectId); + unsubscribe(); + + expect(aggregate.bindings[0]!.deliveries[0]!.state).toBe("acknowledged"); + }); + + it("fails closed when the caller identity is not its trusted session scope", async () => { + const { coordinator, caller, manager, unsubscribe } = await fixture(); + await expect( + coordinator.execute({ ...caller, projectId: "project_00000000-0000-4000-8000-000000000099" }, request), + ).rejects.toEqual( + expect.objectContaining>({ + detail: expect.objectContaining({ code: "capability_scope_mismatch" }), + }), + ); + unsubscribe(); + expect(manager.list()).toHaveLength(1); + }); +}); diff --git a/packages/harness/src/core/subsession-coordinator.ts b/packages/harness/src/core/subsession-coordinator.ts new file mode 100644 index 00000000..4d0d8dc6 --- /dev/null +++ b/packages/harness/src/core/subsession-coordinator.ts @@ -0,0 +1,1037 @@ +import { randomUUID } from "node:crypto"; + +import type { + ProjectAgentSession, + StudioProjectId, +} from "../shared/agent-map.js"; +import { canonicalDigest } from "../shared/agent-map-canonical.js"; +import { + agentMapVersionRefsEqual, + projectBuildPlanVersionRefsEqual, + type AgentBriefVersion, +} from "../shared/build-plan.js"; +import { + parseProjectSubsessionRequest, +} from "../shared/subsession-delegation-codec.js"; +import type { + DelegationError, + DelegationFocusRef, + DelegationItemOutcome, + DelegationItemResult, + ProjectSubsessionRequest, + ProjectSubsessionResult, + SubsessionBindingRecord, + SubsessionProjectionDigest, +} from "../shared/subsession-delegation.js"; +import type { AnalyticsEvent, HarnessSession } from "../shared/types.js"; +import type { BuildPlanStore } from "./build-plan-store.js"; +import type { EventReader } from "./collector/store.js"; +import { + serializeFocusedSessionContext, + type FocusedSessionContextProjection, +} from "./focused-session-context.js"; +import { + SessionNotReadyError, + SubsessionBindingMismatchError, +} from "./errors.js"; +import type { + SessionManager, + TrustedSubsessionBindingMarker, +} from "./session-manager.js"; +import { + SubsessionCoordinatorStore, + SubsessionCoordinatorStoreError, +} from "./subsession-coordinator-store.js"; + +const DEFAULT_READINESS_TIMEOUT_MS = 30_000; +const KICKOFF_MARKER = //u; + +export interface SubsessionCoordinatorEvent { + name: + | "subsession.requested" + | "subsession.created" + | "subsession.reused" + | "subsession.ready" + | "subsession.failed" + | "subsession.kickoff_submitted" + | "subsession.kickoff_acknowledged" + | "subsession.kickoff_uncertain" + | "subsession.context_stale" + | "subsession.manual_session_protected"; + projectId: StudioProjectId; + sessionId?: string; + code?: DelegationError["code"]; +} + +export class SubsessionCoordinatorError extends Error { + constructor(readonly detail: DelegationError) { + super(detail.code); + this.name = "SubsessionCoordinatorError"; + } +} + +export interface SubsessionCoordinatorOptions { + store: SubsessionCoordinatorStore; + sessionManager: SessionManager; + planningStore: BuildPlanStore; + eventReader: EventReader; + ownerId?: string; + readinessTimeoutMs?: number; + onEvent?: (event: SubsessionCoordinatorEvent) => void | Promise; +} + +type ResolvedFocus = Readonly<{ + state: "none" | "current" | "stale"; + projection: FocusedSessionContextProjection | null; + projectionDigest: SubsessionProjectionDigest | null; +}>; + +type DelegateRequest = Omit & + Readonly<{ + operation: Extract< + ProjectSubsessionRequest["operation"], + { kind: "delegate" } + >; + }>; +type RefreshRequest = Omit & + Readonly<{ + operation: Extract< + ProjectSubsessionRequest["operation"], + { kind: "refresh-focused-context" } + >; + }>; + +const error = ( + code: DelegationError["code"], + retryable: boolean, + recovery: DelegationError["recovery"], +): DelegationError => ({ code, retryable, recovery }); + +const currentDelivery = (binding: SubsessionBindingRecord) => + binding.deliveries.find( + ({ contextEpoch }) => contextEpoch === binding.contextEpoch, + ) ?? binding.deliveries.at(-1)!; + +const bindingIdentity = ( + caller: ProjectAgentSession, + binding: SubsessionBindingRecord, +): ProjectAgentSession => ({ + projectId: binding.projectId, + userId: caller.userId, + sessionId: binding.parentSessionId, +}); + +const markerFor = ( + binding: SubsessionBindingRecord, + incarnation: number, +): TrustedSubsessionBindingMarker => ({ + projectId: binding.projectId, + parentSessionId: binding.parentSessionId, + bindingId: binding.bindingId, + sessionId: binding.sessionId, + incarnation, + spawnEpoch: binding.spawnEpoch, +}); + +const refsMatchBrief = ( + brief: AgentBriefVersion, + focus: Extract, +) => + brief.projectId === focus.brief.projectId && + brief.briefId === focus.brief.briefId && + brief.versionId === focus.brief.versionId && + brief.semanticDigest === focus.brief.semanticDigest; + +export class SubsessionCoordinator { + private readonly ownerId: string; + private readonly readinessTimeoutMs: number; + + constructor(private readonly options: SubsessionCoordinatorOptions) { + this.ownerId = options.ownerId ?? `coordinator_${randomUUID()}`; + this.readinessTimeoutMs = + options.readinessTimeoutMs ?? DEFAULT_READINESS_TIMEOUT_MS; + } + + private emit(event: SubsessionCoordinatorEvent): void { + try { + void Promise.resolve(this.options.onEvent?.(event)).catch(() => {}); + } catch { + // Content-free telemetry cannot change delegation behavior. + } + } + + async execute( + identity: ProjectAgentSession, + rawRequest: unknown, + ): Promise { + let request: ProjectSubsessionRequest; + try { + request = parseProjectSubsessionRequest(rawRequest, identity.projectId); + } catch { + throw new SubsessionCoordinatorError( + error("invalid_request", false, "correct"), + ); + } + this.assertCaller(identity); + this.emit({ name: "subsession.requested", projectId: identity.projectId }); + if (request.operation.kind === "refresh-focused-context") + return this.refresh(identity, request as RefreshRequest); + return this.delegate(identity, request as DelegateRequest); + } + + private assertCaller(identity: ProjectAgentSession): HarnessSession { + const caller = this.options.sessionManager.get(identity.sessionId); + if ( + !caller?.agentMapIdentity || + caller.agentMapIdentity.projectId !== identity.projectId || + caller.agentMapIdentity.userId !== identity.userId || + caller.agentMapIdentity.sessionId !== identity.sessionId + ) { + throw new SubsessionCoordinatorError( + error("capability_scope_mismatch", false, "none"), + ); + } + return caller; + } + + private async delegate( + identity: ProjectAgentSession, + request: DelegateRequest, + ): Promise { + const caller = this.assertCaller(identity); + let reserved; + try { + reserved = await this.options.store.reserveDelegations(identity, request, { + harness: caller.harness, + projectRoot: caller.cwd, + }); + } catch (cause) { + throw this.wholeCallError(cause); + } + const results: DelegationItemResult[] = []; + for (const binding of reserved.bindings) { + results.push(await this.reconcileBinding(identity, binding, reserved.replayed)); + } + return { + schemaVersion: 1, + requestKey: request.requestKey, + requestDigest: reserved.requestDigest, + replayed: reserved.replayed, + results: results.sort((left, right) => + left.delegationKey.localeCompare(right.delegationKey), + ), + }; + } + + private async refresh( + identity: ProjectAgentSession, + request: RefreshRequest, + ): Promise { + let refreshed; + try { + refreshed = await this.options.store.refreshFocusedContext(identity, request); + } catch (cause) { + throw this.wholeCallError(cause, true); + } + const result = await this.reconcileBinding( + identity, + refreshed.binding, + refreshed.replayed, + ); + return { + schemaVersion: 1, + requestKey: request.requestKey, + requestDigest: refreshed.requestDigest, + replayed: refreshed.replayed, + results: [result], + }; + } + + private async reconcileBinding( + caller: ProjectAgentSession, + initial: SubsessionBindingRecord, + replayed: boolean, + ): Promise { + const identity = bindingIdentity(caller, initial); + let binding = initial; + try { + binding = await this.reconcileHistoricalDelivery(identity, binding); + const focused = await this.resolveFocus(binding); + binding = await this.options.store.setFocusedContextState( + identity, + binding.bindingId, + { + expectedContextEpoch: binding.contextEpoch, + expectedContextDigest: binding.contextDigest, + state: focused.state, + projectionDigest: focused.projectionDigest, + }, + ); + if (focused.state === "stale") { + this.emit({ + name: "subsession.context_stale", + projectId: binding.projectId, + sessionId: binding.sessionId, + code: "context_stale", + }); + return this.failedResult( + binding, + error("context_stale", false, "refresh_context"), + ); + } + const ensured = await this.ensureSession(caller, binding, focused.projection); + binding = ensured.binding; + if (binding.sessionState !== "ready") + return this.result(binding, "already-running"); + binding = await this.deliver(identity, binding, focused.projection); + const outcome: DelegationItemOutcome = ensured.created + ? "created" + : replayed || ensured.reused + ? "reused" + : "already-running"; + this.emit({ + name: ensured.created ? "subsession.created" : "subsession.reused", + projectId: binding.projectId, + sessionId: binding.sessionId, + }); + return this.result(binding, outcome); + } catch (cause) { + const detail = this.itemError(cause); + this.emit({ + name: + detail.code === "binding_session_mismatch" + ? "subsession.manual_session_protected" + : "subsession.failed", + projectId: binding.projectId, + sessionId: binding.sessionId, + code: detail.code, + }); + return this.failedResult(binding, detail); + } + } + + private async resolveFocus( + binding: SubsessionBindingRecord, + ): Promise { + const focus = binding.currentFocus; + if (!focus) + return { state: "none", projection: null, projectionDigest: null }; + const aggregate = await this.options.planningStore.read(binding.projectId); + const mapRef = focus.kind === "brief" ? null : focus.map; + const map = aggregate.mapVersions.find( + ({ versionId }) => versionId === (mapRef?.versionId ?? ""), + ); + if (focus.kind === "brief") { + const brief = Object.values(aggregate.briefVersionsById) + .flat() + .find((candidate) => refsMatchBrief(candidate, focus)); + if (!brief) throw error("context_not_found", false, "reread"); + const exactMap = aggregate.mapVersions.find( + ({ versionId }) => versionId === brief.map.versionId, + ); + const exactPlan = aggregate.buildPlanVersions.find( + ({ versionId }) => versionId === brief.plan.versionId, + ); + if (!exactMap || !exactPlan) + throw error("context_not_found", false, "reread"); + const pointer = Object.values(aggregate.current.briefsByScope).find( + ({ briefId }) => briefId === brief.briefId, + ); + const stale = + !aggregate.current.map || + !aggregate.current.buildPlan || + !agentMapVersionRefsEqual(aggregate.current.map, brief.map) || + !projectBuildPlanVersionRefsEqual( + aggregate.current.buildPlan, + brief.plan, + ) || + !pointer || + pointer.status !== "active" || + pointer.version.versionId !== brief.versionId || + pointer.version.semanticDigest !== brief.semanticDigest; + if (stale) + return { state: "stale", projection: null, projectionDigest: null }; + const projection = serializeFocusedSessionContext({ + map: exactMap, + plan: exactPlan, + brief, + }); + if (!projection.ok) + throw error("context_not_found", false, "reread"); + return { + state: "current", + projection: projection.projection, + projectionDigest: canonicalDigest( + "sapiom.subsession.focused-projection.v1", + projection.projection, + ) as SubsessionProjectionDigest, + }; + } + if ( + !map || + !agentMapVersionRefsEqual( + { + projectId: map.projectId, + versionId: map.versionId, + contentDigest: map.contentDigest, + }, + focus.map, + ) + ) { + throw error("context_not_found", false, "reread"); + } + const currentMap = aggregate.current.map; + let stale = !currentMap || !agentMapVersionRefsEqual(currentMap, focus.map); + if (focus.kind === "assignment") { + const plan = aggregate.buildPlanVersions.find( + ({ versionId }) => versionId === focus.plan.versionId, + ); + if ( + !plan || + !projectBuildPlanVersionRefsEqual( + { + projectId: plan.projectId, + planId: plan.planId, + versionId: plan.versionId, + semanticDigest: plan.semanticDigest, + }, + focus.plan, + ) || + !plan.content.assignments.some(({ id }) => id === focus.assignmentId) + ) { + throw error("context_not_found", false, "reread"); + } + stale = + stale || + !aggregate.current.buildPlan || + !projectBuildPlanVersionRefsEqual( + aggregate.current.buildPlan, + focus.plan, + ); + } else { + if (!map.graph.nodes.some(({ id }) => id === focus.nodeId)) + throw error("context_not_found", false, "reread"); + if (focus.plan) { + const plan = aggregate.buildPlanVersions.find( + ({ versionId }) => versionId === focus.plan!.versionId, + ); + if ( + !plan || + !projectBuildPlanVersionRefsEqual( + { + projectId: plan.projectId, + planId: plan.planId, + versionId: plan.versionId, + semanticDigest: plan.semanticDigest, + }, + focus.plan, + ) + ) { + throw error("context_not_found", false, "reread"); + } + stale = + stale || + !aggregate.current.buildPlan || + !projectBuildPlanVersionRefsEqual( + aggregate.current.buildPlan, + focus.plan, + ); + } + } + return { + state: stale ? "stale" : "current", + projection: null, + projectionDigest: null, + }; + } + + private async ensureSession( + caller: ProjectAgentSession, + initial: SubsessionBindingRecord, + projection: FocusedSessionContextProjection | null, + ): Promise<{ + binding: SubsessionBindingRecord; + created: boolean; + reused: boolean; + }> { + const identity = bindingIdentity(caller, initial); + let binding = await this.options.store.readBinding(identity, { + kind: "binding-id", + bindingId: initial.bindingId, + }); + const existing = this.options.sessionManager.get(binding.sessionId); + const privateMarker = + this.options.sessionManager.getSubsessionBinding(binding.sessionId); + if (existing || privateMarker) { + const incarnation = binding.runtime?.incarnation ?? privateMarker?.incarnation ?? 1; + const expected = markerFor(binding, incarnation); + if ( + !privateMarker || + !this.options.sessionManager.matchesSubsessionBinding(expected) + ) { + throw new SubsessionBindingMismatchError(); + } + if (!existing) { + if ( + !binding.spawnClaim || + binding.spawnClaim.expiresAt > new Date().toISOString() + ) { + return { binding, created: false, reused: true }; + } + const session = await this.options.sessionManager.createReserved( + binding.sessionId, + { cwd: binding.projectRoot, harness: binding.harness }, + expected, + { + agentMapIdentity: (sessionId) => ({ + projectId: binding.projectId, + userId: caller.userId, + sessionId, + }), + initialTitle: this.title(binding.outcome), + ...(projection ? { focusedContext: () => projection } : {}), + }, + ); + const runtimeToken = this.options.sessionManager.getRuntimeEpoch( + session.id, + ); + if (!runtimeToken) + throw error("session_create_failed", true, "retry"); + binding = await this.options.store.attachSpawnedRuntime( + identity, + binding.bindingId, + { + claimId: binding.spawnClaim.claimId, + spawnEpoch: binding.spawnEpoch, + runtimeToken, + incarnation: expected.incarnation, + }, + ); + binding = await this.advanceToReady(identity, binding, runtimeToken); + return { binding, created: false, reused: true }; + } + if (this.options.sessionManager.wasSubsessionClosedByUser(expected)) + throw error("session_closed", false, "inspect_session"); + if (this.options.sessionManager.isLive(binding.sessionId)) { + const runtimeToken = + this.options.sessionManager.getRuntimeEpoch(binding.sessionId)!; + if (binding.runtime?.runtimeToken === runtimeToken) { + binding = await this.advanceToReady(identity, binding, runtimeToken); + return { binding, created: false, reused: true }; + } + if (binding.spawnClaim) { + if (binding.spawnClaim.expiresAt > new Date().toISOString()) + return { binding, created: false, reused: true }; + binding = await this.options.store.attachSpawnedRuntime( + identity, + binding.bindingId, + { + claimId: binding.spawnClaim.claimId, + spawnEpoch: binding.spawnEpoch, + runtimeToken, + incarnation: privateMarker.incarnation, + }, + ); + binding = await this.advanceToReady(identity, binding, runtimeToken); + return { binding, created: false, reused: true }; + } + throw error("binding_session_mismatch", false, "inspect_session"); + } + throw error("session_unreachable", true, "inspect_session"); + } + + let claim = await this.options.store.claimSpawn( + identity, + binding.bindingId, + { + ownerId: this.ownerId, + expectedLifecycleEpoch: binding.lifecycleEpoch, + expectedSpawnEpoch: binding.spawnEpoch, + }, + ); + if (!claim.claimed) { + if (claim.reason !== "expired-requires-inspection") + return { binding: claim.binding, created: false, reused: true }; + const expired = claim.binding.spawnClaim; + if (!expired) + throw error("session_unreachable", true, "inspect_session"); + claim = await this.options.store.takeoverExpiredSpawnClaim( + identity, + claim.binding.bindingId, + { + ownerId: this.ownerId, + expiredClaimId: expired.claimId, + expectedLifecycleEpoch: claim.binding.lifecycleEpoch, + expectedSpawnEpoch: claim.binding.spawnEpoch, + }, + ); + if (!claim.claimed) + return { binding: claim.binding, created: false, reused: true }; + } + binding = claim.binding; + const spawnClaim = binding.spawnClaim!; + const marker = markerFor(binding, 1); + let session: HarnessSession; + try { + session = await this.options.sessionManager.createReserved( + binding.sessionId, + { cwd: binding.projectRoot, harness: binding.harness }, + marker, + { + agentMapIdentity: (sessionId) => ({ + projectId: binding.projectId, + userId: caller.userId, + sessionId, + }), + initialTitle: this.title(binding.outcome), + ...(projection + ? { focusedContext: () => projection } + : {}), + }, + ); + } catch (cause) { + // A missing session row is positive proof that createWithId never + // reached its first durable row/process side effect. Every later failure + // keeps the claim fenced for exact inspection instead of guessing. + if (!this.options.sessionManager.get(binding.sessionId)) { + await this.options.store + .releaseUnspawnedClaim(identity, binding.bindingId, { + claimId: spawnClaim.claimId, + spawnEpoch: binding.spawnEpoch, + proof: "no-process-created", + }) + .catch(() => {}); + } + throw cause; + } + const runtimeToken = this.options.sessionManager.getRuntimeEpoch(session.id); + if (!runtimeToken) + throw error("session_create_failed", true, "retry"); + binding = await this.options.store.attachSpawnedRuntime( + identity, + binding.bindingId, + { + claimId: spawnClaim.claimId, + spawnEpoch: binding.spawnEpoch, + runtimeToken, + incarnation: marker.incarnation, + }, + ); + binding = await this.advanceToReady(identity, binding, runtimeToken); + return { binding, created: true, reused: false }; + } + + private async advanceToReady( + identity: ProjectAgentSession, + initial: SubsessionBindingRecord, + runtimeToken: string, + ): Promise { + let binding = initial; + if (binding.sessionState === "starting") { + binding = await this.options.store.transitionSession( + identity, + binding.bindingId, + { + expectedLifecycleEpoch: binding.lifecycleEpoch, + expectedSpawnEpoch: binding.spawnEpoch, + expectedRuntimeToken: runtimeToken, + state: this.options.sessionManager.get(binding.sessionId)?.ready + ? "ready" + : "awaiting-ready", + }, + ); + } + if (binding.sessionState === "awaiting-ready") { + const ready = await this.waitForReady(binding.sessionId, runtimeToken); + if (!ready) throw new SessionNotReadyError(binding.sessionId); + binding = await this.options.store.transitionSession( + identity, + binding.bindingId, + { + expectedLifecycleEpoch: binding.lifecycleEpoch, + expectedSpawnEpoch: binding.spawnEpoch, + expectedRuntimeToken: runtimeToken, + state: "ready", + }, + ); + } + if (binding.sessionState === "ready") + this.emit({ + name: "subsession.ready", + projectId: binding.projectId, + sessionId: binding.sessionId, + }); + return binding; + } + + private waitForReady(sessionId: string, runtimeToken: string): Promise { + if ( + this.options.sessionManager.get(sessionId)?.ready && + this.options.sessionManager.isCurrentRuntimeEpoch(sessionId, runtimeToken) + ) { + return Promise.resolve(true); + } + return new Promise((resolve) => { + let settled = false; + const finish = (value: boolean) => { + if (settled) return; + settled = true; + clearTimeout(timer); + unsubscribe(); + resolve(value); + }; + const unsubscribe = this.options.sessionManager.onStatusChange( + (session, context) => { + if (session.id !== sessionId) return; + if (context.runtimeEpoch !== runtimeToken || session.status === "exited") + finish(false); + else if (session.ready) finish(true); + }, + ); + const timer = setTimeout( + () => finish(false), + this.readinessTimeoutMs, + ); + }); + } + + private async deliver( + identity: ProjectAgentSession, + initial: SubsessionBindingRecord, + projection: FocusedSessionContextProjection | null, + ): Promise { + let binding = await this.reconcileHistoricalDelivery(identity, initial); + let delivery = currentDelivery(binding); + if (delivery.state !== "pending") return binding; + const watermark = await this.latestEventId(binding.sessionId); + const claimed = await this.options.store.claimKickoff( + identity, + binding.bindingId, + { + ownerId: this.ownerId, + expectedLifecycleEpoch: binding.lifecycleEpoch, + expectedSpawnEpoch: binding.spawnEpoch, + expectedContextEpoch: binding.contextEpoch, + eventWatermark: watermark, + }, + ); + if (!claimed.claimed) { + if (claimed.reason === "expired-requires-reconciliation") + return this.options.store.markKickoffUncertain( + identity, + binding.bindingId, + { + contextEpoch: binding.contextEpoch, + deliveryId: delivery.deliveryId, + inputId: delivery.inputId, + }, + ); + return claimed.binding; + } + binding = claimed.binding; + delivery = currentDelivery(binding); + const runtimeToken = binding.runtime?.runtimeToken; + if (!runtimeToken) + throw error("session_unreachable", true, "inspect_session"); + const prompt = this.kickoffPrompt(binding, delivery, projection); + const tracked = await this.options.sessionManager.submitInputTracked( + binding.sessionId, + prompt, + { + background: true, + canWrite: async () => { + const current = await this.options.store.readBinding(identity, { + kind: "binding-id", + bindingId: binding.bindingId, + }); + return ( + current.lifecycleEpoch === binding.lifecycleEpoch && + current.spawnEpoch === binding.spawnEpoch && + current.contextEpoch === binding.contextEpoch && + current.contextDigest === binding.contextDigest && + current.runtime?.runtimeToken === runtimeToken && + this.options.sessionManager.isCurrentRuntimeEpoch( + binding.sessionId, + runtimeToken, + ) + ); + }, + }, + ); + const after = await this.options.store.readBinding(identity, { + kind: "binding-id", + bindingId: binding.bindingId, + }); + if (currentDelivery(after).state === "acknowledged") return after; + binding = await this.options.store.recordKickoffWrite( + identity, + binding.bindingId, + { + contextEpoch: binding.contextEpoch, + deliveryId: delivery.deliveryId, + inputId: delivery.inputId, + claimId: delivery.claim!.claimId, + phase: tracked.phase, + }, + ); + const state = currentDelivery(binding).state; + this.emit({ + name: + state === "uncertain" + ? "subsession.kickoff_uncertain" + : "subsession.kickoff_submitted", + projectId: binding.projectId, + sessionId: binding.sessionId, + }); + return binding; + } + + private kickoffPrompt( + binding: SubsessionBindingRecord, + delivery: ReturnType, + projection: FocusedSessionContextProjection | null, + ): string { + const marker = ``; + if (binding.contextEpoch > 1) { + return [ + "Focused project context refresh for your existing delegated task.", + ...(projection + ? [projection] + : ["The optional focused overlay is now cleared or reference-only. Read current shared project state through the common tools when needed."]), + "This context update does not change your tools, writable policy, or authority.", + marker, + ].join("\n\n"); + } + return [ + "You are an ordinary writable project session delegated by another project session.", + `Outcome: ${binding.outcome}`, + ...(binding.kickoffContext + ? [`Bounded kickoff context:\n${binding.kickoffContext}`] + : []), + "Plan or implement directly as appropriate. Keep shared project state current and delegate further when useful.", + marker, + ].join("\n\n"); + } + + async onEventPersisted( + event: AnalyticsEvent, + runtimeToken: string, + ): Promise { + if (event.type !== "prompt.submitted") return; + const prompt = + typeof event.payload.prompt === "string" ? event.payload.prompt : ""; + const match = KICKOFF_MARKER.exec(prompt); + if (!match) return; + const session = this.options.sessionManager.get(event.harnessSessionId); + if (!session?.agentMapIdentity) return; + const aggregate = await this.options.store.read( + session.agentMapIdentity.projectId, + ); + const binding = aggregate.bindings.find( + ({ sessionId, bindingId }) => + sessionId === event.harnessSessionId && bindingId === match[1], + ); + if ( + !binding || + binding.runtime?.runtimeToken !== runtimeToken || + !this.options.sessionManager.isCurrentRuntimeEpoch( + binding.sessionId, + runtimeToken, + ) + ) { + return; + } + const delivery = binding.deliveries.find( + ({ deliveryId, inputId, contextEpoch }) => + deliveryId === match[2] && + inputId === match[3] && + contextEpoch === Number(match[4]), + ); + if ( + !delivery || + binding.spawnEpoch !== Number(match[5]) || + !delivery.eventWatermark + ) { + return; + } + const identity: ProjectAgentSession = { + projectId: binding.projectId, + userId: session.agentMapIdentity.userId, + sessionId: binding.parentSessionId, + }; + await this.options.store.acknowledgeKickoff( + identity, + binding.bindingId, + { + contextEpoch: binding.contextEpoch, + deliveryId: delivery.deliveryId, + inputId: delivery.inputId, + eventWatermark: delivery.eventWatermark, + }, + ); + this.emit({ + name: "subsession.kickoff_acknowledged", + projectId: binding.projectId, + sessionId: binding.sessionId, + }); + } + + private async reconcileHistoricalDelivery( + identity: ProjectAgentSession, + initial: SubsessionBindingRecord, + ): Promise { + const delivery = currentDelivery(initial); + if ( + !["claimed", "submitted-unacknowledged", "uncertain"].includes( + delivery.state, + ) + ) { + return initial; + } + for await (const event of this.options.eventReader.read({ + harnessSessionId: initial.sessionId, + types: ["prompt.submitted"], + })) { + const prompt = + typeof event.payload.prompt === "string" ? event.payload.prompt : ""; + const match = KICKOFF_MARKER.exec(prompt); + if ( + match?.[1] === initial.bindingId && + match[2] === delivery.deliveryId && + match[3] === delivery.inputId && + Number(match[4]) === initial.contextEpoch && + Number(match[5]) === initial.spawnEpoch && + delivery.eventWatermark + ) { + return this.options.store.acknowledgeKickoff( + identity, + initial.bindingId, + { + contextEpoch: initial.contextEpoch, + deliveryId: delivery.deliveryId, + inputId: delivery.inputId, + eventWatermark: delivery.eventWatermark, + }, + ); + } + } + if ( + delivery.state === "submitted-unacknowledged" || + delivery.state === "uncertain" || + (delivery.state === "claimed" && + delivery.claim !== null && + delivery.claim.expiresAt <= new Date().toISOString()) + ) { + return this.options.store.markKickoffUncertain( + identity, + initial.bindingId, + { + contextEpoch: initial.contextEpoch, + deliveryId: delivery.deliveryId, + inputId: delivery.inputId, + }, + ); + } + return initial; + } + + private async latestEventId(sessionId: string): Promise { + let latest = "event_none"; + for await (const event of this.options.eventReader.read({ + harnessSessionId: sessionId, + })) { + latest = event.eventId; + } + return latest; + } + + private title(outcome: string): string { + const first = outcome.split("\n", 1)[0]?.trim() || "Delegated task"; + return [...first].slice(0, 80).join(""); + } + + private result( + binding: SubsessionBindingRecord, + outcome: DelegationItemOutcome, + ): DelegationItemResult { + return { + delegationKey: binding.delegationKey, + bindingId: binding.bindingId, + sessionId: binding.sessionId, + outcome, + sessionState: binding.sessionState, + contextState: binding.contextState, + kickoffState: currentDelivery(binding).state, + }; + } + + private failedResult( + binding: SubsessionBindingRecord, + detail: DelegationError, + ): DelegationItemResult { + return { ...this.result(binding, "failed"), error: detail }; + } + + private wholeCallError(cause: unknown, refresh = false): SubsessionCoordinatorError { + if (cause instanceof SubsessionCoordinatorError) return cause; + if (cause instanceof SubsessionCoordinatorStoreError) { + if (cause.code === "request_key_reused") + return new SubsessionCoordinatorError( + error("request_key_reused", false, "new_request_key"), + ); + if (cause.code === "delegation_key_reused") + return new SubsessionCoordinatorError( + error("delegation_key_reused", false, "new_delegation_key"), + ); + if (cause.code === "capacity_exceeded") + return new SubsessionCoordinatorError( + error("capacity_exceeded", false, "reduce_request"), + ); + if (cause.code === "storage_unavailable") + return new SubsessionCoordinatorError( + error("storage_unavailable", true, "retry"), + ); + if (refresh && cause.code === "binding_not_found") + return new SubsessionCoordinatorError( + error("context_not_found", false, "reread"), + ); + if (refresh && cause.code === "lifecycle_conflict") + return new SubsessionCoordinatorError( + error("context_refresh_conflict", false, "reread"), + ); + if (refresh && cause.code === "claim_conflict") + return new SubsessionCoordinatorError( + error("kickoff_failed", false, "inspect_session"), + ); + } + return new SubsessionCoordinatorError( + error("internal_error", true, "retry"), + ); + } + + private itemError(cause: unknown): DelegationError { + if ( + typeof cause === "object" && + cause !== null && + "code" in cause && + typeof cause.code === "string" && + "retryable" in cause && + "recovery" in cause + ) { + return cause as DelegationError; + } + if (cause instanceof SubsessionBindingMismatchError) + return error("binding_session_mismatch", false, "inspect_session"); + if (cause instanceof SessionNotReadyError) + return error("readiness_timeout", true, "retry"); + if (cause instanceof SubsessionCoordinatorStoreError) { + if (cause.code === "storage_unavailable") + return error("storage_unavailable", true, "retry"); + if (cause.code === "session_closed") + return error("session_closed", false, "inspect_session"); + if (["lifecycle_conflict", "claim_conflict"].includes(cause.code)) + return error("session_unreachable", true, "inspect_session"); + } + return error("session_create_failed", true, "retry"); + } +} From a99a83f73c086ac8c80e1877ab1dbcd77f515897 Mon Sep 17 00:00:00 2001 From: Yash Date: Fri, 4 Sep 2026 18:57:10 +0000 Subject: [PATCH 05/19] feat(harness): expose subsession delegation to every agent Refs: SAP-3151 --- .../harness/src/server/agent-map-mcp-tools.ts | 75 ++++++++++++++++++- .../src/server/agent-map-mcp-wiring.test.ts | 74 ++++++++++++++++++ .../harness/src/server/agent-map-mcp.test.ts | 34 ++++++++- packages/harness/src/server/agent-map-mcp.ts | 4 +- packages/harness/src/server/index.ts | 57 +++++++++++++- packages/harness/src/shared/types.ts | 15 ++++ 6 files changed, 254 insertions(+), 5 deletions(-) diff --git a/packages/harness/src/server/agent-map-mcp-tools.ts b/packages/harness/src/server/agent-map-mcp-tools.ts index 67fe520c..c85e2602 100644 --- a/packages/harness/src/server/agent-map-mcp-tools.ts +++ b/packages/harness/src/server/agent-map-mcp-tools.ts @@ -13,6 +13,10 @@ import { proposalBatchRequestSchema } from "../core/agent-map-proposal-schema.js import { AgentMapWorkspaceStoreError } from "../core/agent-map-workspace-store.js"; import { AgentBriefService, AgentBriefServiceError } from "../core/agent-brief-service.js"; import { BuildPlanService, BuildPlanServiceError } from "../core/build-plan-service.js"; +import { + SubsessionCoordinator, + SubsessionCoordinatorError, +} from "../core/subsession-coordinator.js"; import { agentBriefRefreshRequestSchema, buildPlanApplyRequestSchema, @@ -55,10 +59,63 @@ const batchSchema = z }) .strict(); +const versionId = z.string().min(1).max(128); +const digest = z.string().regex(/^sha256:[0-9a-f]{64}$/u); +const mapVersionRefSchema = z.object({ + projectId: versionId, + versionId, + contentDigest: digest, +}).strict(); +const planVersionRefSchema = z.object({ + projectId: versionId, + planId: versionId, + versionId, + semanticDigest: digest, +}).strict(); +const briefVersionRefSchema = z.object({ + projectId: versionId, + briefId: versionId, + versionId, + semanticDigest: digest, +}).strict(); +const delegationFocusSchema = z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("assignment"), map: mapVersionRefSchema, + plan: planVersionRefSchema, assignmentId: versionId }).strict(), + z.object({ kind: z.literal("map-node"), map: mapVersionRefSchema, + plan: planVersionRefSchema.nullable(), nodeId: versionId }).strict(), + z.object({ kind: z.literal("brief"), brief: briefVersionRefSchema }).strict(), +]); +const delegationKey = z.string().min(1).max(128).regex(/^[A-Za-z0-9._-]+$/u); +const projectSubsessionRequestSchema = z.object({ + schemaVersion: z.literal(1), + requestKey: delegationKey, + operation: z.discriminatedUnion("kind", [ + z.object({ + kind: z.literal("delegate"), + delegations: z.array(z.object({ + delegationKey, + outcome: z.string().min(1).max(4_096), + kickoffContext: z.string().min(1).max(16_384).optional(), + focus: delegationFocusSchema.optional(), + }).strict()).min(1).max(16), + }).strict(), + z.object({ + kind: z.literal("refresh-focused-context"), + target: z.discriminatedUnion("kind", [ + z.object({ kind: z.literal("self") }).strict(), + z.object({ kind: z.literal("child"), delegationKey }).strict(), + ]), + expectedContextEpoch: z.number().int().positive(), + expectedContextDigest: digest, + focus: delegationFocusSchema.nullable(), + }).strict(), + ]), +}).strict(); + export interface AgentMapToolEvent { tool: "agent_map_read" | "agent_map_validate" | "agent_map_propose" | "build_plan_read" | "build_plan_validate" | "build_plan_apply" | "build_plan_rebase" | - "build_plan_brief_refresh"; + "build_plan_brief_refresh" | "project_subsession_delegate"; outcome: "ok" | "error"; errorCode?: string; latencyMs: number; @@ -107,6 +164,8 @@ function errorResult(error: unknown) { recovery: error.code === "request_id_reused" || error.code === "request_id_expired" ? "new_request" : error.code === "source_mismatch" ? "reread" : error.code === "malformed_input" ? "correct" : "retry" } + : error instanceof SubsessionCoordinatorError + ? error.detail : error instanceof AgentMapWorkspaceStoreError ? { code: error.code, recovery: error.code === "storage_unavailable" ? "retry" : "reread" } : { code: "internal_error", recovery: "retry" }; @@ -130,6 +189,7 @@ export function createAgentMapToolServer( service: AgentMapProposalService, buildPlanService: BuildPlanService, agentBriefService: AgentBriefService, + subsessionCoordinator: SubsessionCoordinator, options: AgentMapMcpToolsOptions = {}, ): McpServer { const server = new McpServer({ @@ -315,5 +375,18 @@ export function createAgentMapToolServer( }), ); + server.registerTool( + "project_subsession_delegate", + { + description: "Create or reuse one or a bounded batch of ordinary writable project subsessions, or refresh exact focused context, using caller-owned idempotency keys.", + inputSchema: projectSubsessionRequestSchema, + annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false }, + }, + async (request) => instrument("project_subsession_delegate", async () => { + const result = await subsessionCoordinator.execute(identity, request); + return toolResult(result, `Delegation reconciled ${result.results.length} project subsession${result.results.length === 1 ? "" : "s"}.`); + }), + ); + return server; } diff --git a/packages/harness/src/server/agent-map-mcp-wiring.test.ts b/packages/harness/src/server/agent-map-mcp-wiring.test.ts index bda819cc..5906d531 100644 --- a/packages/harness/src/server/agent-map-mcp-wiring.test.ts +++ b/packages/harness/src/server/agent-map-mcp-wiring.test.ts @@ -117,6 +117,7 @@ it("uses the actual ephemeral port and revokes private MCP launch authority on e "build_plan_read", "build_plan_rebase", "build_plan_validate", + "project_subsession_delegate", ]); const snapshot = await client.callTool({ name: "agent_map_read", @@ -128,6 +129,78 @@ it("uses the actual ephemeral port and revokes private MCP launch authority on e project: { projectId: session.agentMapIdentity!.projectId }, proposal: null, }); + const stopReadyBridge = server.sessionManager.onStatusChange( + (candidate, context) => { + if ( + candidate.id !== session.id && + candidate.status === "running" && + !candidate.ready && + context.runtimeEpoch + ) { + server!.sessionManager.setReady(candidate.id, context.runtimeEpoch); + } + }, + ); + const delegationArguments = { + schemaVersion: 1, + requestKey: "wiring-delegation", + operation: { + kind: "delegate", + delegations: [{ + delegationKey: "child", + outcome: "Implement the focused child task", + }], + }, + }; + const delegated = await client.callTool({ + name: "project_subsession_delegate", + arguments: delegationArguments, + }); + const childMcp = launchOpts?.agentMapMcp; + expect(childMcp).toBeDefined(); + const childClient = new Client({ name: "nested-delegation-test", version: "1" }); + await childClient.connect(new StreamableHTTPClientTransport(new URL(childMcp!.url), { + requestInit: { headers: { Authorization: `Bearer ${childMcp!.bearerToken}` } }, + })); + expect((await childClient.listTools()).tools.map(({ name }) => name)).toContain( + "project_subsession_delegate", + ); + const nested = await childClient.callTool({ + name: "project_subsession_delegate", + arguments: { + schemaVersion: 1, + requestKey: "nested-request", + operation: { + kind: "delegate", + delegations: [{ + delegationKey: "grandchild", + outcome: "Implement the nested task", + }], + }, + }, + }); + expect(nested.structuredContent).toMatchObject({ + results: [{ outcome: "created", sessionState: "ready" }], + }); + await childClient.close(); + const retried = await client.callTool({ + name: "project_subsession_delegate", + arguments: delegationArguments, + }); + stopReadyBridge(); + expect(delegated.isError).not.toBe(true); + expect(delegated.structuredContent).toMatchObject({ + requestKey: "wiring-delegation", + results: [{ outcome: "created", sessionState: "ready" }], + }); + expect(retried.structuredContent).toMatchObject({ + replayed: true, + results: [{ + outcome: "reused", + sessionId: (delegated.structuredContent as { results: Array<{ sessionId: string }> }).results[0]!.sessionId, + }], + }); + expect(server.sessionManager.list()).toHaveLength(3); await client.close(); await server.sessionManager.kill(session.id); @@ -391,6 +464,7 @@ it("gives every signed-out project session the same coding prompt and Agent Map "build_plan_read", "build_plan_rebase", "build_plan_validate", + "project_subsession_delegate", ]); const proposalEvents: BusMessage[] = []; diff --git a/packages/harness/src/server/agent-map-mcp.test.ts b/packages/harness/src/server/agent-map-mcp.test.ts index b25ade20..c3c4f280 100644 --- a/packages/harness/src/server/agent-map-mcp.test.ts +++ b/packages/harness/src/server/agent-map-mcp.test.ts @@ -15,6 +15,7 @@ import { AgentMapWorkspaceStore } from "../core/agent-map-workspace-store.js"; import { BuildPlanService } from "../core/build-plan-service.js"; import { BuildPlanStore } from "../core/build-plan-store.js"; import { AgentBriefService } from "../core/agent-brief-service.js"; +import type { SubsessionCoordinator } from "../core/subsession-coordinator.js"; import { createAgentMapMcpRouter, type AgentMapMcpRouterOptions, @@ -56,7 +57,17 @@ async function fixture( const buildPlanService = new BuildPlanService(new BuildPlanStore(workspaceStore)); const briefStore = new BuildPlanStore(workspaceStore); const agentBriefService = createAgentBriefService?.(briefStore) ?? new AgentBriefService(briefStore); - const mcp = createAgentMapMcpRouter({ capabilities, service, buildPlanService, agentBriefService, ...routerOptions }); + const subsessionCoordinator = { + execute: vi.fn(async (_identity, request: { requestKey: string }) => ({ + schemaVersion: 1 as const, + requestKey: request.requestKey, + requestDigest: `sha256:${"0".repeat(64)}`, + replayed: false, + results: [], + })), + } as unknown as SubsessionCoordinator; + const mcp = createAgentMapMcpRouter({ capabilities, service, buildPlanService, + agentBriefService, subsessionCoordinator, ...routerOptions }); const app = express(); app.use(express.json()); app.use(mcp.router); @@ -103,6 +114,7 @@ describe("Agent Map Streamable HTTP MCP", () => { "build_plan_read", "build_plan_rebase", "build_plan_validate", + "project_subsession_delegate", ]); const nonStrict = tools.tools.filter((tool) => !(tool.inputSchema.additionalProperties === false || (Array.isArray(tool.inputSchema.anyOf) && tool.inputSchema.anyOf.every((variant) => @@ -116,6 +128,26 @@ describe("Agent Map Streamable HTTP MCP", () => { isError: true, structuredContent: { code: "malformed_input", recovery: "correct" }, }); + await expect(client.callTool({ + name: "project_subsession_delegate", + arguments: { + schemaVersion: 1, + requestKey: "delegate-one", + operation: { + kind: "delegate", + delegations: [{ + delegationKey: "focused-task", + outcome: "Implement the focused task", + }], + }, + }, + })).resolves.toMatchObject({ + structuredContent: { + schemaVersion: 1, + requestKey: "delegate-one", + replayed: false, + }, + }); const validate = tools.tools.find( ({ name }) => name === "agent_map_validate", )!; diff --git a/packages/harness/src/server/agent-map-mcp.ts b/packages/harness/src/server/agent-map-mcp.ts index 8bda0f4f..00a36b33 100644 --- a/packages/harness/src/server/agent-map-mcp.ts +++ b/packages/harness/src/server/agent-map-mcp.ts @@ -15,6 +15,7 @@ import { import type { AgentMapProposalService } from "../core/agent-map-proposal-service.js"; import type { BuildPlanService } from "../core/build-plan-service.js"; import type { AgentBriefService } from "../core/agent-brief-service.js"; +import type { SubsessionCoordinator } from "../core/subsession-coordinator.js"; import { createAgentMapToolServer, type AgentMapMcpToolsOptions } from "./agent-map-mcp-tools.js"; interface BoundTransport { @@ -30,6 +31,7 @@ export interface AgentMapMcpRouterOptions service: AgentMapProposalService; buildPlanService: BuildPlanService; agentBriefService: AgentBriefService; + subsessionCoordinator: SubsessionCoordinator; readSnapshotFor?: (identity: ResolvedAgentMapCapability["identity"]) => Promise; maxSessions?: number; now?: () => number; @@ -156,7 +158,7 @@ export function createAgentMapMcpRouter(options: AgentMapMcpRouterOptions): Agen if (sessionId) sessions.delete(sessionId); }; const server = createToolServer(capability.identity, options.service, options.buildPlanService, - options.agentBriefService, { + options.agentBriefService, options.subsessionCoordinator, { onEvent: options.onEvent, ...(options.readSnapshotFor ? { diff --git a/packages/harness/src/server/index.ts b/packages/harness/src/server/index.ts index 8b26f6e2..dc89b182 100644 --- a/packages/harness/src/server/index.ts +++ b/packages/harness/src/server/index.ts @@ -177,6 +177,14 @@ import { AgentMapProposalService } from "../core/agent-map-proposal-service.js"; import { BuildPlanService } from "../core/build-plan-service.js"; import { AgentBriefService } from "../core/agent-brief-service.js"; import { BuildPlanStore } from "../core/build-plan-store.js"; +import { + SubsessionCoordinator, + type SubsessionCoordinatorEvent, +} from "../core/subsession-coordinator.js"; +import { + SubsessionCoordinatorStore, + type SubsessionCoordinatorStoreEvent, +} from "../core/subsession-coordinator-store.js"; import { AgentMapCapabilityRegistry, type AgentMapCapabilityEvent, @@ -3064,8 +3072,9 @@ export const startServer = async ( bus.publish({ type: "agent-map.proposal.changed", delta }), }, ); + const buildPlanStore = new BuildPlanStore(agentMapWorkspaceStore); const buildPlanService = new BuildPlanService( - new BuildPlanStore(agentMapWorkspaceStore), + buildPlanStore, { onOutcome: (event) => { const analyticsEvent: AnalyticsEvent = { @@ -3094,7 +3103,7 @@ export const startServer = async ( }, ); const agentBriefService = new AgentBriefService( - new BuildPlanStore(agentMapWorkspaceStore), + buildPlanStore, { onOutcome: (event) => { const analyticsEvent: AnalyticsEvent = { @@ -3127,6 +3136,46 @@ export const startServer = async ( }, }, ); + const emitSubsessionEvent = ( + event: SubsessionCoordinatorEvent | SubsessionCoordinatorStoreEvent, + ): void => { + const eventSessionId = + "sessionId" in event && event.sessionId + ? event.sessionId + : `subsession-${event.projectId}`; + const analyticsEvent: AnalyticsEvent = { + eventId: randomUUID(), + seq: seqCounter.next(eventSessionId), + ts: new Date().toISOString(), + userId: identity?.userId ?? null, + tenantId: identity?.tenantId ?? null, + machineId, + harnessSessionId: eventSessionId, + agentSessionId: null, + harness: sessionManager.get(eventSessionId)?.harness ?? "claude-code", + type: event.name, + payload: { + project_id: event.projectId, + ...("count" in event && event.count !== undefined + ? { count: Math.max(0, Math.min(16, event.count)) } + : {}), + ...("code" in event && event.code ? { error_code: event.code } : {}), + }, + }; + void eventStore.append(analyticsEvent).catch(() => {}); + batcher.enqueue(analyticsEvent); + }; + const subsessionCoordinatorStore = new SubsessionCoordinatorStore( + statePaths.agentMap, + { onEvent: emitSubsessionEvent }, + ); + const subsessionCoordinator = new SubsessionCoordinator({ + store: subsessionCoordinatorStore, + sessionManager, + planningStore: buildPlanStore, + eventReader: eventStore, + onEvent: emitSubsessionEvent, + }); emitAgentMapCapabilityEvent = (event) => { const analyticsEvent: AnalyticsEvent = { eventId: randomUUID(), @@ -3152,6 +3201,7 @@ export const startServer = async ( service: agentMapProposalService, buildPlanService, agentBriefService, + subsessionCoordinator, readSnapshotFor: async ({ projectId }) => { const project = await studioProjectCatalog.resolve(projectId); if (!project) throw new AgentMapMcpProjectUnavailableError(); @@ -4106,6 +4156,9 @@ export const startServer = async ( void projectBootstrap!.onEventPersisted(event, runtimeEpoch).catch(() => { console.error("[harness] project bootstrap completion failed"); }); + void subsessionCoordinator.onEventPersisted(event, runtimeEpoch).catch(() => { + console.error("[harness] subsession acknowledgement failed"); + }); const recordChanged = sessionRecordChangedMessage(event); if (recordChanged) bus.publish(recordChanged); // The normal end of a session: the SessionEnd hook's event is in the diff --git a/packages/harness/src/shared/types.ts b/packages/harness/src/shared/types.ts index 5723dc4e..005f55d2 100644 --- a/packages/harness/src/shared/types.ts +++ b/packages/harness/src/shared/types.ts @@ -837,6 +837,21 @@ export type AnalyticsEventType = | "agent_map.capability" | "build_plan.operation" | "agent_brief.refresh" + | "subsession.store_initialized" + | "subsession.binding_reserved" + | "subsession.duplicate_prevented" + | "subsession.spawn_claimed" + | "subsession.requested" + | "subsession.created" + | "subsession.reused" + | "subsession.ready" + | "subsession.failed" + | "subsession.kickoff_claimed" + | "subsession.kickoff_submitted" + | "subsession.kickoff_acknowledged" + | "subsession.kickoff_uncertain" + | "subsession.context_stale" + | "subsession.manual_session_protected" | "project_agent.identity_migrated" | "project_agent.identity_rejected" | "project_bootstrap.scheduled" From b84d878c6b3f6fd4eefac954fccdc352ed58637b Mon Sep 17 00:00:00 2001 From: Yash Date: Fri, 4 Sep 2026 19:00:17 +0000 Subject: [PATCH 06/19] feat(harness): recover bound subsession runtimes Refs: SAP-3151 --- .../harness/src/core/session-manager.test.ts | 37 ++++++ packages/harness/src/core/session-manager.ts | 65 ++++++++++ .../src/core/subsession-coordinator.test.ts | 55 +++++++- .../src/core/subsession-coordinator.ts | 118 +++++++++++++++++- 4 files changed, 267 insertions(+), 8 deletions(-) diff --git a/packages/harness/src/core/session-manager.test.ts b/packages/harness/src/core/session-manager.test.ts index c499d9cd..6bedd114 100644 --- a/packages/harness/src/core/session-manager.test.ts +++ b/packages/harness/src/core/session-manager.test.ts @@ -330,6 +330,43 @@ describe("SessionManager", () => { expect(adapter.launch).toHaveBeenCalledTimes(2); }); + it("resumes an exact coordinator-owned conversation under an advanced marker", async () => { + const { manager, adapter, spawns } = makeManager(); + const sessionId = "00000000-0000-4000-8000-000000000115"; + const input = delegatedCreate(sessionId); + await manager.createReserved( + sessionId, + { cwd: input.cwd, harness: input.harness }, + marker(sessionId), + input.trusted, + ); + const firstRuntime = manager.getRuntimeEpoch(sessionId)!; + await manager.setAgentSessionId( + sessionId, + "agent-session-1", + "startup", + firstRuntime, + ); + spawns[0]!.emitExit(0); + await manager.flush(); + + const resumed = await manager.resumeBound( + sessionId, + marker(sessionId), + marker(sessionId, 2, 2), + ); + + expect(resumed).toMatchObject({ id: sessionId, status: "running" }); + expect(manager.getSubsessionBinding(sessionId)).toEqual( + marker(sessionId, 2, 2), + ); + expect(adapter.resume).toHaveBeenCalledWith( + "agent-session-1", + expect.objectContaining({ harnessSessionId: sessionId }), + ); + expect(manager.list().filter(({ id }) => id === sessionId)).toHaveLength(1); + }); + it("refuses a fresh bound restart when any recorded turn exists", async () => { const { manager, spawns } = makeManager(); const sessionId = "00000000-0000-4000-8000-000000000113"; diff --git a/packages/harness/src/core/session-manager.ts b/packages/harness/src/core/session-manager.ts index 1a14cdc7..09c65d6a 100644 --- a/packages/harness/src/core/session-manager.ts +++ b/packages/harness/src/core/session-manager.ts @@ -805,6 +805,11 @@ export interface TrustedSessionResumeOptions { /** Recomputed focused context for the resumed process. */ promptAppendix?: string; focusedContext?: FocusedSessionContextProjection; + /** Private two-sided coordinator transition, never accepted by REST. */ + subsessionBindingTransition?: Readonly<{ + expected: TrustedSubsessionBindingMarker; + next: TrustedSubsessionBindingMarker; + }>; } interface PtyHandle { @@ -1118,6 +1123,16 @@ export class SessionManager { return this.sessions.get(id); } + /** Read-only vendor-history probe used before a coordinator claims recovery. */ + async canResumeSession(id: string): Promise { + const session = this.sessions.get(id); + if (!session?.agentSessionId) return false; + return this.getAdapter(session.harness).canResume( + session.agentSessionId, + session.cwd, + ); + } + /** True only when this process owns the live PTY behind the record. */ isLive(id: string): boolean { return this.ptys.has(id); @@ -1637,6 +1652,33 @@ export class SessionManager { if (this.rejectedProjectSessionMetadata.has(id)) { throw new ProjectSessionScopeUnavailableError(id); } + const bindingTransition = trusted.subsessionBindingTransition; + if (bindingTransition) { + const expected = parseTrustedSubsessionBindingMarker( + bindingTransition.expected, + id, + ); + const next = parseTrustedSubsessionBindingMarker( + bindingTransition.next, + id, + ); + const current = this.subsessionBindings.get(id); + if ( + !expected || + !next || + !current || + !sameSubsessionBinding(current, expected) || + next.projectId !== expected.projectId || + next.parentSessionId !== expected.parentSessionId || + next.bindingId !== expected.bindingId || + next.sessionId !== expected.sessionId || + next.incarnation !== expected.incarnation + 1 || + next.spawnEpoch <= expected.spawnEpoch || + this.userClosedSubsessions.has(id) + ) { + throw new SubsessionBindingMismatchError(); + } + } const adapter = this.getAdapter(session.harness); // Pre-flight against the agent's OWN store before touching the record. // Holding an agentSessionId only means our SessionStart hook fired once; @@ -1655,6 +1697,16 @@ export class SessionManager { `Sessions that ended before their first prompt are never written to the coding agent's history, so there is nothing to resume — start a new session in this directory instead.`, ); } + if (bindingTransition) { + const current = this.subsessionBindings.get(id)!; + this.subsessionBindings.set(id, bindingTransition.next); + try { + await this.persistSubsessionBindings(); + } catch (error) { + this.subsessionBindings.set(id, current); + throw error; + } + } const trustedIdentity = session.agentMapIdentity; const agentMapIdentity = this.resolveAgentMapIdentity ? await this.resolveAgentMapIdentity(id, session.cwd, trustedIdentity) @@ -1757,6 +1809,19 @@ export class SessionManager { return session; } + /** Server-only same-ID resume fenced by the coordinator's private marker. */ + resumeBound( + id: string, + expected: TrustedSubsessionBindingMarker, + next: TrustedSubsessionBindingMarker, + trusted: Omit = {}, + ): Promise { + return this.resume(id, { + ...trusted, + subsessionBindingTransition: { expected, next }, + }); + } + /** * Signals the session's pty to exit and returns a Promise that resolves * once the process is **actually gone** — not fire-and-forget. diff --git a/packages/harness/src/core/subsession-coordinator.test.ts b/packages/harness/src/core/subsession-coordinator.test.ts index 1b272b02..3184e3fb 100644 --- a/packages/harness/src/core/subsession-coordinator.test.ts +++ b/packages/harness/src/core/subsession-coordinator.test.ts @@ -27,7 +27,7 @@ const identity: ProjectAgentSession = { sessionId: parentId, }; -function adapter(): HarnessAdapter { +function adapter(resumable = false): HarnessAdapter { const spec = (cwd: string): SpawnSpec => ({ command: "fake-claude", args: [], @@ -41,7 +41,7 @@ function adapter(): HarnessAdapter { resume: (_id, { cwd }) => spec(cwd), doctor: async () => [], listPastSessions: async () => [], - canResume: async () => false, + canResume: async () => resumable, }; } @@ -64,6 +64,8 @@ function fakePty() { }, } as unknown as ReturnType, writes, + emitExit: (exitCode = 0) => + exits.forEach((listener) => listener({ exitCode })), }; } @@ -79,7 +81,7 @@ describe("SubsessionCoordinator", () => { ); }); - async function fixture() { + async function fixture(resumable = false) { const root = await fs.mkdtemp(path.join(os.tmpdir(), "subsession-service-")); roots.push(root); const spawned: ReturnType[] = []; @@ -89,7 +91,7 @@ describe("SubsessionCoordinator", () => { return spawnedPty.pty; }); const manager = new SessionManager({ - adapters: { "claude-code": adapter() }, + adapters: { "claude-code": adapter(resumable) }, ingestUrl: "http://127.0.0.1:4100/ingest", ingestCredentials: new IngestCredentialRegistry(), sessionsPath: path.join(root, "sessions.json"), @@ -235,6 +237,51 @@ describe("SubsessionCoordinator", () => { expect(aggregate.bindings[0]!.deliveries[0]!.state).toBe("acknowledged"); }); + it("never fresh-restarts an exited child after kickoff delivery becomes uncertain", async () => { + const { coordinator, caller, manager, store, spawned, spawnPty, unsubscribe } = + await fixture(); + const first = await coordinator.execute(caller, request); + const childId = first.results[0]!.sessionId!; + spawned[1]!.emitExit(1); + await manager.flush(); + + const retried = await coordinator.execute(caller, request); + const aggregate = await store.read(projectId); + unsubscribe(); + + expect(retried.results[0]).toMatchObject({ + outcome: "failed", + sessionId: childId, + kickoffState: "uncertain", + error: { code: "session_unreachable", retryable: false }, + }); + expect(spawnPty).toHaveBeenCalledTimes(2); + expect(aggregate.bindings[0]!.deliveries[0]!.state).toBe("uncertain"); + }); + + it("resumes an exited coordinator-owned vendor conversation under the same Harness id", async () => { + const { coordinator, caller, manager, spawned, spawnPty, unsubscribe } = + await fixture(true); + const first = await coordinator.execute(caller, request); + const childId = first.results[0]!.sessionId!; + const runtime = manager.getRuntimeEpoch(childId)!; + await manager.setAgentSessionId(childId, "agent-child-1", "startup", runtime); + spawned[1]!.emitExit(0); + await manager.flush(); + + const retried = await coordinator.execute(caller, request); + unsubscribe(); + + expect(retried.results[0]).toMatchObject({ + outcome: "reused", + sessionId: childId, + sessionState: "ready", + kickoffState: "uncertain", + }); + expect(manager.list().filter(({ id }) => id === childId)).toHaveLength(1); + expect(spawnPty).toHaveBeenCalledTimes(3); + }); + it("fails closed when the caller identity is not its trusted session scope", async () => { const { coordinator, caller, manager, unsubscribe } = await fixture(); await expect( diff --git a/packages/harness/src/core/subsession-coordinator.ts b/packages/harness/src/core/subsession-coordinator.ts index 4d0d8dc6..6d562bd6 100644 --- a/packages/harness/src/core/subsession-coordinator.ts +++ b/packages/harness/src/core/subsession-coordinator.ts @@ -30,10 +30,7 @@ import { serializeFocusedSessionContext, type FocusedSessionContextProjection, } from "./focused-session-context.js"; -import { - SessionNotReadyError, - SubsessionBindingMismatchError, -} from "./errors.js"; +import { SessionNotReadyError, SubsessionBindingMismatchError } from "./errors.js"; import type { SessionManager, TrustedSubsessionBindingMarker, @@ -537,6 +534,16 @@ export class SubsessionCoordinator { } throw error("binding_session_mismatch", false, "inspect_session"); } + if (existing.status === "exited") { + binding = await this.recoverExitedSession( + caller, + identity, + binding, + privateMarker, + projection, + ); + return { binding, created: false, reused: true }; + } throw error("session_unreachable", true, "inspect_session"); } @@ -621,6 +628,109 @@ export class SubsessionCoordinator { return { binding, created: true, reused: false }; } + private async recoverExitedSession( + caller: ProjectAgentSession, + identity: ProjectAgentSession, + initial: SubsessionBindingRecord, + currentMarker: TrustedSubsessionBindingMarker, + projection: FocusedSessionContextProjection | null, + ): Promise { + let binding = initial; + if ( + binding.runtime && + ["starting", "awaiting-ready", "ready"].includes(binding.sessionState) + ) { + binding = await this.options.store.transitionSession( + identity, + binding.bindingId, + { + expectedLifecycleEpoch: binding.lifecycleEpoch, + expectedSpawnEpoch: binding.spawnEpoch, + expectedRuntimeToken: binding.runtime.runtimeToken, + state: "exited", + }, + ); + } + const resumable = await this.options.sessionManager.canResumeSession( + binding.sessionId, + ); + if (!resumable && currentDelivery(binding).state !== "pending") { + throw error("session_unreachable", false, "inspect_session"); + } + let claim; + if (binding.sessionState === "spawn-claimed" && binding.spawnClaim) { + if (binding.spawnClaim.expiresAt > new Date().toISOString()) return binding; + claim = await this.options.store.takeoverExpiredSpawnClaim( + identity, + binding.bindingId, + { + ownerId: this.ownerId, + expiredClaimId: binding.spawnClaim.claimId, + expectedLifecycleEpoch: binding.lifecycleEpoch, + expectedSpawnEpoch: binding.spawnEpoch, + }, + ); + } else { + claim = await this.options.store.claimSpawn( + identity, + binding.bindingId, + { + ownerId: this.ownerId, + expectedLifecycleEpoch: binding.lifecycleEpoch, + expectedSpawnEpoch: binding.spawnEpoch, + }, + ); + } + if (!claim.claimed) return claim.binding; + binding = claim.binding; + const spawnClaim = binding.spawnClaim!; + const nextMarker = markerFor(binding, currentMarker.incarnation + 1); + const trusted = projection ? { focusedContext: projection } : {}; + if (resumable) { + await this.options.sessionManager.resumeBound( + binding.sessionId, + currentMarker, + nextMarker, + trusted, + ); + } else { + const index = await this.options.eventReader.index(); + const hasRecordedTurns = async () => + (index.bySession.get(binding.sessionId)?.turnCount ?? 0) > 0; + await this.options.sessionManager.restartFreshBound( + binding.sessionId, + currentMarker, + nextMarker, + { + agentMapIdentity: (sessionId) => ({ + projectId: binding.projectId, + userId: caller.userId, + sessionId, + }), + initialTitle: this.title(binding.outcome), + ...(projection ? { focusedContext: () => projection } : {}), + }, + hasRecordedTurns, + ); + } + const runtimeToken = this.options.sessionManager.getRuntimeEpoch( + binding.sessionId, + ); + if (!runtimeToken) + throw error("session_restart_failed", true, "retry"); + binding = await this.options.store.attachSpawnedRuntime( + identity, + binding.bindingId, + { + claimId: spawnClaim.claimId, + spawnEpoch: binding.spawnEpoch, + runtimeToken, + incarnation: nextMarker.incarnation, + }, + ); + return this.advanceToReady(identity, binding, runtimeToken); + } + private async advanceToReady( identity: ProjectAgentSession, initial: SubsessionBindingRecord, From 9b14572209f564c4e3ebec19b37caf0c5dffc2dc Mon Sep 17 00:00:00 2001 From: Yash Date: Fri, 4 Sep 2026 19:11:20 +0000 Subject: [PATCH 07/19] fix(harness): fence codex rollout ownership Refs: SAP-3151 --- .../collector/codex-rollout-broker.test.ts | 150 ++++++++++++++++++ .../core/collector/codex-rollout-broker.ts | 150 ++++++++++++++++++ .../src/core/collector/codex-tailer.ts | 32 +++- .../harness/src/core/session-manager.test.ts | 19 +++ packages/harness/src/core/session-manager.ts | 44 +++++ .../src/core/subsession-coordinator.test.ts | 49 +++++- .../src/core/subsession-coordinator.ts | 36 ++++- .../src/server/codex-tailer-wiring.test.ts | 21 ++- packages/harness/src/server/index.ts | 78 ++++++--- 9 files changed, 536 insertions(+), 43 deletions(-) create mode 100644 packages/harness/src/core/collector/codex-rollout-broker.test.ts create mode 100644 packages/harness/src/core/collector/codex-rollout-broker.ts diff --git a/packages/harness/src/core/collector/codex-rollout-broker.test.ts b/packages/harness/src/core/collector/codex-rollout-broker.test.ts new file mode 100644 index 00000000..5364f560 --- /dev/null +++ b/packages/harness/src/core/collector/codex-rollout-broker.test.ts @@ -0,0 +1,150 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { CodexRolloutBroker } from "./codex-rollout-broker.js"; + +const meta = (id: string, cwd: string, timestamp: string) => + `${JSON.stringify({ type: "session_meta", payload: { id, cwd, timestamp } })}\n`; + +describe("CodexRolloutBroker", () => { + const roots: string[] = []; + + afterEach(async () => { + await Promise.all( + roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), + ); + }); + + async function fixture() { + const home = await mkdtemp(join(tmpdir(), "codex-rollout-broker-")); + const cwd = join(home, "project"); + const sessions = join(home, ".codex", "sessions", "2026", "09", "04"); + await Promise.all([mkdir(cwd), mkdir(sessions, { recursive: true })]); + roots.push(home); + return { home, cwd, sessions }; + } + + it("uniquely assigns concurrent same-root rollouts by process epoch", async () => { + const { home, cwd, sessions } = await fixture(); + const firstTime = Date.parse("2026-09-04T10:00:00.000Z"); + const secondTime = Date.parse("2026-09-04T10:00:01.000Z"); + const firstPath = join(sessions, "rollout-first.jsonl"); + const secondPath = join(sessions, "rollout-second.jsonl"); + await writeFile( + firstPath, + meta("agent-first", cwd, "2026-09-04T10:00:00.500Z"), + ); + await writeFile( + secondPath, + meta("agent-second", cwd, "2026-09-04T10:00:01.500Z"), + ); + const broker = new CodexRolloutBroker(home); + broker.register({ + sessionId: "first", + runtimeEpoch: "runtime-1", + cwd, + sinceMs: firstTime, + }); + broker.register({ + sessionId: "second", + runtimeEpoch: "runtime-2", + cwd, + sinceMs: secondTime, + }); + + await expect( + broker.claimFresh({ + sessionId: "first", + runtimeEpoch: "runtime-1", + cwd, + sinceMs: firstTime, + }), + ).resolves.toEqual({ outcome: "claimed", path: firstPath }); + await expect( + broker.claimFresh({ + sessionId: "second", + runtimeEpoch: "runtime-2", + cwd, + sinceMs: secondTime, + }), + ).resolves.toEqual({ outcome: "claimed", path: secondPath }); + }); + + it("fails closed when same-root process epochs cannot distinguish candidates", async () => { + const { home, cwd, sessions } = await fixture(); + const sinceMs = Date.parse("2026-09-04T10:00:00.000Z"); + await writeFile( + join(sessions, "rollout-a.jsonl"), + meta("agent-a", cwd, "2026-09-04T10:00:01.000Z"), + ); + await writeFile( + join(sessions, "rollout-b.jsonl"), + meta("agent-b", cwd, "2026-09-04T10:00:02.000Z"), + ); + const broker = new CodexRolloutBroker(home); + broker.register({ + sessionId: "first", + runtimeEpoch: "runtime-1", + cwd, + sinceMs, + }); + broker.register({ + sessionId: "second", + runtimeEpoch: "runtime-2", + cwd, + sinceMs, + }); + + await expect( + broker.claimFresh({ + sessionId: "first", + runtimeEpoch: "runtime-1", + cwd, + sinceMs, + }), + ).resolves.toEqual({ outcome: "ambiguous", path: null }); + await expect( + broker.claimFresh({ + sessionId: "second", + runtimeEpoch: "runtime-2", + cwd, + sinceMs, + }), + ).resolves.toEqual({ outcome: "ambiguous", path: null }); + }); + + it("allows only the same Harness session to reclaim an exact rollout on resume", async () => { + const { home, cwd, sessions } = await fixture(); + const rolloutPath = join(sessions, "rollout-resume.jsonl"); + await writeFile( + rolloutPath, + meta("agent-resume", cwd, "2026-09-04T10:00:01.000Z"), + ); + const broker = new CodexRolloutBroker(home); + const base = { cwd, sinceMs: 0, agentSessionId: "agent-resume" }; + await expect( + broker.claimExact({ + ...base, + sessionId: "owner", + runtimeEpoch: "runtime-1", + }), + ).resolves.toEqual({ outcome: "claimed", path: rolloutPath }); + broker.release("owner", "runtime-1"); + await expect( + broker.claimExact({ + ...base, + sessionId: "owner", + runtimeEpoch: "runtime-2", + }), + ).resolves.toEqual({ outcome: "claimed", path: rolloutPath }); + await expect( + broker.claimExact({ + ...base, + sessionId: "foreign", + runtimeEpoch: "runtime-3", + }), + ).resolves.toEqual({ outcome: "pending", path: null }); + }); +}); diff --git a/packages/harness/src/core/collector/codex-rollout-broker.ts b/packages/harness/src/core/collector/codex-rollout-broker.ts new file mode 100644 index 00000000..a8c18eb8 --- /dev/null +++ b/packages/harness/src/core/collector/codex-rollout-broker.ts @@ -0,0 +1,150 @@ +import { + findRolloutCandidates, + type CodexRolloutCandidate, +} from "./codex-tailer.js"; + +export type CodexRolloutClaimResult = + | Readonly<{ outcome: "claimed"; path: string }> + | Readonly<{ outcome: "pending" | "ambiguous"; path: null }>; + +type PendingRuntime = Readonly<{ + sessionId: string; + runtimeEpoch: string; + cwd: string; + sinceMs: number; +}>; + +const runtimeKey = (sessionId: string, runtimeEpoch: string) => + `${sessionId}\0${runtimeEpoch}`; + +/** + * Process-epoch rollout ownership for fresh Codex sessions. A path is claimed + * at most once. Singleton elimination across every same-root pending launch + * handles the common A={a,b}, B={b} race without guessing; an unresolved + * many-to-many match remains explicitly ambiguous. + */ +export class CodexRolloutBroker { + private readonly pending = new Map(); + private readonly assignments = new Map(); + private readonly claimedPaths = new Map(); + private queue: Promise = Promise.resolve(); + + constructor(private readonly homeDir?: string) {} + + register(input: PendingRuntime): void { + const key = runtimeKey(input.sessionId, input.runtimeEpoch); + if (this.assignments.has(key) || this.pending.has(key)) return; + this.pending.set(key, { ...input }); + } + + release(sessionId: string, runtimeEpoch: string): void { + const key = runtimeKey(sessionId, runtimeEpoch); + this.pending.delete(key); + const assigned = this.assignments.get(key); + this.assignments.delete(key); + // Keep the path tombstone. A rollout is never adopted by another Harness + // session, though an exact resume of the same session may reclaim it. + void assigned; + } + + releaseSession(sessionId: string): void { + for (const [key, pending] of this.pending) { + if (pending.sessionId === sessionId) this.pending.delete(key); + } + for (const key of this.assignments.keys()) { + if (key.startsWith(`${sessionId}\0`)) this.assignments.delete(key); + } + } + + async claimExact( + input: PendingRuntime & { agentSessionId: string }, + ): Promise { + return this.serialized(async () => { + const key = runtimeKey(input.sessionId, input.runtimeEpoch); + const assigned = this.assignments.get(key); + if (assigned) return { outcome: "claimed", path: assigned } as const; + const candidates = await findRolloutCandidates({ + cwd: input.cwd, + agentSessionId: input.agentSessionId, + homeDir: this.homeDir, + }); + const candidate = candidates.find(({ path }) => { + const owner = this.claimedPaths.get(path); + return !owner || owner.startsWith(`${input.sessionId}\0`); + }); + if (!candidate) return { outcome: "pending", path: null } as const; + this.assign(key, candidate.path, input.sessionId); + return { outcome: "claimed", path: candidate.path } as const; + }); + } + + async claimFresh(input: PendingRuntime): Promise { + this.register(input); + return this.serialized(async () => { + const key = runtimeKey(input.sessionId, input.runtimeEpoch); + const assigned = this.assignments.get(key); + if (assigned) return { outcome: "claimed", path: assigned } as const; + + const group = [...this.pending.entries()].filter( + ([, candidate]) => candidate.cwd === input.cwd, + ); + const possibilities = new Map(); + for (const [candidateKey, pending] of group) { + possibilities.set( + candidateKey, + await findRolloutCandidates({ + cwd: pending.cwd, + sinceMs: pending.sinceMs, + homeDir: this.homeDir, + excludePaths: new Set(this.claimedPaths.keys()), + }), + ); + } + + let changed = true; + while (changed) { + changed = false; + const singles = [...possibilities.entries()] + .filter(([, candidates]) => candidates.length === 1) + .sort(([left], [right]) => left.localeCompare(right)); + for (const [candidateKey, [candidate]] of singles) { + if (!candidate || this.claimedPaths.has(candidate.path)) continue; + this.assign(candidateKey, candidate.path); + possibilities.delete(candidateKey); + for (const remaining of possibilities.values()) { + const index = remaining.findIndex( + ({ path }) => path === candidate.path, + ); + if (index >= 0) remaining.splice(index, 1); + } + changed = true; + } + } + + const resolved = this.assignments.get(key); + if (resolved) return { outcome: "claimed", path: resolved } as const; + const remaining = possibilities.get(key) ?? []; + return { + outcome: remaining.length > 1 ? "ambiguous" : "pending", + path: null, + } as const; + }); + } + + private assign(key: string, path: string, resumableSessionId?: string): void { + const owner = this.claimedPaths.get(path); + if (owner && !owner.startsWith(`${resumableSessionId ?? ""}\0`)) return; + this.assignments.set(key, path); + this.claimedPaths.set(path, key); + this.pending.delete(key); + } + + private serialized(operation: () => Promise): Promise { + const result = this.queue.catch(() => {}).then(operation); + this.queue = result.then( + () => undefined, + () => undefined, + ); + return result; + } +} diff --git a/packages/harness/src/core/collector/codex-tailer.ts b/packages/harness/src/core/collector/codex-tailer.ts index b319b1e3..a2d4a771 100644 --- a/packages/harness/src/core/collector/codex-tailer.ts +++ b/packages/harness/src/core/collector/codex-tailer.ts @@ -269,6 +269,15 @@ export interface FindRolloutFileOptions { agentSessionId?: string; /** Overridable for tests. */ homeDir?: string; + /** Exact paths already owned by another live Harness runtime. */ + excludePaths?: ReadonlySet; +} + +export interface CodexRolloutCandidate { + path: string; + agentSessionId: string | null; + timestampMs: number | null; + mtimeMs: number; } interface RolloutSessionMeta { @@ -342,6 +351,14 @@ async function collectRolloutFiles(dir: string, depth = 0): Promise { * exact rollout path in advance (it's a timestamp+UUID Codex generates itself). */ export async function findRolloutFile(options: FindRolloutFileOptions): Promise { + const candidates = await findRolloutCandidates(options); + return candidates.at(-1)?.path ?? null; +} + +/** Returns every compatible rollout in deterministic chronological order. */ +export async function findRolloutCandidates( + options: FindRolloutFileOptions, +): Promise { const homeDir = options.homeDir ?? homedir(); const root = join(homeDir, ".codex", "sessions"); const files = await collectRolloutFiles(root); @@ -356,21 +373,28 @@ export async function findRolloutFile(options: FindRolloutFileOptions): Promise< // exited, or a test double that never touches the real filesystem). const resolvedCwd = await realpath(options.cwd).catch(() => options.cwd); - let best: { path: string; mtimeMs: number } | null = null; + const candidates: CodexRolloutCandidate[] = []; for (const filePath of files) { + if (options.excludePaths?.has(filePath)) continue; const meta = await readSessionMetaHead(filePath); if (!meta || meta.cwd !== resolvedCwd) continue; if (options.agentSessionId !== undefined) { if (meta.id !== options.agentSessionId) continue; - return filePath; + const fileStat = await stat(filePath).catch(() => null); + if (fileStat) + candidates.push({ path: filePath, agentSessionId: meta.id, + timestampMs: meta.timestampMs, mtimeMs: fileStat.mtimeMs }); + continue; } if (options.sinceMs !== undefined && meta.timestampMs !== null && meta.timestampMs < options.sinceMs) continue; const fileStat = await stat(filePath).catch(() => null); if (!fileStat) continue; - if (!best || fileStat.mtimeMs > best.mtimeMs) best = { path: filePath, mtimeMs: fileStat.mtimeMs }; + candidates.push({ path: filePath, agentSessionId: meta.id, + timestampMs: meta.timestampMs, mtimeMs: fileStat.mtimeMs }); } - return best?.path ?? null; + return candidates.sort((left, right) => + left.mtimeMs - right.mtimeMs || left.path.localeCompare(right.path)); } diff --git a/packages/harness/src/core/session-manager.test.ts b/packages/harness/src/core/session-manager.test.ts index 6bedd114..b40e66d6 100644 --- a/packages/harness/src/core/session-manager.test.ts +++ b/packages/harness/src/core/session-manager.test.ts @@ -4576,6 +4576,25 @@ describe("SessionManager", () => { ).toBe(true); }); + it("fences transcript identity state to the exact live runtime epoch", async () => { + const { manager } = makeManager({ + adapter: createFakeAdapter({ eventSource: "transcript-tail" }), + ingestCredentials: new IngestCredentialRegistry( + () => "token-a", + () => "epoch-a", + ), + }); + const session = await manager.create({ + cwd: "/tmp/proj", + harness: "claude-code", + }); + + expect(manager.getAdapterIdentityState(session.id, "epoch-a")).toBe("pending"); + expect(manager.setAdapterIdentityState(session.id, "stale-epoch", "ready")).toBe(false); + expect(manager.setAdapterIdentityState(session.id, "epoch-a", "ready")).toBe(true); + expect(manager.getAdapterIdentityState(session.id, "epoch-a")).toBe("ready"); + }); + it("does not publish a PTY when the runtime epoch transition fails", async () => { const spawnPty = vi.fn(() => { return createFakePty().pty as unknown as ReturnType; diff --git a/packages/harness/src/core/session-manager.ts b/packages/harness/src/core/session-manager.ts index 09c65d6a..dc5fcac0 100644 --- a/packages/harness/src/core/session-manager.ts +++ b/packages/harness/src/core/session-manager.ts @@ -629,6 +629,13 @@ export interface SessionStatusContext { runtimeEpoch: string | null; } +export type AdapterIdentityState = + | "not-required" + | "pending" + | "ready" + | "ambiguous" + | "unavailable"; + export type SessionStatusListener = ( session: HarnessSession, context: SessionStatusContext, @@ -950,6 +957,13 @@ export class SessionManager { /** Last cleanly retired PTY generation. It may finish already-admitted ingest * work while the session is exited, but loses immediately to a replacement. */ private readonly retiredRuntimeEpochs = new Map(); + /** Adapter-owned correlation for transcript-backed runtimes. This is kept + * separate from terminal readiness: a TUI can be interactive before its + * exact vendor transcript has been identified. */ + private readonly adapterIdentityStates = new Map< + string, + { runtimeEpoch: string; state: AdapterIdentityState } + >(); /** Monotonic raw-input observations used to preempt background injection. */ private readonly terminalInputEpochs = new Map(); /** One text→Enter transaction may be staged per session. */ @@ -1143,6 +1157,26 @@ export class SessionManager { return this.ptys.get(id)?.runtimeEpoch ?? null; } + /** Exact-runtime adapter identity used by trusted background delivery. */ + getAdapterIdentityState(id: string, runtimeEpoch: string): AdapterIdentityState { + const state = this.adapterIdentityStates.get(id); + if (!state || state.runtimeEpoch !== runtimeEpoch) return "pending"; + return state.state; + } + + /** Server-only acknowledgement from an adapter-owned identity broker. */ + setAdapterIdentityState( + id: string, + runtimeEpoch: string, + state: Exclude, + ): boolean { + const current = this.adapterIdentityStates.get(id); + if (!current || current.runtimeEpoch !== runtimeEpoch || + !this.isCurrentRuntimeEpoch(id, runtimeEpoch)) return false; + current.state = state; + return true; + } + /** True only for the exact PTY generation that is live right now. */ isCurrentRuntimeEpoch(id: string, runtimeEpoch: string): boolean { const live = this.ptys.get(id); @@ -3093,6 +3127,10 @@ export class SessionManager { env.COLORTERM = "truecolor"; env[ENV.ingestUrl] = `${this.ingestUrl.replace(/\/$/, "")}/ingest`; const ingestCredential = this.issueIngestCredential(session.id); + this.adapterIdentityStates.set(session.id, { + runtimeEpoch: ingestCredential.runtimeEpoch, + state: adapter.eventSource === "transcript-tail" ? "pending" : "not-required", + }); env[ENV.ingestToken] = ingestCredential.token; env[ENV.sessionId] = session.id; if (this.collectorUrl) env[ENV.collectorUrl] = this.collectorUrl; @@ -3134,6 +3172,9 @@ export class SessionManager { }); } catch (error) { this.revokeIngestToken(session.id); + const identityState = this.adapterIdentityStates.get(session.id); + if (identityState?.runtimeEpoch === ingestCredential.runtimeEpoch) + this.adapterIdentityStates.delete(session.id); if (epochTransitioned) { await Promise.resolve( this.onRuntimeEpochTransition?.({ ...session }, null), @@ -3312,6 +3353,9 @@ export class SessionManager { ? sanitizeExitTail(handle.buffer) : null; this.ptys.delete(id); + const identityState = this.adapterIdentityStates.get(id); + if (identityState?.runtimeEpoch === handle.runtimeEpoch) + this.adapterIdentityStates.delete(id); this.retiredRuntimeEpochs.set(id, handle.runtimeEpoch); this.lastActivityBroadcast.delete(id); // Resolve after the pty map is cleaned up. transitionExited runs diff --git a/packages/harness/src/core/subsession-coordinator.test.ts b/packages/harness/src/core/subsession-coordinator.test.ts index 3184e3fb..0605c7ec 100644 --- a/packages/harness/src/core/subsession-coordinator.test.ts +++ b/packages/harness/src/core/subsession-coordinator.test.ts @@ -27,7 +27,10 @@ const identity: ProjectAgentSession = { sessionId: parentId, }; -function adapter(resumable = false): HarnessAdapter { +function adapter( + resumable = false, + eventSource: HarnessAdapter["eventSource"] = "hooks", +): HarnessAdapter { const spec = (cwd: string): SpawnSpec => ({ command: "fake-claude", args: [], @@ -36,7 +39,7 @@ function adapter(resumable = false): HarnessAdapter { }); return { id: "claude-code", - eventSource: "hooks", + eventSource, launch: ({ cwd }) => spec(cwd), resume: (_id, { cwd }) => spec(cwd), doctor: async () => [], @@ -71,9 +74,11 @@ function fakePty() { describe("SubsessionCoordinator", () => { const roots: string[] = []; + const managers: SessionManager[] = []; afterEach(async () => { vi.useRealTimers(); + await Promise.all(managers.splice(0).map((manager) => manager.flush())); await Promise.all( roots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true }), @@ -81,7 +86,10 @@ describe("SubsessionCoordinator", () => { ); }); - async function fixture(resumable = false) { + async function fixture( + resumable = false, + childIdentityState?: "ready" | "ambiguous", + ) { const root = await fs.mkdtemp(path.join(os.tmpdir(), "subsession-service-")); roots.push(root); const spawned: ReturnType[] = []; @@ -91,13 +99,19 @@ describe("SubsessionCoordinator", () => { return spawnedPty.pty; }); const manager = new SessionManager({ - adapters: { "claude-code": adapter(resumable) }, + adapters: { + "claude-code": adapter( + resumable, + childIdentityState ? "transcript-tail" : "hooks", + ), + }, ingestUrl: "http://127.0.0.1:4100/ingest", ingestCredentials: new IngestCredentialRegistry(), sessionsPath: path.join(root, "sessions.json"), spawnPty, resolveAgentMapIdentity: async (_sessionId, _cwd, persisted) => persisted, }); + managers.push(manager); await manager.init(); await manager.create( { cwd: root, harness: "claude-code" }, @@ -119,6 +133,12 @@ describe("SubsessionCoordinator", () => { context.runtimeEpoch ) { manager.setReady(session.id, context.runtimeEpoch); + if (childIdentityState) + manager.setAdapterIdentityState( + session.id, + context.runtimeEpoch, + childIdentityState, + ); } }); const events: AnalyticsEvent[] = []; @@ -294,4 +314,25 @@ describe("SubsessionCoordinator", () => { unsubscribe(); expect(manager.list()).toHaveLength(1); }); + + it("writes no kickoff when adapter identity correlation is ambiguous", async () => { + const { coordinator, caller, spawned, unsubscribe } = await fixture( + false, + "ambiguous", + ); + const result = await coordinator.execute(caller, request); + unsubscribe(); + + expect(result.results[0]).toMatchObject({ + outcome: "failed", + sessionState: "awaiting-ready", + kickoffState: "pending", + error: { + code: "adapter_identity_ambiguous", + retryable: false, + recovery: "inspect_session", + }, + }); + expect(spawned[1]!.writes).toEqual([]); + }); }); diff --git a/packages/harness/src/core/subsession-coordinator.ts b/packages/harness/src/core/subsession-coordinator.ts index 6d562bd6..d7e23679 100644 --- a/packages/harness/src/core/subsession-coordinator.ts +++ b/packages/harness/src/core/subsession-coordinator.ts @@ -41,6 +41,7 @@ import { } from "./subsession-coordinator-store.js"; const DEFAULT_READINESS_TIMEOUT_MS = 30_000; +const ADAPTER_IDENTITY_POLL_MS = 25; const KICKOFF_MARKER = //u; export interface SubsessionCoordinatorEvent { @@ -293,6 +294,13 @@ export class SubsessionCoordinator { }); return this.result(binding, outcome); } catch (cause) { + binding = + (await this.options.store + .readBinding(identity, { + kind: "binding-id", + bindingId: binding.bindingId, + }) + .catch(() => null)) ?? binding; const detail = this.itemError(cause); this.emit({ name: @@ -745,15 +753,14 @@ export class SubsessionCoordinator { expectedLifecycleEpoch: binding.lifecycleEpoch, expectedSpawnEpoch: binding.spawnEpoch, expectedRuntimeToken: runtimeToken, - state: this.options.sessionManager.get(binding.sessionId)?.ready - ? "ready" - : "awaiting-ready", + state: "awaiting-ready", }, ); } if (binding.sessionState === "awaiting-ready") { const ready = await this.waitForReady(binding.sessionId, runtimeToken); if (!ready) throw new SessionNotReadyError(binding.sessionId); + await this.waitForAdapterIdentity(binding.sessionId, runtimeToken); binding = await this.options.store.transitionSession( identity, binding.bindingId, @@ -774,6 +781,29 @@ export class SubsessionCoordinator { return binding; } + private async waitForAdapterIdentity( + sessionId: string, + runtimeToken: string, + ): Promise { + const deadline = Date.now() + this.readinessTimeoutMs; + for (;;) { + if (!this.options.sessionManager.isCurrentRuntimeEpoch(sessionId, runtimeToken)) + throw error("session_unreachable", true, "inspect_session"); + const state = this.options.sessionManager.getAdapterIdentityState( + sessionId, + runtimeToken, + ); + if (state === "ready" || state === "not-required") return; + if (state === "ambiguous") + throw error("adapter_identity_ambiguous", false, "inspect_session"); + if (state === "unavailable") + throw error("adapter_unavailable", true, "retry"); + if (Date.now() >= deadline) + throw error("adapter_unavailable", true, "retry"); + await new Promise((resolve) => setTimeout(resolve, ADAPTER_IDENTITY_POLL_MS)); + } + } + private waitForReady(sessionId: string, runtimeToken: string): Promise { if ( this.options.sessionManager.get(sessionId)?.ready && diff --git a/packages/harness/src/server/codex-tailer-wiring.test.ts b/packages/harness/src/server/codex-tailer-wiring.test.ts index fcc21ae8..945b25a3 100644 --- a/packages/harness/src/server/codex-tailer-wiring.test.ts +++ b/packages/harness/src/server/codex-tailer-wiring.test.ts @@ -13,10 +13,10 @@ import { join } from "node:path"; vi.mock("../core/collector/codex-tailer.js", () => ({ tailCodexRollout: vi.fn(), - findRolloutFile: vi.fn(), + findRolloutCandidates: vi.fn(), })); -import { tailCodexRollout, findRolloutFile, type CodexEventListener } from "../core/collector/codex-tailer.js"; +import { tailCodexRollout, findRolloutCandidates, type CodexEventListener } from "../core/collector/codex-tailer.js"; import { startServer, type HarnessServer } from "./index.js"; import type { HarnessAdapter, LaunchOpts, SpawnSpec } from "../shared/types.js"; @@ -74,7 +74,12 @@ describe("codex tailer lifecycle wiring", () => { tailerEvents.push(opts.onEvent); return fakeHandle; }); - vi.mocked(findRolloutFile).mockReset().mockResolvedValue("/fake/rollout/path.jsonl"); + vi.mocked(findRolloutCandidates).mockReset().mockResolvedValue([{ + path: "/fake/rollout/path.jsonl", + agentSessionId: "agent-fixture", + timestampMs: Date.now(), + mtimeMs: Date.now(), + }]); }); afterEach(async () => { @@ -106,16 +111,16 @@ describe("codex tailer lifecycle wiring", () => { expect(session.status).toBe("running"); await vi.waitFor(() => { - expect(findRolloutFile).toHaveBeenCalled(); + expect(findRolloutCandidates).toHaveBeenCalled(); expect(tailCodexRollout).toHaveBeenCalledWith( expect.objectContaining({ rolloutPath: "/fake/rollout/path.jsonl" }), ); }); - // findRolloutFile should have been asked for this session's cwd, and (a + // Rollout discovery should have been asked for this session's cwd, and (a // fresh launch has no agentSessionId yet) bounded by sinceMs rather than // an exact id. - expect(findRolloutFile).toHaveBeenCalledWith( + expect(findRolloutCandidates).toHaveBeenCalledWith( expect.objectContaining({ cwd, sinceMs: expect.any(Number) }), ); @@ -165,7 +170,7 @@ describe("codex tailer lifecycle wiring", () => { const resumed = await server.sessionManager.resume(historical.id); await vi.waitFor(() => { - expect(findRolloutFile).toHaveBeenCalledWith( + expect(findRolloutCandidates).toHaveBeenCalledWith( expect.objectContaining({ cwd, agentSessionId: "agent-resumed" }), ); }); @@ -263,7 +268,7 @@ describe("codex tailer lifecycle wiring", () => { // Give any (incorrect) codex wiring a chance to fire before asserting it didn't. await new Promise((resolve) => setTimeout(resolve, 50)); - expect(findRolloutFile).not.toHaveBeenCalled(); + expect(findRolloutCandidates).not.toHaveBeenCalled(); expect(tailCodexRollout).not.toHaveBeenCalled(); // See the first test's comment: wait for the real spawned process to diff --git a/packages/harness/src/server/index.ts b/packages/harness/src/server/index.ts index dc89b182..330dc91b 100644 --- a/packages/harness/src/server/index.ts +++ b/packages/harness/src/server/index.ts @@ -94,11 +94,8 @@ import { migrateHarnessIdentity } from "../core/collector/identity-migration.js" import { normalizeHookEvent } from "../core/collector/normalizer.js"; import { enrichTurnCompleted } from "../core/collector/transcript.js"; import { createSeqCounter } from "../core/collector/seq.js"; -import { - findRolloutFile, - tailCodexRollout, - type CodexTailerHandle, -} from "../core/collector/codex-tailer.js"; +import { tailCodexRollout, type CodexTailerHandle } from "../core/collector/codex-tailer.js"; +import { CodexRolloutBroker } from "../core/collector/codex-rollout-broker.js"; import { getOrCreateMachineId } from "../cli/machine-id.js"; import { loadSettings, pruneDeadRecentDirs } from "../cli/settings.js"; import type { HarnessIdentity } from "../cli/auth.js"; @@ -704,6 +701,7 @@ export const startServer = async ( organizationName: identity?.organizationName ?? null, }); const statePaths = resolveStatePaths(options.stateRoot); + const codexRolloutBroker = new CodexRolloutBroker(options.codexHomeDir); const projectBootstrapOutbox = new ProjectBootstrapOutbox( join(statePaths.projectBootstrap, "project-outbox"), ); @@ -1501,6 +1499,18 @@ export const startServer = async ( onTerminalInput: (sessionId, context) => projectBootstrap?.onTerminalInput(sessionId, context), onRuntimeEpochTransition: async (session, runtimeEpoch) => { + if (adapters[session.harness]?.eventSource === "transcript-tail") { + if (runtimeEpoch) { + codexRolloutBroker.register({ + sessionId: session.id, + runtimeEpoch, + cwd: session.cwd, + sinceMs: Date.now(), + }); + } else { + codexRolloutBroker.releaseSession(session.id); + } + } if (!session.projectBootstrap) return; if (!projectBootstrap) { throw new Error("project bootstrap coordinator unavailable"); @@ -4187,25 +4197,30 @@ export const startServer = async ( async function discoverCodexRolloutPath( session: HarnessSession, - ): Promise { + runtimeEpoch: string, + ): Promise<{ path: string | null; ambiguous: boolean }> { const deadline = Date.now() + CODEX_ROLLOUT_DISCOVERY_TIMEOUT_MS; const sinceMs = Date.parse(session.createdAt); + let ambiguous = false; for (;;) { - const found = await findRolloutFile( - session.agentSessionId - ? { - cwd: session.cwd, - agentSessionId: session.agentSessionId, - homeDir: options.codexHomeDir, - } - : { - cwd: session.cwd, - sinceMs: Number.isNaN(sinceMs) ? undefined : sinceMs, - homeDir: options.codexHomeDir, - }, - ); - if (found) return found; - if (Date.now() >= deadline) return null; + const claim = session.agentSessionId + ? await codexRolloutBroker.claimExact({ + sessionId: session.id, + runtimeEpoch, + cwd: session.cwd, + sinceMs: Number.isNaN(sinceMs) ? Date.now() : sinceMs, + agentSessionId: session.agentSessionId, + }) + : await codexRolloutBroker.claimFresh({ + sessionId: session.id, + runtimeEpoch, + cwd: session.cwd, + sinceMs: Number.isNaN(sinceMs) ? Date.now() : sinceMs, + }); + if (claim.outcome === "claimed") + return { path: claim.path, ambiguous: false }; + if (claim.outcome === "ambiguous") ambiguous = true; + if (Date.now() >= deadline) return { path: null, ambiguous }; await new Promise((resolve) => setTimeout(resolve, CODEX_ROLLOUT_DISCOVERY_POLL_MS), ); @@ -4218,10 +4233,16 @@ export const startServer = async ( const runtimeEpoch = sessionManager.getRuntimeEpoch(harnessSessionId); if (!session || runtimeEpoch === null) return; - const rolloutPath = await discoverCodexRolloutPath(session); + const discovery = await discoverCodexRolloutPath(session, runtimeEpoch); + const rolloutPath = discovery.path; if (!rolloutPath) { + sessionManager.setAdapterIdentityState( + harnessSessionId, + runtimeEpoch, + discovery.ambiguous ? "ambiguous" : "unavailable", + ); console.error( - `[harness] codex tailer: no rollout file found for session ${harnessSessionId} (cwd=${session.cwd}) within ${CODEX_ROLLOUT_DISCOVERY_TIMEOUT_MS}ms`, + `[harness] codex tailer: rollout identity ${discovery.ambiguous ? "ambiguous" : "unavailable"} for session ${harnessSessionId} within ${CODEX_ROLLOUT_DISCOVERY_TIMEOUT_MS}ms`, ); return; } @@ -4259,9 +4280,10 @@ export const startServer = async ( console.error("[harness] codex tailer parse error:", err), }); codexTailers.set(harnessSessionId, tailer); + sessionManager.setAdapterIdentityState(harnessSessionId, runtimeEpoch, "ready"); } - sessionManager.onStatusChange((session) => { + sessionManager.onStatusChange((session, context) => { // The codex tailer is only needed for harnesses whose analytics come // from the rollout file (eventSource: "transcript-tail"). Harnesses with // eventSource: "hooks" (claude-code) drive the same pipeline via real @@ -4271,9 +4293,17 @@ export const startServer = async ( if (adapters[session.harness]?.eventSource !== "transcript-tail") return; if (session.status === "running") { startCodexTailerFor(session.id).catch((err: unknown) => { + if (context.runtimeEpoch) + sessionManager.setAdapterIdentityState( + session.id, + context.runtimeEpoch, + "unavailable", + ); console.error("[harness] codex tailer startup failed:", err); }); } else if (session.status === "exited") { + if (context.runtimeEpoch) + codexRolloutBroker.release(session.id, context.runtimeEpoch); const tailer = codexTailers.get(session.id); if (tailer) { tailer.emitSessionEnd( From 1f9fd2f1ae84c2450aa02a0cbe705dac4b7f60b6 Mon Sep 17 00:00:00 2001 From: Yash Date: Fri, 4 Sep 2026 19:32:59 +0000 Subject: [PATCH 08/19] test(harness): complete writable subsession acceptance Refs: SAP-3151 --- .changeset/writable-project-subsessions.md | 21 ++ packages/harness/docs/shared-build-plan.md | 33 +++ .../src/core/subsession-coordinator.test.ts | 209 +++++++++++++++++- .../harness/src/profiles/project-agent.ts | 2 +- .../web/e2e/subsession-delegation.spec.ts | 69 ++++++ 5 files changed, 327 insertions(+), 7 deletions(-) create mode 100644 .changeset/writable-project-subsessions.md create mode 100644 packages/harness/web/e2e/subsession-delegation.spec.ts diff --git a/.changeset/writable-project-subsessions.md b/.changeset/writable-project-subsessions.md new file mode 100644 index 00000000..a66dee7c --- /dev/null +++ b/.changeset/writable-project-subsessions.md @@ -0,0 +1,21 @@ +--- +"@sapiom/harness": minor +--- + +Add capability-scoped `project_subsession_delegate` support for creating or +reusing bounded batches of ordinary writable project sessions. Delegations use +durable parent/key bindings, canonical request digests, transactional spawn and +kickoff claims, exact focused-context references, readiness-gated delivery, +restart recovery, nested common-tool composition, and real session IDs. + +**Breaking for embedders** (minor while `@sapiom/harness` is pre-1.0): internal +session hosts that construct the Agent Map MCP router must provide the shared +`SubsessionCoordinator`; session hosts that tail transcript-backed adapters must +also complete exact runtime identity correlation before trusted background +kickoff. No REST delegation endpoint or model-controlled project/session +selector is added. + +Manual sessions remain outside coordinator ownership. Consumers should treat +`uncertain` kickoff delivery as terminal until an exact persisted +acknowledgement arrives, and should use a new request/delegation key when the +corresponding canonical content changes. diff --git a/packages/harness/docs/shared-build-plan.md b/packages/harness/docs/shared-build-plan.md index 8d9f1cd6..e237e641 100644 --- a/packages/harness/docs/shared-build-plan.md +++ b/packages/harness/docs/shared-build-plan.md @@ -91,3 +91,36 @@ and passing the branded `projection` through `TrustedSessionCreateOptions` or project-agent identity. Ordinary callers cannot construct the branded value, and authored data must never be appended to a prompt by another serialization path. + +## Writable project subsessions + +Every ordinary project session discovers `project_subsession_delegate` beside +the shared map, plan, and brief tools. The operation creates or reuses one to +sixteen ordinary writable sessions. Each child receives the same common +project-agent prompt, coding capabilities, project tools, and delegation tool, +so nested delegation follows the same path. An exact assignment, map node, or +brief may focus the child, but focus never changes its tools or authority. + +Callers provide both a request key and a delegation key. Identity is scoped by +the private session capability to the trusted project and parent session. +Identical retries converge on the same durable binding and real Harness session +ID; changing canonical request or binding content under an existing key fails +explicitly. All binding IDs and session IDs for a bounded batch are reserved in +one durable transaction before the first process is spawned. + +The coordinator waits for canonical adapter readiness and exact transcript +identity, then uses fenced spawn and delivery epochs to submit one kickoff. +Delivery states distinguish pending, claimed, submitted without acknowledgement, +acknowledged, and uncertain. An uncertain delivery is never resent blindly. +Exact focused references are checked before delivery, and stale context returns +an explicit refresh path without closing the session or changing writability. + +Coordinator recovery starts from its own two-sided private binding marker. It +does not infer ownership from cwd, title, assignment, map membership, or process +similarity, and it never adopts, renames, resumes, closes, or removes an +unrelated manual session. Tabs remain projections of ordinary live sessions, +deduplicated by the real session ID and exact server-derived project identity. + +Delegation telemetry contains only event names, project/session identifiers, +and bounded error codes. Task text, kickoff context, focused prose, source, +paths, secrets, credentials, and raw adapter output are excluded. diff --git a/packages/harness/src/core/subsession-coordinator.test.ts b/packages/harness/src/core/subsession-coordinator.test.ts index 0605c7ec..5bdb60a3 100644 --- a/packages/harness/src/core/subsession-coordinator.test.ts +++ b/packages/harness/src/core/subsession-coordinator.test.ts @@ -4,12 +4,29 @@ import * as path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import type { ProjectAgentSession } from "../shared/agent-map.js"; +import type { AgentMapGraph, AgentMapVersion, AgentMapVersionId, PlanNodeId } from "../shared/agent-map.js"; +import { + computeAgentMapVersionRecordDigest, + computeGraphContentDigest, +} from "../shared/agent-map-canonical.js"; +import type { + BuildPlanAssignmentIntent, + ProjectBuildPlanId, + ProjectBuildPlanVersion, + ProjectBuildPlanVersionId, +} from "../shared/build-plan.js"; import type { AnalyticsEvent, HarnessAdapter, SpawnSpec, } from "../shared/types.js"; import type { BuildPlanStore } from "./build-plan-store.js"; +import { + computeBuildPlanRecordDigest, + computeBuildPlanSemanticDigest, +} from "./build-plan-canonicalization.js"; +import { compileCanonicalWorkstreamBriefs } from "./agent-brief-compiler.js"; +import { createEmptyProjectPlanningAggregate } from "./agent-map-aggregate-migration.js"; import type { EventReader } from "./collector/store.js"; import { IngestCredentialRegistry } from "./ingest-credentials.js"; import { SessionManager, type PtySpawnFn } from "./session-manager.js"; @@ -26,6 +43,73 @@ const identity: ProjectAgentSession = { userId: "user-1", sessionId: parentId, }; +const plannedAgentId = "node_018f0000-0000-7000-8000-000000000010" as PlanNodeId; +const assignmentId = "work_018f0000-0000-7000-8000-000000000020" as BuildPlanAssignmentIntent["id"]; + +function focusedPlanningAggregate() { + const graph: AgentMapGraph = { + nodes: [{ id: plannedAgentId, kind: "agent", name: "Research", purpose: "Rank stocks", + ownerAgentId: null, contractRefs: ["ResearchReport"] }], + relationships: [], + }; + const contentDigest = computeGraphContentDigest(graph); + const mapBase = { + schemaVersion: 1 as const, projectId, + versionId: "mapv_018f0000-0000-7000-8000-000000000001" as AgentMapVersionId, + version: 1, parentVersionId: null, changeKind: "created" as const, + restoredFromVersionId: null, graph, contentDigest, + authoredBy: { userId: "user-1", sessionId: parentId }, + createdAt: "2026-09-04T00:00:00.000Z", + origin: { kind: "request" as const, requestDigest: `sha256:${"1".repeat(64)}`, + operationIds: [], touchKeys: [] }, + }; + const map: AgentMapVersion = { ...mapBase, recordDigest: computeAgentMapVersionRecordDigest(mapBase) }; + const content = { + outcome: "Publish ranked stocks", nonGoals: [], milestones: [], sequenceGates: [], + sharedConstraints: [], repositoryIntents: [], integrationCriteria: [], acceptanceCriteria: [], + decisions: [], unresolvedDecisions: [], risks: [], + assignments: [{ id: assignmentId, plannedAgentId, briefId: null, + mission: "Rank ten stocks", scope: ["Research"], nonGoals: [], dependencies: [] }], + }; + const semanticDigest = computeBuildPlanSemanticDigest(content); + const planBase = { + schemaVersion: 1 as const, projectId, + planId: "plan_018f0000-0000-7000-8000-000000000001" as ProjectBuildPlanId, + versionId: "planv_018f0000-0000-7000-8000-000000000001" as ProjectBuildPlanVersionId, + version: 1, parentVersionId: null, changeKind: "created" as const, + restoredFromVersionId: null, + map: { projectId, versionId: map.versionId, contentDigest: map.contentDigest }, + content, semanticDigest, authoredBy: { userId: "user-1", sessionId: parentId }, + createdAt: "2026-09-04T00:00:01.000Z", + origin: { kind: "request" as const, requestDigest: `sha256:${"2".repeat(64)}`, + operationIds: [], touchKeys: [] }, + }; + const plan: ProjectBuildPlanVersion = { + ...planBase, + recordDigest: computeBuildPlanRecordDigest(planBase), + }; + const brief = compileCanonicalWorkstreamBriefs({ + projectId, map, plan, mapHistory: [map], planHistory: [plan], previousBriefs: [], + }).briefs[0]!.brief; + const aggregate = createEmptyProjectPlanningAggregate(projectId, "2026-09-04T00:00:00.000Z"); + aggregate.mapVersions.push(map); + aggregate.buildPlanVersions.push(plan); + aggregate.briefVersionsById[brief.briefId] = [brief]; + aggregate.current.map = { + projectId, versionId: map.versionId, contentDigest: map.contentDigest, + }; + aggregate.current.buildPlan = { + projectId, planId: plan.planId, versionId: plan.versionId, + semanticDigest: plan.semanticDigest, + }; + aggregate.current.briefsByScope[brief.scopeKey] = { + scopeKey: brief.scopeKey, focusScope: brief.focusScope, briefId: brief.briefId, + status: "active", + version: { projectId, briefId: brief.briefId, versionId: brief.versionId, + semanticDigest: brief.semanticDigest }, + }; + return { aggregate, brief }; +} function adapter( resumable = false, @@ -93,6 +177,7 @@ describe("SubsessionCoordinator", () => { const root = await fs.mkdtemp(path.join(os.tmpdir(), "subsession-service-")); roots.push(root); const spawned: ReturnType[] = []; + const launchContexts: Array[0]["buildLaunchOpts"]>>[2]> = []; const spawnPty = vi.fn(() => { const spawnedPty = fakePty(); spawned.push(spawnedPty); @@ -109,6 +194,10 @@ describe("SubsessionCoordinator", () => { ingestCredentials: new IngestCredentialRegistry(), sessionsPath: path.join(root, "sessions.json"), spawnPty, + buildLaunchOpts: (_sessionId, _request, context) => { + launchContexts.push(context); + return {}; + }, resolveAgentMapIdentity: async (_sessionId, _cwd, persisted) => persisted, }); managers.push(manager); @@ -160,27 +249,34 @@ describe("SubsessionCoordinator", () => { const store = new SubsessionCoordinatorStore( path.join(root, "agent-map"), ); + const telemetry: unknown[] = []; const planningStore = { read: vi.fn(async () => { throw new Error("no focused context expected"); }), } as unknown as BuildPlanStore; - const coordinator = new SubsessionCoordinator({ - store, - sessionManager: manager, - planningStore, - eventReader, + const newCoordinator = (ownerId?: string) => new SubsessionCoordinator({ + store, sessionManager: manager, planningStore, eventReader, readinessTimeoutMs: 500, + onEvent: (event) => { + telemetry.push(event); + }, + ...(ownerId ? { ownerId } : {}), }); + const coordinator = newCoordinator(); return { root, manager, caller, store, coordinator, + newCoordinator, + planningStore, events, spawnPty, spawned, + launchContexts, + telemetry, unsubscribe, }; } @@ -201,7 +297,7 @@ describe("SubsessionCoordinator", () => { } as const; it("creates one ordinary writable child and reuses it on retry", async () => { - const { coordinator, caller, manager, spawnPty, unsubscribe } = + const { coordinator, caller, manager, spawnPty, telemetry, unsubscribe } = await fixture(); const first = await coordinator.execute(caller, request); const replay = await coordinator.execute(caller, request); @@ -225,6 +321,32 @@ describe("SubsessionCoordinator", () => { userId: caller.userId, sessionId: first.results[0]!.sessionId, }); + expect(telemetry).toEqual(expect.arrayContaining([ + expect.objectContaining({ name: "subsession.requested", projectId }), + expect.objectContaining({ name: "subsession.created", projectId }), + expect.objectContaining({ name: "subsession.ready", projectId }), + expect.objectContaining({ name: "subsession.kickoff_submitted", projectId }), + ])); + expect(JSON.stringify(telemetry)).not.toContain("Implement the research slice"); + expect(JSON.stringify(telemetry)).not.toContain("Run the focused tests"); + }); + + it("converges independent coordinator instances on one child process", async () => { + const { coordinator, newCoordinator, caller, manager, spawnPty, unsubscribe } = + await fixture(); + const other = newCoordinator("other-coordinator"); + const [first, second] = await Promise.all([ + coordinator.execute(caller, request), + other.execute(caller, request), + ]); + unsubscribe(); + + expect(first.results[0]!.sessionId).toBe(second.results[0]!.sessionId); + expect([first.results[0]!.outcome, second.results[0]!.outcome]).toEqual( + expect.arrayContaining(["created", "already-running"]), + ); + expect(manager.list()).toHaveLength(2); + expect(spawnPty).toHaveBeenCalledTimes(2); }); it("acknowledges only the exact persisted kickoff marker", async () => { @@ -335,4 +457,79 @@ describe("SubsessionCoordinator", () => { }); expect(spawned[1]!.writes).toEqual([]); }); + + it("delivers one exact brief overlay and surfaces later staleness without restricting the child", async () => { + const { aggregate, brief } = focusedPlanningAggregate(); + const { + coordinator, + caller, + manager, + planningStore, + spawned, + launchContexts, + unsubscribe, + } = await fixture(); + vi.mocked(planningStore.read).mockResolvedValue(aggregate); + const focusedRequest = { + schemaVersion: 1, + requestKey: "focused-request", + operation: { + kind: "delegate", + delegations: [{ + delegationKey: "focused-research", + outcome: "Implement the focused research slice", + focus: { + kind: "brief", + brief: { + projectId, + briefId: brief.briefId, + versionId: brief.versionId, + semanticDigest: brief.semanticDigest, + }, + }, + }], + }, + } as const; + + const first = await coordinator.execute(caller, focusedRequest); + const replay = await coordinator.execute(caller, focusedRequest); + const childId = first.results[0]!.sessionId!; + const kickoffWrites = spawned[1]!.writes.filter((value) => + value.includes("sapiom-project-delegation"), + ); + expect(first.results[0]).toMatchObject({ + outcome: "created", + contextState: "current", + sessionState: "ready", + }); + expect(replay.results[0]).toMatchObject({ + outcome: "reused", + sessionId: childId, + }); + expect(kickoffWrites).toHaveLength(1); + expect(launchContexts[1]?.focusedContext).toContain("focused-project-context"); + expect(launchContexts[1]?.focusedContext).toContain(brief.versionId); + expect(kickoffWrites[0]).not.toContain("focused-project-context"); + + aggregate.current.briefsByScope[brief.scopeKey] = { + ...aggregate.current.briefsByScope[brief.scopeKey]!, + status: "retired", + }; + const stale = await coordinator.execute(caller, focusedRequest); + unsubscribe(); + + expect(stale.results[0]).toMatchObject({ + outcome: "failed", + sessionId: childId, + contextState: "stale", + error: { code: "context_stale", recovery: "refresh_context" }, + }); + expect(manager.get(childId)).toMatchObject({ + status: "running", + agentMapIdentity: { projectId, sessionId: childId }, + }); + expect(spawned[1]!.writes.filter((value) => + value.includes("sapiom-project-delegation"), + )).toHaveLength(1); + }); }); diff --git a/packages/harness/src/profiles/project-agent.ts b/packages/harness/src/profiles/project-agent.ts index 830d7197..8eefa94f 100644 --- a/packages/harness/src/profiles/project-agent.ts +++ b/packages/harness/src/profiles/project-agent.ts @@ -12,7 +12,7 @@ Use agent_map_read when the current project architecture is relevant. When the w Keep internal implementation details local: library choices, ordinary implementation steps, incidental model or tool calls, and refactors that do not change a meaningful project boundary do not belong in the Agent Map. Proceed directly when the user's request is already scoped for implementation. -Focused assignments, map-node references, bootstrap context, and future briefs are context only. They never grant or remove authority. Delegate focused work when decomposition improves delivery, and never relabel, close, or otherwise reconcile unrelated user-created sessions. +Focused assignments, map-node references, bootstrap context, and focused briefs are context only. They never grant or remove authority. Use project_subsession_delegate when decomposition improves delivery, and never relabel, close, or otherwise reconcile unrelated user-created sessions. `; /** diff --git a/packages/harness/web/e2e/subsession-delegation.spec.ts b/packages/harness/web/e2e/subsession-delegation.spec.ts new file mode 100644 index 00000000..89d909bf --- /dev/null +++ b/packages/harness/web/e2e/subsession-delegation.spec.ts @@ -0,0 +1,69 @@ +import { expect, test } from "@playwright/test"; + +test("delegation retries project one ordinary tab per real session without touching manual tabs", async ({ + page, +}) => { + await page.goto("/?seed=0&mockStudioProjects=present"); + await expect(page.locator(".rail-workflows")).toBeVisible(); + await page.getByTestId("project-select-acme-app").click(); + await expect(page.getByTestId("agent-map-frame")).toBeVisible(); + const projectId = "project_00000000-0000-4000-8000-000000000001"; + const tabs = page.getByRole("tablist", { name: "Sessions" }).getByRole("tab"); + const manualTabCount = await tabs.count(); + + await page.evaluate((selectedProjectId) => { + const publish = ( + window as unknown as { + __HARNESS_TEST__?: { + publish?: (message: Record) => void; + }; + } + ).__HARNESS_TEST__?.publish; + const session = (id: string, title: string) => ({ + id, + agentSessionId: null, + harness: "claude-code" as const, + cwd: "/Users/demo/acme-app", + boundWorkflowPath: null, + title, + status: "running" as const, + exitCode: null, + ready: true, + createdAt: "2026-09-04T10:00:00.000Z", + lastActiveAt: "2026-09-04T10:00:00.000Z", + agentMapIdentity: { + projectId: selectedProjectId, + userId: "user_mock", + sessionId: id, + }, + }); + publish?.({ + type: "session.status", + session: session("sess-delegated", "Focused research"), + }); + // A coordinator replay/status refresh carries the same real Harness ID. + publish?.({ + type: "session.status", + session: session("sess-delegated", "Focused research"), + }); + }, projectId); + + await expect(tabs).toHaveCount(manualTabCount + 1); + await expect(page.getByTestId("session-tab-sess-delegated")).toHaveCount(1); + await expect(page.getByTestId("session-tab-sess-boot")).toHaveCount(1); + + await page + .getByTestId("session-tab-sess-delegated") + .getByRole("tab") + .click(); + await expect(page.getByTestId("session-context")).toHaveAttribute( + "data-session-id", + "sess-delegated", + ); + await expect(page.getByTestId("agent-map-frame")).toHaveCount(0); + await expect(page.locator(".harness-terminal")).toBeVisible(); + + await page.getByTestId("project-select-acme-app").click(); + await expect(page.getByTestId("agent-map-frame")).toBeVisible(); + await expect(page.getByTestId("session-tab-sess-boot")).toHaveCount(1); +}); From 0cd41b91363401f9810ccff2141507c692683d1f Mon Sep 17 00:00:00 2001 From: Yash Date: Fri, 4 Sep 2026 20:01:15 +0000 Subject: [PATCH 09/19] fix(harness): bound subsession recovery state Refs: SAP-3151 --- .changeset/writable-project-subsessions.md | 5 +- packages/harness/docs/shared-build-plan.md | 6 + .../harness/src/core/session-manager.test.ts | 41 ++- packages/harness/src/core/session-manager.ts | 30 +- .../core/subsession-coordinator-store.test.ts | 116 +++++++ .../src/core/subsession-coordinator-store.ts | 297 ++++++++++++++++-- .../src/core/subsession-coordinator.test.ts | 141 ++++++++- .../src/core/subsession-coordinator.ts | 53 +++- packages/harness/src/server/index.ts | 14 + .../src/shared/subsession-delegation.ts | 5 +- 10 files changed, 666 insertions(+), 42 deletions(-) diff --git a/.changeset/writable-project-subsessions.md b/.changeset/writable-project-subsessions.md index a66dee7c..e2453e4a 100644 --- a/.changeset/writable-project-subsessions.md +++ b/.changeset/writable-project-subsessions.md @@ -18,4 +18,7 @@ selector is added. Manual sessions remain outside coordinator ownership. Consumers should treat `uncertain` kickoff delivery as terminal until an exact persisted acknowledgement arrives, and should use a new request/delegation key when the -corresponding canonical content changes. +corresponding canonical content changes. Nested delegation is bounded to four +levels and 64 concurrently live coordinator-owned sessions per project. Closing +a delegated tab starts PTY termination before its private user-close tombstone +is persisted, so a storage error cannot leave the process running. diff --git a/packages/harness/docs/shared-build-plan.md b/packages/harness/docs/shared-build-plan.md index e237e641..e54090e0 100644 --- a/packages/harness/docs/shared-build-plan.md +++ b/packages/harness/docs/shared-build-plan.md @@ -100,6 +100,8 @@ sixteen ordinary writable sessions. Each child receives the same common project-agent prompt, coding capabilities, project tools, and delegation tool, so nested delegation follows the same path. An exact assignment, map node, or brief may focus the child, but focus never changes its tools or authority. +Delegation is bounded to four levels and 64 concurrently live +coordinator-owned sessions per project. Callers provide both a request key and a delegation key. Identity is scoped by the private session capability to the trusted project and parent session. @@ -107,6 +109,10 @@ Identical retries converge on the same durable binding and real Harness session ID; changing canonical request or binding content under an existing key fails explicitly. All binding IDs and session IDs for a bounded batch are reserved in one durable transaction before the first process is spawned. +Older request receipts compact into permanent key tombstones, and closed +bindings compact into ownership tombstones once no retained receipt references +them. Exhausted permanent history is a terminal capacity condition rather than +a retryable request-size error. The coordinator waits for canonical adapter readiness and exact transcript identity, then uses fenced spawn and delivery epochs to submit one kickoff. diff --git a/packages/harness/src/core/session-manager.test.ts b/packages/harness/src/core/session-manager.test.ts index 9fc75d54..17a019da 100644 --- a/packages/harness/src/core/session-manager.test.ts +++ b/packages/harness/src/core/session-manager.test.ts @@ -135,6 +135,7 @@ describe("SessionManager", () => { onProjectAgentIdentityMigration?: SessionManagerOptions["onProjectAgentIdentityMigration"]; onProjectBootstrapSession?: SessionManagerOptions["onProjectBootstrapSession"]; onRuntimeEpochTransition?: SessionManagerOptions["onRuntimeEpochTransition"]; + onSubsessionUserClosed?: SessionManagerOptions["onSubsessionUserClosed"]; writeWorkspaceContext?: SessionManagerOptions["writeWorkspaceContext"]; prepareWorkspaceContext?: SessionManagerOptions["prepareWorkspaceContext"]; ensureCanvasTemplate?: SessionManagerOptions["ensureCanvasTemplate"]; @@ -177,6 +178,7 @@ describe("SessionManager", () => { onProjectAgentIdentityMigration: opts.onProjectAgentIdentityMigration, onProjectBootstrapSession: opts.onProjectBootstrapSession, onRuntimeEpochTransition: opts.onRuntimeEpochTransition, + onSubsessionUserClosed: opts.onSubsessionUserClosed, writeWorkspaceContext: opts.writeWorkspaceContext, prepareWorkspaceContext: opts.prepareWorkspaceContext, ensureCanvasTemplate: opts.ensureCanvasTemplate, @@ -402,9 +404,10 @@ describe("SessionManager", () => { marker(sessionId), input.trusted, ); - await manager.close(sessionId); - expect(manager.wasSubsessionClosedByUser(marker(sessionId))).toBe(true); + const closing = manager.close(sessionId); spawns[0]!.emitExit(0); + await closing; + expect(manager.wasSubsessionClosedByUser(marker(sessionId))).toBe(true); await manager.flush(); await expect( manager.restartFreshBound( @@ -422,6 +425,40 @@ describe("SessionManager", () => { ).toMatchObject({ closedSessionIds: [sessionId] }); }); + it("terminates a delegated PTY even when its user-close tombstone cannot persist", async () => { + let failCloseWrite = false; + const writeSubsessionBindingRegistry = vi.fn(async () => { + if (failCloseWrite) throw new Error("injected close persistence failure"); + }); + const onSubsessionUserClosed = vi.fn(async () => {}); + const { manager, spawns } = makeManager({ + writeSubsessionBindingRegistry, + onSubsessionUserClosed, + }); + const sessionId = "00000000-0000-4000-8000-000000000115"; + const input = delegatedCreate(sessionId); + await manager.createReserved( + sessionId, + { cwd: input.cwd, harness: input.harness }, + marker(sessionId), + input.trusted, + ); + failCloseWrite = true; + + const closing = manager.close(sessionId); + expect(spawns[0]!.pty.kill).toHaveBeenCalledTimes(1); + spawns[0]!.emitExit(0); + await expect(closing).rejects.toThrow("injected close persistence failure"); + expect(manager.get(sessionId)).toMatchObject({ status: "exited" }); + expect(manager.wasSubsessionClosedByUser(marker(sessionId))).toBe(true); + expect(onSubsessionUserClosed).toHaveBeenCalledWith(marker(sessionId)); + + failCloseWrite = false; + await expect(manager.close(sessionId)).resolves.toBe(false); + expect(writeSubsessionBindingRegistry).toHaveBeenCalledTimes(3); + expect(onSubsessionUserClosed).toHaveBeenCalledTimes(2); + }); + it("reports exact input write phases and kills only an exact runtime", async () => { const { manager, spawns } = makeManager(); const session = await manager.create({ diff --git a/packages/harness/src/core/session-manager.ts b/packages/harness/src/core/session-manager.ts index 0c35c3d1..2cf90faf 100644 --- a/packages/harness/src/core/session-manager.ts +++ b/packages/harness/src/core/session-manager.ts @@ -728,6 +728,10 @@ export interface SessionManagerOptions { session: HarnessSession, runtimeEpoch: string | null, ) => Promise | void; + /** Mirrors an explicit user close into the coordinator-owned aggregate. */ + onSubsessionUserClosed?: ( + marker: TrustedSubsessionBindingMarker, + ) => Promise | void; /** Revokes launch capabilities/transports after every exit path. */ onAgentMapSessionExit?: (sessionId: string) => void | Promise; now?: () => string; @@ -928,6 +932,7 @@ export class SessionManager { private readonly onProjectAgentIdentityMigration: SessionManagerOptions["onProjectAgentIdentityMigration"]; private readonly onProjectBootstrapSession: SessionManagerOptions["onProjectBootstrapSession"]; private readonly onRuntimeEpochTransition: SessionManagerOptions["onRuntimeEpochTransition"]; + private readonly onSubsessionUserClosed: SessionManagerOptions["onSubsessionUserClosed"]; private readonly now: () => string; private readonly generateId: () => string; private readonly writeSessionRegistry: @@ -1038,6 +1043,7 @@ export class SessionManager { options.onProjectAgentIdentityMigration; this.onProjectBootstrapSession = options.onProjectBootstrapSession; this.onRuntimeEpochTransition = options.onRuntimeEpochTransition; + this.onSubsessionUserClosed = options.onSubsessionUserClosed; this.now = options.now ?? (() => new Date().toISOString()); this.generateId = options.generateId ?? randomUUID; this.writeSessionRegistry = options.writeSessionRegistry; @@ -1886,18 +1892,30 @@ export class SessionManager { * Returns true (resolved on actual death) when a pty was signalled. */ async close(id: string): Promise { - if (this.subsessionBindings.has(id) && !this.userClosedSubsessions.has(id)) { + const binding = this.subsessionBindings.get(id); + if (binding) { this.userClosedSubsessions.add(id); + } + // Start termination before persistence so a sidecar fsync failure cannot + // leave a delegated PTY running after the user closes its tab. Keep the + // in-memory tombstone on failure and let a later close retry persistence. + const termination = this.kill(id); + let persistenceError: unknown; + if (binding) { try { await this.persistSubsessionBindings(); } catch (error) { - this.userClosedSubsessions.delete(id); - throw error; + persistenceError = error; + } + try { + await this.onSubsessionUserClosed?.(binding); + } catch (error) { + persistenceError ??= error; } } - const live = this.ptys.has(id); - void this.kill(id).catch(() => {}); - return live; + const killed = await termination; + if (persistenceError !== undefined) throw persistenceError; + return killed; } kill(id: string): Promise { diff --git a/packages/harness/src/core/subsession-coordinator-store.test.ts b/packages/harness/src/core/subsession-coordinator-store.test.ts index c4a99b49..3de31461 100644 --- a/packages/harness/src/core/subsession-coordinator-store.test.ts +++ b/packages/harness/src/core/subsession-coordinator-store.test.ts @@ -424,4 +424,120 @@ describe("SubsessionCoordinatorStore", () => { ).rejects.toMatchObject({ code: "binding_scope_mismatch" }); expect((await store.read(projectId)).bindings[0]).toEqual(binding); }); + + it("bounds nested delegation depth and concurrently live coordinator sessions", async () => { + const root = await fixture(); + const store = new SubsessionCoordinatorStore(root, { + maxDelegationDepth: 2, + liveSessionLimit: 2, + }); + const first = ( + await store.reserveDelegations(identity, delegate(), target) + ).bindings[0]!; + const child: ProjectAgentSession = { + ...identity, + sessionId: first.sessionId, + }; + const second = ( + await store.reserveDelegations( + child, + delegate("nested-1", [ + { delegationKey: "nested", outcome: "Nested task" }, + ]), + target, + ) + ).bindings[0]!; + + expect(first).toMatchObject({ parentBindingId: null, delegationDepth: 1 }); + expect(second).toMatchObject({ + parentBindingId: first.bindingId, + delegationDepth: 2, + }); + await expect( + store.reserveDelegations( + { ...identity, sessionId: second.sessionId }, + delegate("too-deep", [ + { delegationKey: "third", outcome: "Too deep" }, + ]), + target, + ), + ).rejects.toMatchObject({ code: "delegation_depth_exceeded" }); + await expect( + store.reserveDelegations( + identity, + delegate("over-live-limit", [ + { delegationKey: "publisher", outcome: "Publish evidence" }, + ]), + target, + ), + ).rejects.toMatchObject({ code: "live_session_limit_reached" }); + expect((await store.read(projectId)).bindings).toHaveLength(2); + }); + + it("expires receipts into tombstones and reclaims closed bindings without reopening their keys", async () => { + const root = await fixture(); + const store = new SubsessionCoordinatorStore(root, { + receiptRetentionLimit: 1, + }); + const first = ( + await store.reserveDelegations(identity, delegate(), target) + ).bindings[0]!; + await store.closeBinding(identity, first.bindingId, first.sessionId); + await store.reserveDelegations( + identity, + delegate("request-2", [ + { delegationKey: "publisher", outcome: "Publish evidence" }, + ]), + target, + ); + + const aggregate = await store.read(projectId); + expect(aggregate.requestReceipts.map(({ requestKey }) => requestKey)).toEqual([ + "request-2", + ]); + expect(aggregate.requestTombstones).toContainEqual( + expect.objectContaining({ requestKey: "request-1" }), + ); + expect(aggregate.bindings.map(({ delegationKey }) => delegationKey)).toEqual([ + "publisher", + ]); + expect(aggregate.bindingTombstones).toContainEqual( + expect.objectContaining({ + bindingId: first.bindingId, + delegationKey: "research", + sessionId: first.sessionId, + }), + ); + await expect( + store.reserveDelegations(identity, delegate(), target), + ).rejects.toMatchObject({ code: "session_closed" }); + await expect( + store.reserveDelegations( + identity, + delegate("request-3", [ + { delegationKey: "research", outcome: "Collect evidence" }, + ]), + target, + ), + ).rejects.toMatchObject({ code: "session_closed" }); + }); + + it("reports exhausted permanent tombstone history as a terminal quota", async () => { + const root = await fixture(); + const store = new SubsessionCoordinatorStore(root, { + receiptRetentionLimit: 1, + historyTombstoneLimit: 1, + }); + await store.reserveDelegations(identity, delegate("request-1"), target); + await store.reserveDelegations(identity, delegate("request-2"), target); + await expect( + store.reserveDelegations(identity, delegate("request-1"), target), + ).resolves.toMatchObject({ replayed: true }); + const before = await store.read(projectId); + + await expect( + store.reserveDelegations(identity, delegate("request-3"), target), + ).rejects.toMatchObject({ code: "history_quota_exceeded" }); + expect(await store.read(projectId)).toEqual(before); + }); }); diff --git a/packages/harness/src/core/subsession-coordinator-store.ts b/packages/harness/src/core/subsession-coordinator-store.ts index de61676d..c729a7d3 100644 --- a/packages/harness/src/core/subsession-coordinator-store.ts +++ b/packages/harness/src/core/subsession-coordinator-store.ts @@ -17,6 +17,8 @@ import { } from "../shared/subsession-delegation-codec.js"; import { PROJECT_SUBSESSION_CLAIM_TTL_MS, + PROJECT_SUBSESSION_LIVE_SESSION_LIMIT, + PROJECT_SUBSESSION_MAX_DEPTH, PROJECT_SUBSESSION_SCHEMA_VERSION, SUBSESSION_COORDINATOR_STORAGE_SCHEMA_VERSION, type CanonicalDelegationRequestDigest, @@ -34,6 +36,7 @@ import { isStudioProjectId } from "./studio-project-catalog.js"; export const SUBSESSION_COORDINATOR_BINDING_LIMIT = 8_192; export const SUBSESSION_COORDINATOR_RECEIPT_LIMIT = 8_192; +export const SUBSESSION_COORDINATOR_RECEIPT_RETENTION_LIMIT = 1_024; export const SUBSESSION_COORDINATOR_DELIVERY_LIMIT = 64; export type SubsessionCoordinatorStoreErrorCode = @@ -41,6 +44,9 @@ export type SubsessionCoordinatorStoreErrorCode = | "unsupported_schema" | "storage_unavailable" | "capacity_exceeded" + | "history_quota_exceeded" + | "live_session_limit_reached" + | "delegation_depth_exceeded" | "request_key_reused" | "delegation_key_reused" | "binding_not_found" @@ -77,12 +83,24 @@ export type SubsessionCoordinatorRequestReceipt = Readonly<{ export type SubsessionCoordinatorRequestTombstone = SubsessionCoordinatorRequestReceipt; +export type SubsessionCoordinatorBindingTombstone = Readonly<{ + bindingId: SubsessionBindingId; + parentSessionId: string; + parentBindingId: SubsessionBindingId | null; + delegationDepth: number; + delegationKey: string; + bindingDigest: string; + sessionId: string; + closedAt: string; +}>; + export type SubsessionCoordinatorAggregate = Readonly<{ schemaVersion: typeof SUBSESSION_COORDINATOR_STORAGE_SCHEMA_VERSION; recordVersion: number; projectId: StudioProjectId; requestReceipts: readonly SubsessionCoordinatorRequestReceipt[]; requestTombstones: readonly SubsessionCoordinatorRequestTombstone[]; + bindingTombstones: readonly SubsessionCoordinatorBindingTombstone[]; bindings: readonly SubsessionBindingRecord[]; createdAt: string; updatedAt: string; @@ -149,10 +167,11 @@ type MutableBinding = Omit< }; type MutableAggregate = Omit< ShallowMutable, - "requestReceipts" | "requestTombstones" | "bindings" + "requestReceipts" | "requestTombstones" | "bindingTombstones" | "bindings" > & { requestReceipts: SubsessionCoordinatorRequestReceipt[]; requestTombstones: SubsessionCoordinatorRequestTombstone[]; + bindingTombstones: SubsessionCoordinatorBindingTombstone[]; bindings: MutableBinding[]; }; @@ -293,6 +312,8 @@ function parseBinding( "bindingId", "projectId", "parentSessionId", + "parentBindingId", + "delegationDepth", "delegationKey", "bindingDigest", "outcome", @@ -319,6 +340,11 @@ function parseBinding( value.projectId !== projectId || !identifier(value.bindingId, "binding") || !identifier(value.parentSessionId) || + (value.parentBindingId !== null && + !identifier(value.parentBindingId, "binding")) || + !Number.isSafeInteger(value.delegationDepth) || + (value.delegationDepth as number) < 1 || + (value.delegationDepth as number) > PROJECT_SUBSESSION_MAX_DEPTH || !identifier(value.sessionId) || !["claude-code", "codex"].includes(String(value.harness)) || typeof value.projectRoot !== "string" || @@ -485,6 +511,38 @@ function parseReceipt( return structuredClone(value) as unknown as SubsessionCoordinatorRequestReceipt; } +function parseBindingTombstone( + value: unknown, +): SubsessionCoordinatorBindingTombstone { + if ( + !isRecord(value) || + !exact(value, [ + "bindingId", + "parentSessionId", + "parentBindingId", + "delegationDepth", + "delegationKey", + "bindingDigest", + "sessionId", + "closedAt", + ]) || + !identifier(value.bindingId, "binding") || + !identifier(value.parentSessionId) || + (value.parentBindingId !== null && + !identifier(value.parentBindingId, "binding")) || + !Number.isSafeInteger(value.delegationDepth) || + (value.delegationDepth as number) < 1 || + (value.delegationDepth as number) > PROJECT_SUBSESSION_MAX_DEPTH || + !identifier(value.delegationKey) || + !digest(value.bindingDigest) || + !identifier(value.sessionId) || + !timestamp(value.closedAt) + ) { + throw new SubsessionCoordinatorStoreError("malformed_state"); + } + return structuredClone(value) as unknown as SubsessionCoordinatorBindingTombstone; +} + export function parseSubsessionCoordinatorAggregate( value: unknown, expectedProjectId: StudioProjectId, @@ -506,6 +564,7 @@ export function parseSubsessionCoordinatorAggregate( "projectId", "requestReceipts", "requestTombstones", + "bindingTombstones", "bindings", "createdAt", "updatedAt", @@ -518,6 +577,8 @@ export function parseSubsessionCoordinatorAggregate( value.requestReceipts.length > SUBSESSION_COORDINATOR_RECEIPT_LIMIT || !Array.isArray(value.requestTombstones) || value.requestTombstones.length > SUBSESSION_COORDINATOR_RECEIPT_LIMIT || + !Array.isArray(value.bindingTombstones) || + value.bindingTombstones.length > SUBSESSION_COORDINATOR_BINDING_LIMIT || !Array.isArray(value.bindings) || value.bindings.length > SUBSESSION_COORDINATOR_BINDING_LIMIT || !timestamp(value.createdAt) || @@ -530,6 +591,7 @@ export function parseSubsessionCoordinatorAggregate( } const requestReceipts = value.requestReceipts.map(parseReceipt); const requestTombstones = value.requestTombstones.map(parseReceipt); + const bindingTombstones = value.bindingTombstones.map(parseBindingTombstone); const bindings = value.bindings.map((entry) => parseBinding(entry, expectedProjectId), ); @@ -540,27 +602,52 @@ export function parseSubsessionCoordinatorAggregate( ({ parentSessionId, delegationKey }) => `${parentSessionId}\0${delegationKey}`, ); + const allBindings = [...bindings, ...bindingTombstones]; + const allBindingKeys = [ + ...bindingKeys, + ...bindingTombstones.map( + ({ parentSessionId, delegationKey }) => + `${parentSessionId}\0${delegationKey}`, + ), + ]; if ( new Set(requestKeys).size !== requestKeys.length || - new Set(bindingKeys).size !== bindingKeys.length || - new Set(bindings.map(({ bindingId }) => bindingId)).size !== - bindings.length || - new Set(bindings.map(({ sessionId }) => sessionId)).size !== bindings.length + new Set(allBindingKeys).size !== allBindings.length || + new Set(allBindings.map(({ bindingId }) => bindingId)).size !== + allBindings.length || + new Set(allBindings.map(({ sessionId }) => sessionId)).size !== + allBindings.length ) { throw new SubsessionCoordinatorStoreError("malformed_state"); } - const bindingIds = new Set(bindings.map(({ bindingId }) => bindingId)); if ( [...requestReceipts, ...requestTombstones].some(({ bindingIds: ids }) => - ids.some((bindingId) => !bindingIds.has(bindingId)), + ids.some( + (bindingId) => + !allBindings.some((binding) => binding.bindingId === bindingId), + ), ) ) { throw new SubsessionCoordinatorStoreError("malformed_state"); } + const bindingsById = new Map( + allBindings.map((binding) => [binding.bindingId, binding]), + ); + if ( + allBindings.some((binding) => { + if (binding.parentBindingId === null) + return binding.delegationDepth !== 1; + const parent = bindingsById.get(binding.parentBindingId); + return !parent || binding.delegationDepth !== parent.delegationDepth + 1; + }) + ) { + throw new SubsessionCoordinatorStoreError("malformed_state"); + } return { ...structuredClone(value), requestReceipts, requestTombstones, + bindingTombstones, bindings, } as unknown as SubsessionCoordinatorAggregate; } @@ -586,6 +673,10 @@ export class SubsessionCoordinatorStore { generateId?: () => string; generateSessionId?: () => string; claimTtlMs?: number; + receiptRetentionLimit?: number; + historyTombstoneLimit?: number; + liveSessionLimit?: number; + maxDelegationDepth?: number; onEvent?: (event: SubsessionCoordinatorStoreEvent) => void | Promise; beforePersistStep?: ( step: "write" | "file-sync" | "rename" | "directory-sync", @@ -610,6 +701,76 @@ export class SubsessionCoordinatorStore { return (this.options.generateId ?? randomUUID)(); } + private compactTerminalHistory(aggregate: MutableAggregate): void { + const historyLimit = Math.max( + 1, + Math.min( + this.options.historyTombstoneLimit ?? SUBSESSION_COORDINATOR_RECEIPT_LIMIT, + SUBSESSION_COORDINATOR_RECEIPT_LIMIT, + ), + ); + const retention = Math.max( + 1, + Math.min( + this.options.receiptRetentionLimit ?? + SUBSESSION_COORDINATOR_RECEIPT_RETENTION_LIMIT, + SUBSESSION_COORDINATOR_RECEIPT_LIMIT, + ), + ); + const expiring = Math.max(0, aggregate.requestReceipts.length - retention); + if ( + aggregate.requestTombstones.length + expiring > + historyLimit + ) { + throw new SubsessionCoordinatorStoreError("history_quota_exceeded"); + } + for (let count = 0; count < expiring; count += 1) { + const expired = aggregate.requestReceipts.shift(); + if (expired) { + aggregate.requestTombstones.push({ + parentSessionId: expired.parentSessionId, + requestKey: expired.requestKey, + requestDigest: expired.requestDigest, + operation: expired.operation, + bindingIds: expired.bindingIds, + createdAt: expired.createdAt, + }); + } + } + + const referenced = new Set( + aggregate.requestReceipts.flatMap(({ bindingIds }) => bindingIds), + ); + const reclaimable = aggregate.bindings.filter( + (binding) => + binding.sessionState === "closed" && !referenced.has(binding.bindingId), + ); + if ( + aggregate.bindingTombstones.length + reclaimable.length > + historyLimit + ) { + throw new SubsessionCoordinatorStoreError("history_quota_exceeded"); + } + for (const binding of reclaimable) { + aggregate.bindingTombstones.push({ + bindingId: binding.bindingId, + parentSessionId: binding.parentSessionId, + parentBindingId: binding.parentBindingId, + delegationDepth: binding.delegationDepth, + delegationKey: binding.delegationKey, + bindingDigest: binding.bindingDigest, + sessionId: binding.sessionId, + closedAt: binding.updatedAt, + }); + } + if (reclaimable.length > 0) { + const reclaimed = new Set(reclaimable.map(({ bindingId }) => bindingId)); + aggregate.bindings = aggregate.bindings.filter( + ({ bindingId }) => !reclaimed.has(bindingId), + ); + } + } + private emit(event: SubsessionCoordinatorStoreEvent): void { try { void Promise.resolve(this.options.onEvent?.(event)).catch(() => {}); @@ -626,6 +787,7 @@ export class SubsessionCoordinatorStore { projectId, requestReceipts: [], requestTombstones: [], + bindingTombstones: [], bindings: [], createdAt: now, updatedAt: now, @@ -821,6 +983,47 @@ export class SubsessionCoordinatorStore { }); } + closeBinding( + identity: ProjectAgentSession, + bindingId: SubsessionBindingId, + expectedSessionId: string, + ): Promise { + return this.transact(identity.projectId, async (aggregate) => { + const binding = this.scopedBinding(aggregate, identity, bindingId); + if (binding.sessionId !== expectedSessionId) + throw new SubsessionCoordinatorStoreError("binding_scope_mismatch"); + if (binding.sessionState === "closed") return { value: binding }; + const now = this.now(); + binding.sessionState = "closed"; + binding.lifecycleEpoch += 1; + binding.spawnClaim = null; + binding.runtime = null; + binding.updatedAt = now; + this.compactTerminalHistory(aggregate); + aggregate.recordVersion += 1; + aggregate.updatedAt = now; + return { value: binding, next: aggregate }; + }); + } + + /** Server-only bridge from SessionManager's private two-sided marker. */ + closeOwnedBinding(marker: Readonly<{ + projectId: StudioProjectId; + parentSessionId: string; + bindingId: string; + sessionId: string; + }>): Promise { + return this.closeBinding( + { + projectId: marker.projectId, + userId: "studio-subsession-coordinator", + sessionId: marker.parentSessionId, + }, + marker.bindingId as SubsessionBindingId, + marker.sessionId, + ); + } + refreshFocusedContext( identity: ProjectAgentSession, rawRequest: unknown, @@ -832,7 +1035,9 @@ export class SubsessionCoordinatorStore { const targetSelector = operation.target; const requestDigest = computeCanonicalDelegationRequestDigest(request); return this.transact(identity.projectId, async (aggregate) => { - const sameRequest = (receipt: SubsessionCoordinatorRequestReceipt) => + const sameRequest = ( + receipt: Pick, + ) => receipt.parentSessionId === identity.sessionId && receipt.requestKey === request.requestKey; const previous = @@ -849,16 +1054,15 @@ export class SubsessionCoordinatorStore { const binding = aggregate.bindings.find( ({ bindingId }) => bindingId === previous.bindingIds[0], ); + if (!binding && aggregate.bindingTombstones.some( + ({ bindingId }) => bindingId === previous.bindingIds[0], + )) { + throw new SubsessionCoordinatorStoreError("session_closed"); + } if (!binding) throw new SubsessionCoordinatorStoreError("malformed_state"); return { value: { replayed: true, requestDigest, binding } }; } - if ( - aggregate.requestReceipts.length >= - SUBSESSION_COORDINATOR_RECEIPT_LIMIT - ) { - throw new SubsessionCoordinatorStoreError("capacity_exceeded"); - } const target = targetSelector.kind === "self" ? aggregate.bindings.find( @@ -883,7 +1087,7 @@ export class SubsessionCoordinatorStore { if (currentDelivery?.state === "uncertain") throw new SubsessionCoordinatorStoreError("claim_conflict"); if (target.deliveries.length >= SUBSESSION_COORDINATOR_DELIVERY_LIMIT) - throw new SubsessionCoordinatorStoreError("capacity_exceeded"); + throw new SubsessionCoordinatorStoreError("history_quota_exceeded"); const now = this.now(); target.contextEpoch += 1; @@ -914,6 +1118,7 @@ export class SubsessionCoordinatorStore { bindingIds: [target.bindingId], createdAt: now, }); + this.compactTerminalHistory(aggregate); aggregate.recordVersion += 1; aggregate.updatedAt = now; return { @@ -946,7 +1151,7 @@ export class SubsessionCoordinatorStore { const operation = request.operation; return this.transact(identity.projectId, async (aggregate) => { const sameRequest = ( - receipt: SubsessionCoordinatorRequestReceipt, + receipt: Pick, ): boolean => receipt.parentSessionId === identity.sessionId && receipt.requestKey === request.requestKey; @@ -964,6 +1169,11 @@ export class SubsessionCoordinatorStore { const binding = aggregate.bindings.find( (entry) => entry.bindingId === bindingId, ); + if (!binding && aggregate.bindingTombstones.some( + (entry) => entry.bindingId === bindingId, + )) { + throw new SubsessionCoordinatorStoreError("session_closed"); + } if (!binding) throw new SubsessionCoordinatorStoreError("malformed_state"); return binding; @@ -977,16 +1187,28 @@ export class SubsessionCoordinatorStore { value: { replayed: true, requestDigest, bindings }, }; } - if ( - aggregate.requestReceipts.length >= - SUBSESSION_COORDINATOR_RECEIPT_LIMIT - ) { - throw new SubsessionCoordinatorStoreError("capacity_exceeded"); - } - const now = this.now(); const bindings: SubsessionBindingRecord[] = []; let created = 0; + const live = aggregate.bindings.filter(({ sessionState }) => + ["reserved", "spawn-claimed", "starting", "awaiting-ready", "ready"].includes( + sessionState, + ), + ).length; + const parentBinding = aggregate.bindings.find( + ({ sessionId }) => sessionId === identity.sessionId, + ); + const delegationDepth = (parentBinding?.delegationDepth ?? 0) + 1; + if ( + delegationDepth > + Math.min( + this.options.maxDelegationDepth ?? PROJECT_SUBSESSION_MAX_DEPTH, + PROJECT_SUBSESSION_MAX_DEPTH, + ) + ) { + throw new SubsessionCoordinatorStoreError("delegation_depth_exceeded"); + } + let additionalLive = 0; for (const delegation of operation.delegations) { const bindingDigest = computeCanonicalDelegationBindingDigest(delegation); @@ -1000,21 +1222,37 @@ export class SubsessionCoordinatorStore { throw new SubsessionCoordinatorStoreError( "delegation_key_reused", ); + if (existing.sessionState === "closed") + throw new SubsessionCoordinatorStoreError("session_closed"); + if (["exited", "failed"].includes(existing.sessionState)) + additionalLive += 1; bindings.push(existing); continue; } + const terminal = aggregate.bindingTombstones.find( + (entry) => + entry.parentSessionId === identity.sessionId && + entry.delegationKey === delegation.delegationKey, + ); + if (terminal) { + if (terminal.bindingDigest !== bindingDigest) + throw new SubsessionCoordinatorStoreError("delegation_key_reused"); + throw new SubsessionCoordinatorStoreError("session_closed"); + } if ( - aggregate.bindings.length + created >= - SUBSESSION_COORDINATOR_BINDING_LIMIT + aggregate.bindings.length >= SUBSESSION_COORDINATOR_BINDING_LIMIT ) { - throw new SubsessionCoordinatorStoreError("capacity_exceeded"); + throw new SubsessionCoordinatorStoreError("history_quota_exceeded"); } + additionalLive += 1; const contextFocus = delegation.focus ?? null; const contextEpoch = 1; const binding: SubsessionBindingRecord = { bindingId: `binding_${this.id()}` as SubsessionBindingId, projectId: identity.projectId, parentSessionId: identity.sessionId, + parentBindingId: parentBinding?.bindingId ?? null, + delegationDepth, delegationKey: delegation.delegationKey, bindingDigest, outcome: delegation.outcome, @@ -1054,6 +1292,12 @@ export class SubsessionCoordinatorStore { bindings.push(binding); created += 1; } + if ( + live + additionalLive > + (this.options.liveSessionLimit ?? PROJECT_SUBSESSION_LIVE_SESSION_LIMIT) + ) { + throw new SubsessionCoordinatorStoreError("live_session_limit_reached"); + } const receipt: SubsessionCoordinatorRequestReceipt = { parentSessionId: identity.sessionId, requestKey: request.requestKey, @@ -1063,6 +1307,7 @@ export class SubsessionCoordinatorStore { createdAt: now, }; aggregate.requestReceipts.push(receipt); + this.compactTerminalHistory(aggregate); aggregate.recordVersion += 1; aggregate.updatedAt = now; this.emit({ diff --git a/packages/harness/src/core/subsession-coordinator.test.ts b/packages/harness/src/core/subsession-coordinator.test.ts index 5bdb60a3..29f52ed5 100644 --- a/packages/harness/src/core/subsession-coordinator.test.ts +++ b/packages/harness/src/core/subsession-coordinator.test.ts @@ -34,7 +34,10 @@ import { SubsessionCoordinator, SubsessionCoordinatorError, } from "./subsession-coordinator.js"; -import { SubsessionCoordinatorStore } from "./subsession-coordinator-store.js"; +import { + SubsessionCoordinatorStore, + SubsessionCoordinatorStoreError, +} from "./subsession-coordinator-store.js"; const projectId = "project_00000000-0000-4000-8000-000000000001"; const parentId = "parent-session-1"; @@ -183,6 +186,7 @@ describe("SubsessionCoordinator", () => { spawned.push(spawnedPty); return spawnedPty.pty; }); + const closeStore: { current?: SubsessionCoordinatorStore } = {}; const manager = new SessionManager({ adapters: { "claude-code": adapter( @@ -198,6 +202,9 @@ describe("SubsessionCoordinator", () => { launchContexts.push(context); return {}; }, + onSubsessionUserClosed: async (marker) => { + await closeStore.current?.closeOwnedBinding(marker); + }, resolveAgentMapIdentity: async (_sessionId, _cwd, persisted) => persisted, }); managers.push(manager); @@ -231,6 +238,7 @@ describe("SubsessionCoordinator", () => { } }); const events: AnalyticsEvent[] = []; + const recordedTurnSessionIds = new Set(); const eventReader: EventReader = { async *read(filter) { const ids = filter?.harnessSessionId; @@ -244,11 +252,29 @@ describe("SubsessionCoordinator", () => { yield event; } }, - index: async () => ({ bySession: new Map(), byAgentSession: new Map() }), + index: async () => ({ + bySession: new Map( + [...recordedTurnSessionIds].map((sessionId) => [ + sessionId, + { + harnessSessionId: sessionId, + spans: [], + eventCount: 1, + turnCount: 1, + agentSessionIds: [], + harness: "claude-code" as const, + firstTs: null, + lastTs: null, + }, + ]), + ), + byAgentSession: new Map(), + }), }; const store = new SubsessionCoordinatorStore( path.join(root, "agent-map"), ); + closeStore.current = store; const telemetry: unknown[] = []; const planningStore = { read: vi.fn(async () => { @@ -273,6 +299,7 @@ describe("SubsessionCoordinator", () => { newCoordinator, planningStore, events, + recordedTurnSessionIds, spawnPty, spawned, launchContexts, @@ -437,6 +464,67 @@ describe("SubsessionCoordinator", () => { expect(manager.list()).toHaveLength(1); }); + it("preserves bounded codec codes and issues for callers", async () => { + const { coordinator, caller, unsubscribe } = await fixture(); + await expect( + coordinator.execute(caller, { + schemaVersion: 2, + requestKey: "unsupported", + operation: { kind: "delegate", delegations: [] }, + }), + ).rejects.toMatchObject({ + detail: { + code: "unsupported_schema", + retryable: false, + recovery: "correct", + issues: [{ path: "schemaVersion", code: "unsupported_schema" }], + }, + }); + await expect( + coordinator.execute(caller, { + schemaVersion: 1, + requestKey: "utf8-overflow", + operation: { + kind: "delegate", + delegations: [ + { + delegationKey: "research", + outcome: "界".repeat(2_000), + }, + ], + }, + }), + ).rejects.toMatchObject({ + detail: { + code: "invalid_request", + retryable: false, + recovery: "correct", + issues: [ + { + path: "operation.delegations[0]", + code: "invalid_delegation", + }, + ], + }, + }); + unsubscribe(); + }); + + it("does not advise retry or request reduction for permanent history exhaustion", async () => { + const { coordinator, caller, store, unsubscribe } = await fixture(); + vi.spyOn(store, "reserveDelegations").mockRejectedValueOnce( + new SubsessionCoordinatorStoreError("history_quota_exceeded"), + ); + await expect(coordinator.execute(caller, request)).rejects.toMatchObject({ + detail: { + code: "capacity_exceeded", + retryable: false, + recovery: "none", + }, + }); + unsubscribe(); + }); + it("writes no kickoff when adapter identity correlation is ambiguous", async () => { const { coordinator, caller, spawned, unsubscribe } = await fixture( false, @@ -458,6 +546,55 @@ describe("SubsessionCoordinator", () => { expect(spawned[1]!.writes).toEqual([]); }); + it("reports recorded-turn fresh restart rejection as terminal", async () => { + const { + coordinator, + caller, + manager, + recordedTurnSessionIds, + spawned, + unsubscribe, + } = await fixture(false, "ambiguous"); + const first = await coordinator.execute(caller, request); + const childId = first.results[0]!.sessionId!; + recordedTurnSessionIds.add(childId); + spawned[1]!.emitExit(1); + await manager.flush(); + + const retried = await coordinator.execute(caller, request); + unsubscribe(); + + expect(retried.results[0]).toMatchObject({ + outcome: "failed", + sessionId: childId, + kickoffState: "pending", + error: { + code: "session_restart_failed", + retryable: false, + recovery: "inspect_session", + }, + }); + }); + + it("removes a user-closed child from live coordinator capacity", async () => { + const { coordinator, caller, manager, store, spawned, unsubscribe } = + await fixture(); + const created = await coordinator.execute(caller, request); + const childId = created.results[0]!.sessionId!; + + const closing = manager.close(childId); + spawned[1]!.emitExit(0); + await closing; + const aggregate = await store.read(projectId); + unsubscribe(); + + expect(aggregate.bindings[0]).toMatchObject({ + sessionId: childId, + sessionState: "closed", + runtime: null, + }); + }); + it("delivers one exact brief overlay and surfaces later staleness without restricting the child", async () => { const { aggregate, brief } = focusedPlanningAggregate(); const { diff --git a/packages/harness/src/core/subsession-coordinator.ts b/packages/harness/src/core/subsession-coordinator.ts index d7e23679..e3a99bae 100644 --- a/packages/harness/src/core/subsession-coordinator.ts +++ b/packages/harness/src/core/subsession-coordinator.ts @@ -12,6 +12,7 @@ import { } from "../shared/build-plan.js"; import { parseProjectSubsessionRequest, + SubsessionDelegationValidationError, } from "../shared/subsession-delegation-codec.js"; import type { DelegationError, @@ -30,7 +31,11 @@ import { serializeFocusedSessionContext, type FocusedSessionContextProjection, } from "./focused-session-context.js"; -import { SessionNotReadyError, SubsessionBindingMismatchError } from "./errors.js"; +import { + SessionNotReadyError, + SubsessionBindingMismatchError, + SubsessionFreshRestartForbiddenError, +} from "./errors.js"; import type { SessionManager, TrustedSubsessionBindingMarker, @@ -103,7 +108,13 @@ const error = ( code: DelegationError["code"], retryable: boolean, recovery: DelegationError["recovery"], -): DelegationError => ({ code, retryable, recovery }); + issues?: DelegationError["issues"], +): DelegationError => ({ + code, + retryable, + recovery, + ...(issues === undefined ? {} : { issues }), +}); const currentDelivery = (binding: SubsessionBindingRecord) => binding.deliveries.find( @@ -165,7 +176,17 @@ export class SubsessionCoordinator { let request: ProjectSubsessionRequest; try { request = parseProjectSubsessionRequest(rawRequest, identity.projectId); - } catch { + } catch (cause) { + if (cause instanceof SubsessionDelegationValidationError) { + throw new SubsessionCoordinatorError( + error( + cause.code, + false, + cause.code === "capacity_exceeded" ? "reduce_request" : "correct", + cause.issues, + ), + ); + } throw new SubsessionCoordinatorError( error("invalid_request", false, "correct"), ); @@ -515,8 +536,14 @@ export class SubsessionCoordinator { binding = await this.advanceToReady(identity, binding, runtimeToken); return { binding, created: false, reused: true }; } - if (this.options.sessionManager.wasSubsessionClosedByUser(expected)) + if (this.options.sessionManager.wasSubsessionClosedByUser(expected)) { + await this.options.store.closeBinding( + identity, + binding.bindingId, + binding.sessionId, + ); throw error("session_closed", false, "inspect_session"); + } if (this.options.sessionManager.isLive(binding.sessionId)) { const runtimeToken = this.options.sessionManager.getRuntimeEpoch(binding.sessionId)!; @@ -1127,6 +1154,22 @@ export class SubsessionCoordinator { return new SubsessionCoordinatorError( error("capacity_exceeded", false, "reduce_request"), ); + if (cause.code === "live_session_limit_reached") + return new SubsessionCoordinatorError( + error("capacity_exceeded", true, "retry"), + ); + if ( + cause.code === "delegation_depth_exceeded" || + cause.code === "history_quota_exceeded" + ) { + return new SubsessionCoordinatorError( + error("capacity_exceeded", false, "none"), + ); + } + if (cause.code === "session_closed") + return new SubsessionCoordinatorError( + error("session_closed", false, "inspect_session"), + ); if (cause.code === "storage_unavailable") return new SubsessionCoordinatorError( error("storage_unavailable", true, "retry"), @@ -1162,6 +1205,8 @@ export class SubsessionCoordinator { } if (cause instanceof SubsessionBindingMismatchError) return error("binding_session_mismatch", false, "inspect_session"); + if (cause instanceof SubsessionFreshRestartForbiddenError) + return error("session_restart_failed", false, "inspect_session"); if (cause instanceof SessionNotReadyError) return error("readiness_timeout", true, "retry"); if (cause instanceof SubsessionCoordinatorStoreError) { diff --git a/packages/harness/src/server/index.ts b/packages/harness/src/server/index.ts index 2c79cc1e..c0490218 100644 --- a/packages/harness/src/server/index.ts +++ b/packages/harness/src/server/index.ts @@ -1372,6 +1372,14 @@ export const startServer = async ( // any not-yet-scheduled project IDs in memory so a retry can converge on the // same project instead of creating another one after a transient failure. const projectsAwaitingBootstrapSchedule = new Set(); + const closeCoordinatorOwnedSubsession: { + current?: (marker: { + projectId: string; + parentSessionId: string; + bindingId: string; + sessionId: string; + }) => Promise; + } = {}; const scheduleBootstrapProjects = async ( projectIds: Iterable, userId: string, @@ -1404,6 +1412,9 @@ export const startServer = async ( ingestCredentials, collectorUrl: options.collectorUrl, sessionsPath: options.sessionsPath ?? statePaths.sessions, + onSubsessionUserClosed: async (marker) => { + await closeCoordinatorOwnedSubsession.current?.(marker); + }, buildLaunchOpts, resolveAgentMapIdentity: async (sessionId, cwd, persisted) => { const userId = localProjectPrincipal(projectUserId, machineId); @@ -3181,6 +3192,9 @@ export const startServer = async ( statePaths.agentMap, { onEvent: emitSubsessionEvent }, ); + closeCoordinatorOwnedSubsession.current = async (marker) => { + await subsessionCoordinatorStore.closeOwnedBinding(marker); + }; const subsessionCoordinator = new SubsessionCoordinator({ store: subsessionCoordinatorStore, sessionManager, diff --git a/packages/harness/src/shared/subsession-delegation.ts b/packages/harness/src/shared/subsession-delegation.ts index 444f2ce5..edfe8833 100644 --- a/packages/harness/src/shared/subsession-delegation.ts +++ b/packages/harness/src/shared/subsession-delegation.ts @@ -18,6 +18,8 @@ export const PROJECT_SUBSESSION_OUTCOME_BYTES = 4 * 1_024; export const PROJECT_SUBSESSION_KICKOFF_CONTEXT_BYTES = 16 * 1_024; export const PROJECT_SUBSESSION_REQUEST_BYTES = 64 * 1_024; export const PROJECT_SUBSESSION_CLAIM_TTL_MS = 120_000; +export const PROJECT_SUBSESSION_MAX_DEPTH = 4; +export const PROJECT_SUBSESSION_LIVE_SESSION_LIMIT = 64; type Brand = string & { readonly __brand: TBrand }; @@ -199,6 +201,8 @@ export type SubsessionBindingRecord = Readonly<{ bindingId: SubsessionBindingId; projectId: StudioProjectId; parentSessionId: string; + parentBindingId: SubsessionBindingId | null; + delegationDepth: number; delegationKey: string; bindingDigest: CanonicalDelegationBindingDigest; outcome: string; @@ -222,4 +226,3 @@ export type SubsessionBindingRecord = Readonly<{ createdAt: string; updatedAt: string; }>; - From a67fc46a3fd06826ad31e76faed6d17b632d3012 Mon Sep 17 00:00:00 2001 From: Yash Date: Fri, 4 Sep 2026 20:11:45 +0000 Subject: [PATCH 10/19] fix(harness): expire subsession recovery history Refs: SAP-3151 --- .changeset/writable-project-subsessions.md | 4 +- packages/harness/docs/shared-build-plan.md | 10 ++- .../core/subsession-coordinator-store.test.ts | 73 ++++++++++++++++--- .../src/core/subsession-coordinator-store.ts | 54 ++++++++------ .../src/core/subsession-coordinator.test.ts | 17 ++++- .../src/core/subsession-coordinator.ts | 5 +- 6 files changed, 123 insertions(+), 40 deletions(-) diff --git a/.changeset/writable-project-subsessions.md b/.changeset/writable-project-subsessions.md index e2453e4a..95e028eb 100644 --- a/.changeset/writable-project-subsessions.md +++ b/.changeset/writable-project-subsessions.md @@ -21,4 +21,6 @@ acknowledgement arrives, and should use a new request/delegation key when the corresponding canonical content changes. Nested delegation is bounded to four levels and 64 concurrently live coordinator-owned sessions per project. Closing a delegated tab starts PTY termination before its private user-close tombstone -is persisted, so a storage error cannot leave the process running. +is persisted, so a storage error cannot leave the process running. Request, +binding, and acknowledged-delivery history use bounded retention so long-lived +projects do not dead-end on routine delegation or context refreshes. diff --git a/packages/harness/docs/shared-build-plan.md b/packages/harness/docs/shared-build-plan.md index e54090e0..83514830 100644 --- a/packages/harness/docs/shared-build-plan.md +++ b/packages/harness/docs/shared-build-plan.md @@ -109,10 +109,12 @@ Identical retries converge on the same durable binding and real Harness session ID; changing canonical request or binding content under an existing key fails explicitly. All binding IDs and session IDs for a bounded batch are reserved in one durable transaction before the first process is spawned. -Older request receipts compact into permanent key tombstones, and closed -bindings compact into ownership tombstones once no retained receipt references -them. Exhausted permanent history is a terminal capacity condition rather than -a retryable request-size error. +Older request receipts compact into bounded key tombstones, and closed bindings +compact into bounded ownership tombstones once no retained receipt references +them. The oldest tombstones expire as the retention window advances, so routine +delegation and focused-context refreshes cannot permanently exhaust a project. +Proven acknowledged or unsent delivery epochs are likewise pruned when a newer +focused-context delivery replaces them; ambiguous delivery evidence is retained. The coordinator waits for canonical adapter readiness and exact transcript identity, then uses fenced spawn and delivery epochs to submit one kickoff. diff --git a/packages/harness/src/core/subsession-coordinator-store.test.ts b/packages/harness/src/core/subsession-coordinator-store.test.ts index 3de31461..5c8b0809 100644 --- a/packages/harness/src/core/subsession-coordinator-store.test.ts +++ b/packages/harness/src/core/subsession-coordinator-store.test.ts @@ -145,7 +145,8 @@ describe("SubsessionCoordinatorStore", () => { expect(first.replayed).toBe(false); expect(replay.replayed).toBe(true); expect(first.binding.contextEpoch).toBe(2); - expect(first.binding.deliveries).toHaveLength(2); + expect(first.binding.deliveries).toHaveLength(1); + expect(first.binding.deliveries[0]!.contextEpoch).toBe(2); expect(replay.binding).toEqual(first.binding); await expect( store.refreshFocusedContext(identity, { @@ -510,7 +511,7 @@ describe("SubsessionCoordinatorStore", () => { ); await expect( store.reserveDelegations(identity, delegate(), target), - ).rejects.toMatchObject({ code: "session_closed" }); + ).rejects.toMatchObject({ code: "request_key_expired" }); await expect( store.reserveDelegations( identity, @@ -522,22 +523,72 @@ describe("SubsessionCoordinatorStore", () => { ).rejects.toMatchObject({ code: "session_closed" }); }); - it("reports exhausted permanent tombstone history as a terminal quota", async () => { + it("expires oldest key and ownership tombstones instead of dead-ending the project", async () => { const root = await fixture(); const store = new SubsessionCoordinatorStore(root, { receiptRetentionLimit: 1, historyTombstoneLimit: 1, }); - await store.reserveDelegations(identity, delegate("request-1"), target); - await store.reserveDelegations(identity, delegate("request-2"), target); + const first = ( + await store.reserveDelegations(identity, delegate("request-1"), target) + ).bindings[0]!; + await store.closeBinding(identity, first.bindingId, first.sessionId); + const second = ( + await store.reserveDelegations( + identity, + delegate("request-2", [ + { delegationKey: "publisher", outcome: "Publish evidence" }, + ]), + target, + ) + ).bindings[0]!; + await store.closeBinding(identity, second.bindingId, second.sessionId); await expect( store.reserveDelegations(identity, delegate("request-1"), target), - ).resolves.toMatchObject({ replayed: true }); - const before = await store.read(projectId); + ).rejects.toMatchObject({ code: "request_key_expired" }); + await store.reserveDelegations( + identity, + delegate("request-3", [ + { delegationKey: "writer", outcome: "Write evidence" }, + ]), + target, + ); - await expect( - store.reserveDelegations(identity, delegate("request-3"), target), - ).rejects.toMatchObject({ code: "history_quota_exceeded" }); - expect(await store.read(projectId)).toEqual(before); + const aggregate = await store.read(projectId); + expect(aggregate.requestTombstones).toHaveLength(1); + expect(aggregate.requestTombstones[0]!.requestKey).toBe("request-2"); + expect(aggregate.bindingTombstones).toHaveLength(1); + expect(aggregate.bindingTombstones[0]!.bindingId).toBe(second.bindingId); + }); + + it("prunes proven terminal deliveries so long-lived focused refresh stays writable", async () => { + const root = await fixture(); + const store = new SubsessionCoordinatorStore(root, { + receiptRetentionLimit: 1, + historyTombstoneLimit: 2, + }); + let binding = ( + await store.reserveDelegations(identity, delegate(), target) + ).bindings[0]!; + + for (let index = 1; index <= 70; index += 1) { + binding = ( + await store.refreshFocusedContext(identity, { + schemaVersion: 1, + requestKey: `refresh-${index}`, + operation: { + kind: "refresh-focused-context", + target: { kind: "child", delegationKey: "research" }, + expectedContextEpoch: binding.contextEpoch, + expectedContextDigest: binding.contextDigest, + focus: null, + }, + }) + ).binding; + } + + expect(binding.contextEpoch).toBe(71); + expect(binding.deliveries).toHaveLength(1); + expect(binding.deliveries[0]!.contextEpoch).toBe(71); }); }); diff --git a/packages/harness/src/core/subsession-coordinator-store.ts b/packages/harness/src/core/subsession-coordinator-store.ts index c729a7d3..995f28db 100644 --- a/packages/harness/src/core/subsession-coordinator-store.ts +++ b/packages/harness/src/core/subsession-coordinator-store.ts @@ -48,6 +48,7 @@ export type SubsessionCoordinatorStoreErrorCode = | "live_session_limit_reached" | "delegation_depth_exceeded" | "request_key_reused" + | "request_key_expired" | "delegation_key_reused" | "binding_not_found" | "binding_scope_mismatch" @@ -621,10 +622,10 @@ export function parseSubsessionCoordinatorAggregate( throw new SubsessionCoordinatorStoreError("malformed_state"); } if ( - [...requestReceipts, ...requestTombstones].some(({ bindingIds: ids }) => + requestReceipts.some(({ bindingIds: ids }) => ids.some( (bindingId) => - !allBindings.some((binding) => binding.bindingId === bindingId), + !bindings.some((binding) => binding.bindingId === bindingId), ), ) ) { @@ -638,7 +639,8 @@ export function parseSubsessionCoordinatorAggregate( if (binding.parentBindingId === null) return binding.delegationDepth !== 1; const parent = bindingsById.get(binding.parentBindingId); - return !parent || binding.delegationDepth !== parent.delegationDepth + 1; + return parent !== undefined && + binding.delegationDepth !== parent.delegationDepth + 1; }) ) { throw new SubsessionCoordinatorStoreError("malformed_state"); @@ -718,12 +720,6 @@ export class SubsessionCoordinatorStore { ), ); const expiring = Math.max(0, aggregate.requestReceipts.length - retention); - if ( - aggregate.requestTombstones.length + expiring > - historyLimit - ) { - throw new SubsessionCoordinatorStoreError("history_quota_exceeded"); - } for (let count = 0; count < expiring; count += 1) { const expired = aggregate.requestReceipts.shift(); if (expired) { @@ -737,6 +733,12 @@ export class SubsessionCoordinatorStore { }); } } + if (aggregate.requestTombstones.length > historyLimit) { + aggregate.requestTombstones.splice( + 0, + aggregate.requestTombstones.length - historyLimit, + ); + } const referenced = new Set( aggregate.requestReceipts.flatMap(({ bindingIds }) => bindingIds), @@ -745,12 +747,6 @@ export class SubsessionCoordinatorStore { (binding) => binding.sessionState === "closed" && !referenced.has(binding.bindingId), ); - if ( - aggregate.bindingTombstones.length + reclaimable.length > - historyLimit - ) { - throw new SubsessionCoordinatorStoreError("history_quota_exceeded"); - } for (const binding of reclaimable) { aggregate.bindingTombstones.push({ bindingId: binding.bindingId, @@ -763,6 +759,12 @@ export class SubsessionCoordinatorStore { closedAt: binding.updatedAt, }); } + if (aggregate.bindingTombstones.length > historyLimit) { + aggregate.bindingTombstones.splice( + 0, + aggregate.bindingTombstones.length - historyLimit, + ); + } if (reclaimable.length > 0) { const reclaimed = new Set(reclaimable.map(({ bindingId }) => bindingId)); aggregate.bindings = aggregate.bindings.filter( @@ -1040,9 +1042,7 @@ export class SubsessionCoordinatorStore { ) => receipt.parentSessionId === identity.sessionId && receipt.requestKey === request.requestKey; - const previous = - aggregate.requestReceipts.find(sameRequest) ?? - aggregate.requestTombstones.find(sameRequest); + const previous = aggregate.requestReceipts.find(sameRequest); if (previous) { if ( previous.requestDigest !== requestDigest || @@ -1063,6 +1063,8 @@ export class SubsessionCoordinatorStore { throw new SubsessionCoordinatorStoreError("malformed_state"); return { value: { replayed: true, requestDigest, binding } }; } + if (aggregate.requestTombstones.some(sameRequest)) + throw new SubsessionCoordinatorStoreError("request_key_expired"); const target = targetSelector.kind === "self" ? aggregate.bindings.find( @@ -1084,8 +1086,16 @@ export class SubsessionCoordinatorStore { const currentDelivery = target.deliveries.find( ({ contextEpoch }) => contextEpoch === target.contextEpoch, ); - if (currentDelivery?.state === "uncertain") + if ( + currentDelivery && + ["claimed", "submitted-unacknowledged", "uncertain"].includes( + currentDelivery.state, + ) + ) throw new SubsessionCoordinatorStoreError("claim_conflict"); + target.deliveries = target.deliveries.filter(({ state }) => + ["claimed", "submitted-unacknowledged", "uncertain"].includes(state), + ); if (target.deliveries.length >= SUBSESSION_COORDINATOR_DELIVERY_LIMIT) throw new SubsessionCoordinatorStoreError("history_quota_exceeded"); @@ -1155,9 +1165,7 @@ export class SubsessionCoordinatorStore { ): boolean => receipt.parentSessionId === identity.sessionId && receipt.requestKey === request.requestKey; - const previous = - aggregate.requestReceipts.find(sameRequest) ?? - aggregate.requestTombstones.find(sameRequest); + const previous = aggregate.requestReceipts.find(sameRequest); if (previous) { if ( previous.requestDigest !== requestDigest || @@ -1187,6 +1195,8 @@ export class SubsessionCoordinatorStore { value: { replayed: true, requestDigest, bindings }, }; } + if (aggregate.requestTombstones.some(sameRequest)) + throw new SubsessionCoordinatorStoreError("request_key_expired"); const now = this.now(); const bindings: SubsessionBindingRecord[] = []; let created = 0; diff --git a/packages/harness/src/core/subsession-coordinator.test.ts b/packages/harness/src/core/subsession-coordinator.test.ts index 29f52ed5..48b5332d 100644 --- a/packages/harness/src/core/subsession-coordinator.test.ts +++ b/packages/harness/src/core/subsession-coordinator.test.ts @@ -510,7 +510,7 @@ describe("SubsessionCoordinator", () => { unsubscribe(); }); - it("does not advise retry or request reduction for permanent history exhaustion", async () => { + it("does not advise retry for unreclaimable active history exhaustion", async () => { const { coordinator, caller, store, unsubscribe } = await fixture(); vi.spyOn(store, "reserveDelegations").mockRejectedValueOnce( new SubsessionCoordinatorStoreError("history_quota_exceeded"), @@ -525,6 +525,21 @@ describe("SubsessionCoordinator", () => { unsubscribe(); }); + it("requires a fresh request key after its bounded receipt window expires", async () => { + const { coordinator, caller, store, unsubscribe } = await fixture(); + vi.spyOn(store, "reserveDelegations").mockRejectedValueOnce( + new SubsessionCoordinatorStoreError("request_key_expired"), + ); + await expect(coordinator.execute(caller, request)).rejects.toMatchObject({ + detail: { + code: "request_key_reused", + retryable: false, + recovery: "new_request_key", + }, + }); + unsubscribe(); + }); + it("writes no kickoff when adapter identity correlation is ambiguous", async () => { const { coordinator, caller, spawned, unsubscribe } = await fixture( false, diff --git a/packages/harness/src/core/subsession-coordinator.ts b/packages/harness/src/core/subsession-coordinator.ts index e3a99bae..cdec7fdc 100644 --- a/packages/harness/src/core/subsession-coordinator.ts +++ b/packages/harness/src/core/subsession-coordinator.ts @@ -1142,7 +1142,10 @@ export class SubsessionCoordinator { private wholeCallError(cause: unknown, refresh = false): SubsessionCoordinatorError { if (cause instanceof SubsessionCoordinatorError) return cause; if (cause instanceof SubsessionCoordinatorStoreError) { - if (cause.code === "request_key_reused") + if ( + cause.code === "request_key_reused" || + cause.code === "request_key_expired" + ) return new SubsessionCoordinatorError( error("request_key_reused", false, "new_request_key"), ); From 3a1ea2625b85bed614a332520faeff80684a3792 Mon Sep 17 00:00:00 2001 From: Yash Date: Fri, 4 Sep 2026 20:23:22 +0000 Subject: [PATCH 11/19] fix(harness): reclaim terminal subsession bindings Refs: SAP-3151 --- .changeset/writable-project-subsessions.md | 3 +- packages/harness/docs/shared-build-plan.md | 9 +- .../core/subsession-coordinator-store.test.ts | 121 ++++++++++++++++++ .../src/core/subsession-coordinator-store.ts | 13 +- .../src/core/subsession-coordinator.test.ts | 2 +- .../src/core/subsession-coordinator.ts | 7 +- .../src/shared/subsession-delegation.ts | 1 + 7 files changed, 140 insertions(+), 16 deletions(-) diff --git a/.changeset/writable-project-subsessions.md b/.changeset/writable-project-subsessions.md index 95e028eb..8af5f62f 100644 --- a/.changeset/writable-project-subsessions.md +++ b/.changeset/writable-project-subsessions.md @@ -23,4 +23,5 @@ levels and 64 concurrently live coordinator-owned sessions per project. Closing a delegated tab starts PTY termination before its private user-close tombstone is persisted, so a storage error cannot leave the process running. Request, binding, and acknowledged-delivery history use bounded retention so long-lived -projects do not dead-end on routine delegation or context refreshes. +projects do not dead-end on routine delegation, ordinary session exit, or +context refreshes. diff --git a/packages/harness/docs/shared-build-plan.md b/packages/harness/docs/shared-build-plan.md index 83514830..4ed2b633 100644 --- a/packages/harness/docs/shared-build-plan.md +++ b/packages/harness/docs/shared-build-plan.md @@ -109,10 +109,11 @@ Identical retries converge on the same durable binding and real Harness session ID; changing canonical request or binding content under an existing key fails explicitly. All binding IDs and session IDs for a bounded batch are reserved in one durable transaction before the first process is spawned. -Older request receipts compact into bounded key tombstones, and closed bindings -compact into bounded ownership tombstones once no retained receipt references -them. The oldest tombstones expire as the retention window advances, so routine -delegation and focused-context refreshes cannot permanently exhaust a project. +Older request receipts compact into bounded key tombstones, and closed, exited, +or failed bindings compact into bounded ownership tombstones once no retained +receipt references them. The oldest tombstones expire as the retention window +advances, so routine delegation and focused-context refreshes cannot permanently +exhaust a project. Proven acknowledged or unsent delivery epochs are likewise pruned when a newer focused-context delivery replaces them; ambiguous delivery evidence is retained. diff --git a/packages/harness/src/core/subsession-coordinator-store.test.ts b/packages/harness/src/core/subsession-coordinator-store.test.ts index 5c8b0809..c1dd060d 100644 --- a/packages/harness/src/core/subsession-coordinator-store.test.ts +++ b/packages/harness/src/core/subsession-coordinator-store.test.ts @@ -406,6 +406,75 @@ describe("SubsessionCoordinatorStore", () => { ).rejects.toBeInstanceOf(SubsessionCoordinatorStoreError); }); + it("queues a refresh while the prior delivery awaits acknowledgement", async () => { + const root = await fixture(); + const store = new SubsessionCoordinatorStore(root); + const binding = ( + await store.reserveDelegations(identity, delegate(), target) + ).bindings[0]!; + const spawn = await store.claimSpawn(identity, binding.bindingId, { + ownerId: "coordinator-1", + expectedLifecycleEpoch: binding.lifecycleEpoch, + expectedSpawnEpoch: binding.spawnEpoch, + }); + if (!spawn.claimed || !spawn.binding.spawnClaim) + throw new Error("spawn claim was not acquired"); + const starting = await store.attachSpawnedRuntime( + identity, + binding.bindingId, + { + claimId: spawn.binding.spawnClaim.claimId, + spawnEpoch: spawn.binding.spawnEpoch, + runtimeToken: "runtime-refresh", + incarnation: 1, + }, + ); + const ready = await store.transitionSession(identity, binding.bindingId, { + expectedLifecycleEpoch: starting.lifecycleEpoch, + expectedSpawnEpoch: starting.spawnEpoch, + expectedRuntimeToken: "runtime-refresh", + state: "ready", + }); + const claimed = await store.claimKickoff(identity, binding.bindingId, { + ownerId: "sender-1", + expectedLifecycleEpoch: ready.lifecycleEpoch, + expectedSpawnEpoch: ready.spawnEpoch, + expectedContextEpoch: ready.contextEpoch, + eventWatermark: "event-10", + }); + if (!claimed.claimed || !claimed.binding.deliveries[0]!.claim) + throw new Error("kickoff claim was not acquired"); + const submitted = await store.recordKickoffWrite( + identity, + binding.bindingId, + { + contextEpoch: claimed.binding.contextEpoch, + deliveryId: claimed.binding.deliveries[0]!.deliveryId, + inputId: claimed.binding.deliveries[0]!.inputId, + claimId: claimed.binding.deliveries[0]!.claim!.claimId, + phase: "enter-written", + }, + ); + + const refreshed = await store.refreshFocusedContext(identity, { + schemaVersion: 1, + requestKey: "refresh-while-awaiting-ack", + operation: { + kind: "refresh-focused-context", + target: { kind: "child", delegationKey: "research" }, + expectedContextEpoch: submitted.contextEpoch, + expectedContextDigest: submitted.contextDigest, + focus: null, + }, + }); + + expect(refreshed.binding.deliveries).toHaveLength(2); + expect(refreshed.binding.deliveries.map(({ state }) => state)).toEqual([ + "submitted-unacknowledged", + "pending", + ]); + }); + it("scopes mutations to the trusted parent and never adopts a foreign binding", async () => { const root = await fixture(); const store = new SubsessionCoordinatorStore(root); @@ -561,6 +630,58 @@ describe("SubsessionCoordinatorStore", () => { expect(aggregate.bindingTombstones[0]!.bindingId).toBe(second.bindingId); }); + it("compacts an exited binding after its replay receipt expires", async () => { + const root = await fixture(); + const store = new SubsessionCoordinatorStore(root, { + receiptRetentionLimit: 1, + }); + const first = ( + await store.reserveDelegations(identity, delegate("request-1"), target) + ).bindings[0]!; + const claim = await store.claimSpawn(identity, first.bindingId, { + ownerId: "coordinator-1", + expectedLifecycleEpoch: first.lifecycleEpoch, + expectedSpawnEpoch: first.spawnEpoch, + }); + if (!claim.claimed || !claim.binding.spawnClaim) + throw new Error("spawn claim was not acquired"); + const starting = await store.attachSpawnedRuntime( + identity, + first.bindingId, + { + claimId: claim.binding.spawnClaim.claimId, + spawnEpoch: claim.binding.spawnEpoch, + runtimeToken: "runtime-exited", + incarnation: 1, + }, + ); + await store.transitionSession(identity, first.bindingId, { + expectedLifecycleEpoch: starting.lifecycleEpoch, + expectedSpawnEpoch: starting.spawnEpoch, + expectedRuntimeToken: "runtime-exited", + state: "exited", + }); + + await store.reserveDelegations( + identity, + delegate("request-2", [ + { delegationKey: "publisher", outcome: "Publish evidence" }, + ]), + target, + ); + + const aggregate = await store.read(projectId); + expect(aggregate.bindings.map(({ bindingId }) => bindingId)).not.toContain( + first.bindingId, + ); + expect(aggregate.bindingTombstones).toContainEqual( + expect.objectContaining({ + bindingId: first.bindingId, + sessionId: first.sessionId, + }), + ); + }); + it("prunes proven terminal deliveries so long-lived focused refresh stays writable", async () => { const root = await fixture(); const store = new SubsessionCoordinatorStore(root, { diff --git a/packages/harness/src/core/subsession-coordinator-store.ts b/packages/harness/src/core/subsession-coordinator-store.ts index 995f28db..5fc89cdb 100644 --- a/packages/harness/src/core/subsession-coordinator-store.ts +++ b/packages/harness/src/core/subsession-coordinator-store.ts @@ -36,7 +36,7 @@ import { isStudioProjectId } from "./studio-project-catalog.js"; export const SUBSESSION_COORDINATOR_BINDING_LIMIT = 8_192; export const SUBSESSION_COORDINATOR_RECEIPT_LIMIT = 8_192; -export const SUBSESSION_COORDINATOR_RECEIPT_RETENTION_LIMIT = 1_024; +export const SUBSESSION_COORDINATOR_RECEIPT_RETENTION_LIMIT = 256; export const SUBSESSION_COORDINATOR_DELIVERY_LIMIT = 64; export type SubsessionCoordinatorStoreErrorCode = @@ -745,7 +745,8 @@ export class SubsessionCoordinatorStore { ); const reclaimable = aggregate.bindings.filter( (binding) => - binding.sessionState === "closed" && !referenced.has(binding.bindingId), + ["closed", "exited", "failed"].includes(binding.sessionState) && + !referenced.has(binding.bindingId), ); for (const binding of reclaimable) { aggregate.bindingTombstones.push({ @@ -1086,12 +1087,7 @@ export class SubsessionCoordinatorStore { const currentDelivery = target.deliveries.find( ({ contextEpoch }) => contextEpoch === target.contextEpoch, ); - if ( - currentDelivery && - ["claimed", "submitted-unacknowledged", "uncertain"].includes( - currentDelivery.state, - ) - ) + if (currentDelivery?.state === "uncertain") throw new SubsessionCoordinatorStoreError("claim_conflict"); target.deliveries = target.deliveries.filter(({ state }) => ["claimed", "submitted-unacknowledged", "uncertain"].includes(state), @@ -1197,6 +1193,7 @@ export class SubsessionCoordinatorStore { } if (aggregate.requestTombstones.some(sameRequest)) throw new SubsessionCoordinatorStoreError("request_key_expired"); + this.compactTerminalHistory(aggregate); const now = this.now(); const bindings: SubsessionBindingRecord[] = []; let created = 0; diff --git a/packages/harness/src/core/subsession-coordinator.test.ts b/packages/harness/src/core/subsession-coordinator.test.ts index 48b5332d..76afc1c1 100644 --- a/packages/harness/src/core/subsession-coordinator.test.ts +++ b/packages/harness/src/core/subsession-coordinator.test.ts @@ -532,7 +532,7 @@ describe("SubsessionCoordinator", () => { ); await expect(coordinator.execute(caller, request)).rejects.toMatchObject({ detail: { - code: "request_key_reused", + code: "request_key_expired", retryable: false, recovery: "new_request_key", }, diff --git a/packages/harness/src/core/subsession-coordinator.ts b/packages/harness/src/core/subsession-coordinator.ts index cdec7fdc..d2582975 100644 --- a/packages/harness/src/core/subsession-coordinator.ts +++ b/packages/harness/src/core/subsession-coordinator.ts @@ -1143,12 +1143,15 @@ export class SubsessionCoordinator { if (cause instanceof SubsessionCoordinatorError) return cause; if (cause instanceof SubsessionCoordinatorStoreError) { if ( - cause.code === "request_key_reused" || - cause.code === "request_key_expired" + cause.code === "request_key_reused" ) return new SubsessionCoordinatorError( error("request_key_reused", false, "new_request_key"), ); + if (cause.code === "request_key_expired") + return new SubsessionCoordinatorError( + error("request_key_expired", false, "new_request_key"), + ); if (cause.code === "delegation_key_reused") return new SubsessionCoordinatorError( error("delegation_key_reused", false, "new_delegation_key"), diff --git a/packages/harness/src/shared/subsession-delegation.ts b/packages/harness/src/shared/subsession-delegation.ts index edfe8833..128ad928 100644 --- a/packages/harness/src/shared/subsession-delegation.ts +++ b/packages/harness/src/shared/subsession-delegation.ts @@ -84,6 +84,7 @@ export type DelegationErrorCode = | "unsupported_schema" | "capacity_exceeded" | "request_key_reused" + | "request_key_expired" | "storage_unavailable" | "internal_error" | "delegation_key_reused" From ac7a22b7d9ad20ea1651b8fc2cae4208f27f522c Mon Sep 17 00:00:00 2001 From: Yash Date: Fri, 4 Sep 2026 20:33:17 +0000 Subject: [PATCH 12/19] fix(harness): preserve resumable subsession bindings Refs: SAP-3151 --- .changeset/writable-project-subsessions.md | 5 +- packages/harness/docs/shared-build-plan.md | 11 +- .../core/subsession-coordinator-store.test.ts | 113 ++++++++++-------- .../src/core/subsession-coordinator-store.ts | 17 +-- 4 files changed, 85 insertions(+), 61 deletions(-) diff --git a/.changeset/writable-project-subsessions.md b/.changeset/writable-project-subsessions.md index 8af5f62f..218b56db 100644 --- a/.changeset/writable-project-subsessions.md +++ b/.changeset/writable-project-subsessions.md @@ -23,5 +23,6 @@ levels and 64 concurrently live coordinator-owned sessions per project. Closing a delegated tab starts PTY termination before its private user-close tombstone is persisted, so a storage error cannot leave the process running. Request, binding, and acknowledged-delivery history use bounded retention so long-lived -projects do not dead-end on routine delegation, ordinary session exit, or -context refreshes. +projects do not dead-end on routine delegation or context refreshes. Exited and +failed bindings remain durable so their real Harness sessions can still resume +or recover. diff --git a/packages/harness/docs/shared-build-plan.md b/packages/harness/docs/shared-build-plan.md index 4ed2b633..5e7cc9c5 100644 --- a/packages/harness/docs/shared-build-plan.md +++ b/packages/harness/docs/shared-build-plan.md @@ -109,11 +109,12 @@ Identical retries converge on the same durable binding and real Harness session ID; changing canonical request or binding content under an existing key fails explicitly. All binding IDs and session IDs for a bounded batch are reserved in one durable transaction before the first process is spawned. -Older request receipts compact into bounded key tombstones, and closed, exited, -or failed bindings compact into bounded ownership tombstones once no retained -receipt references them. The oldest tombstones expire as the retention window -advances, so routine delegation and focused-context refreshes cannot permanently -exhaust a project. +Older request receipts compact into bounded key tombstones, and explicitly +closed bindings compact into bounded ownership tombstones once no retained +receipt references them. Exited and failed bindings remain available for the +coordinator's ordinary resume and recovery paths. The oldest tombstones expire +as the retention window advances, so routine delegation and focused-context +refreshes cannot permanently exhaust a project. Proven acknowledged or unsent delivery epochs are likewise pruned when a newer focused-context delivery replaces them; ambiguous delivery evidence is retained. diff --git a/packages/harness/src/core/subsession-coordinator-store.test.ts b/packages/harness/src/core/subsession-coordinator-store.test.ts index c1dd060d..08e35bcb 100644 --- a/packages/harness/src/core/subsession-coordinator-store.test.ts +++ b/packages/harness/src/core/subsession-coordinator-store.test.ts @@ -630,57 +630,76 @@ describe("SubsessionCoordinatorStore", () => { expect(aggregate.bindingTombstones[0]!.bindingId).toBe(second.bindingId); }); - it("compacts an exited binding after its replay receipt expires", async () => { - const root = await fixture(); - const store = new SubsessionCoordinatorStore(root, { - receiptRetentionLimit: 1, - }); - const first = ( - await store.reserveDelegations(identity, delegate("request-1"), target) - ).bindings[0]!; - const claim = await store.claimSpawn(identity, first.bindingId, { - ownerId: "coordinator-1", - expectedLifecycleEpoch: first.lifecycleEpoch, - expectedSpawnEpoch: first.spawnEpoch, - }); - if (!claim.claimed || !claim.binding.spawnClaim) - throw new Error("spawn claim was not acquired"); - const starting = await store.attachSpawnedRuntime( - identity, - first.bindingId, - { - claimId: claim.binding.spawnClaim.claimId, - spawnEpoch: claim.binding.spawnEpoch, - runtimeToken: "runtime-exited", - incarnation: 1, - }, - ); - await store.transitionSession(identity, first.bindingId, { - expectedLifecycleEpoch: starting.lifecycleEpoch, - expectedSpawnEpoch: starting.spawnEpoch, - expectedRuntimeToken: "runtime-exited", - state: "exited", - }); + it.each(["exited", "failed"] as const)( + "keeps a %s binding resumable and counted toward live capacity after receipt churn", + async (sessionState) => { + const root = await fixture(); + const store = new SubsessionCoordinatorStore(root, { + receiptRetentionLimit: 1, + liveSessionLimit: 2, + }); + const first = ( + await store.reserveDelegations(identity, delegate("request-1"), target) + ).bindings[0]!; + const claim = await store.claimSpawn(identity, first.bindingId, { + ownerId: "coordinator-1", + expectedLifecycleEpoch: first.lifecycleEpoch, + expectedSpawnEpoch: first.spawnEpoch, + }); + if (!claim.claimed || !claim.binding.spawnClaim) + throw new Error("spawn claim was not acquired"); + const starting = await store.attachSpawnedRuntime( + identity, + first.bindingId, + { + claimId: claim.binding.spawnClaim.claimId, + spawnEpoch: claim.binding.spawnEpoch, + runtimeToken: "runtime-exited", + incarnation: 1, + }, + ); + await store.transitionSession(identity, first.bindingId, { + expectedLifecycleEpoch: starting.lifecycleEpoch, + expectedSpawnEpoch: starting.spawnEpoch, + expectedRuntimeToken: "runtime-exited", + state: sessionState, + }); - await store.reserveDelegations( - identity, - delegate("request-2", [ - { delegationKey: "publisher", outcome: "Publish evidence" }, - ]), - target, - ); + await store.reserveDelegations( + identity, + delegate("request-2", [ + { delegationKey: "publisher", outcome: "Publish evidence" }, + ]), + target, + ); + const replay = await store.reserveDelegations( + identity, + delegate("request-3"), + target, + ); - const aggregate = await store.read(projectId); - expect(aggregate.bindings.map(({ bindingId }) => bindingId)).not.toContain( - first.bindingId, - ); - expect(aggregate.bindingTombstones).toContainEqual( - expect.objectContaining({ + const aggregate = await store.read(projectId); + expect(replay.bindings[0]).toMatchObject({ bindingId: first.bindingId, sessionId: first.sessionId, - }), - ); - }); + sessionState, + }); + expect(aggregate.bindings).toContainEqual(replay.bindings[0]); + expect(aggregate.bindingTombstones).not.toContainEqual( + expect.objectContaining({ bindingId: first.bindingId }), + ); + await expect( + store.reserveDelegations( + identity, + delegate("request-4", [ + { delegationKey: "writer", outcome: "Write evidence" }, + ]), + target, + ), + ).rejects.toMatchObject({ code: "live_session_limit_reached" }); + expect((await store.read(projectId)).bindings).toHaveLength(2); + }, + ); it("prunes proven terminal deliveries so long-lived focused refresh stays writable", async () => { const root = await fixture(); diff --git a/packages/harness/src/core/subsession-coordinator-store.ts b/packages/harness/src/core/subsession-coordinator-store.ts index 5fc89cdb..6e8db5ad 100644 --- a/packages/harness/src/core/subsession-coordinator-store.ts +++ b/packages/harness/src/core/subsession-coordinator-store.ts @@ -745,8 +745,7 @@ export class SubsessionCoordinatorStore { ); const reclaimable = aggregate.bindings.filter( (binding) => - ["closed", "exited", "failed"].includes(binding.sessionState) && - !referenced.has(binding.bindingId), + binding.sessionState === "closed" && !referenced.has(binding.bindingId), ); for (const binding of reclaimable) { aggregate.bindingTombstones.push({ @@ -1198,9 +1197,15 @@ export class SubsessionCoordinatorStore { const bindings: SubsessionBindingRecord[] = []; let created = 0; const live = aggregate.bindings.filter(({ sessionState }) => - ["reserved", "spawn-claimed", "starting", "awaiting-ready", "ready"].includes( - sessionState, - ), + [ + "reserved", + "spawn-claimed", + "starting", + "awaiting-ready", + "ready", + "exited", + "failed", + ].includes(sessionState), ).length; const parentBinding = aggregate.bindings.find( ({ sessionId }) => sessionId === identity.sessionId, @@ -1231,8 +1236,6 @@ export class SubsessionCoordinatorStore { ); if (existing.sessionState === "closed") throw new SubsessionCoordinatorStoreError("session_closed"); - if (["exited", "failed"].includes(existing.sessionState)) - additionalLive += 1; bindings.push(existing); continue; } From 97dc184acc6e62aad44126f5f3ef09b41bb42fa1 Mon Sep 17 00:00:00 2001 From: Yash Date: Fri, 4 Sep 2026 20:40:25 +0000 Subject: [PATCH 13/19] fix(harness): classify subsession capacity exhaustion Refs: SAP-3151 --- .changeset/writable-project-subsessions.md | 15 ++++++++------- packages/harness/docs/shared-build-plan.md | 5 +++-- .../src/core/subsession-coordinator.test.ts | 15 +++++++++++++++ .../harness/src/core/subsession-coordinator.ts | 2 +- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/.changeset/writable-project-subsessions.md b/.changeset/writable-project-subsessions.md index 218b56db..d1e48062 100644 --- a/.changeset/writable-project-subsessions.md +++ b/.changeset/writable-project-subsessions.md @@ -19,10 +19,11 @@ Manual sessions remain outside coordinator ownership. Consumers should treat `uncertain` kickoff delivery as terminal until an exact persisted acknowledgement arrives, and should use a new request/delegation key when the corresponding canonical content changes. Nested delegation is bounded to four -levels and 64 concurrently live coordinator-owned sessions per project. Closing -a delegated tab starts PTY termination before its private user-close tombstone -is persisted, so a storage error cannot leave the process running. Request, -binding, and acknowledged-delivery history use bounded retention so long-lived -projects do not dead-end on routine delegation or context refreshes. Exited and -failed bindings remain durable so their real Harness sessions can still resume -or recover. +levels and 64 live or resumable coordinator-owned sessions per project; callers +inspect and close sessions instead of blindly retrying when that bound is met. +Closing a delegated tab starts PTY termination before its private user-close +tombstone is persisted, so a storage error cannot leave the process running. +Request, binding, and acknowledged-delivery history use bounded retention so +long-lived projects do not dead-end on routine delegation or context refreshes. +Exited and failed bindings remain durable so their real Harness sessions can +still resume or recover. diff --git a/packages/harness/docs/shared-build-plan.md b/packages/harness/docs/shared-build-plan.md index 5e7cc9c5..204b72db 100644 --- a/packages/harness/docs/shared-build-plan.md +++ b/packages/harness/docs/shared-build-plan.md @@ -100,8 +100,9 @@ sixteen ordinary writable sessions. Each child receives the same common project-agent prompt, coding capabilities, project tools, and delegation tool, so nested delegation follows the same path. An exact assignment, map node, or brief may focus the child, but focus never changes its tools or authority. -Delegation is bounded to four levels and 64 concurrently live -coordinator-owned sessions per project. +Delegation is bounded to four levels and 64 live or resumable +coordinator-owned sessions per project. Reaching that bound directs the caller +to inspect and close sessions; blind retry cannot allocate another session. Callers provide both a request key and a delegation key. Identity is scoped by the private session capability to the trusted project and parent session. diff --git a/packages/harness/src/core/subsession-coordinator.test.ts b/packages/harness/src/core/subsession-coordinator.test.ts index 76afc1c1..a1a6509f 100644 --- a/packages/harness/src/core/subsession-coordinator.test.ts +++ b/packages/harness/src/core/subsession-coordinator.test.ts @@ -525,6 +525,21 @@ describe("SubsessionCoordinator", () => { unsubscribe(); }); + it("directs resumable-session capacity exhaustion to session inspection", async () => { + const { coordinator, caller, store, unsubscribe } = await fixture(); + vi.spyOn(store, "reserveDelegations").mockRejectedValueOnce( + new SubsessionCoordinatorStoreError("live_session_limit_reached"), + ); + await expect(coordinator.execute(caller, request)).rejects.toMatchObject({ + detail: { + code: "capacity_exceeded", + retryable: false, + recovery: "inspect_session", + }, + }); + unsubscribe(); + }); + it("requires a fresh request key after its bounded receipt window expires", async () => { const { coordinator, caller, store, unsubscribe } = await fixture(); vi.spyOn(store, "reserveDelegations").mockRejectedValueOnce( diff --git a/packages/harness/src/core/subsession-coordinator.ts b/packages/harness/src/core/subsession-coordinator.ts index d2582975..c1158f2e 100644 --- a/packages/harness/src/core/subsession-coordinator.ts +++ b/packages/harness/src/core/subsession-coordinator.ts @@ -1162,7 +1162,7 @@ export class SubsessionCoordinator { ); if (cause.code === "live_session_limit_reached") return new SubsessionCoordinatorError( - error("capacity_exceeded", true, "retry"), + error("capacity_exceeded", false, "inspect_session"), ); if ( cause.code === "delegation_depth_exceeded" || From d00469d7ff07799f62ba48b1bddab6a6c40ea15a Mon Sep 17 00:00:00 2001 From: Yash Date: Fri, 4 Sep 2026 20:57:36 +0000 Subject: [PATCH 14/19] feat(harness): release owned project subsessions Refs: SAP-3151 --- .changeset/writable-project-subsessions.md | 20 +-- packages/harness/docs/shared-build-plan.md | 18 ++- .../harness/src/core/session-manager.test.ts | 26 +++ packages/harness/src/core/session-manager.ts | 21 +++ .../core/subsession-coordinator-store.test.ts | 153 ++++++++++++++++++ .../src/core/subsession-coordinator-store.ts | 124 +++++++++++++- .../src/core/subsession-coordinator.test.ts | 122 ++++++++++++++ .../src/core/subsession-coordinator.ts | 113 +++++++++++++ .../harness/src/profiles/project-agent.ts | 2 +- .../harness/src/server/agent-map-mcp-tools.ts | 8 +- .../src/server/agent-map-mcp-wiring.test.ts | 19 +++ .../subsession-delegation-codec.test.ts | 40 ++++- .../src/shared/subsession-delegation-codec.ts | 22 ++- .../src/shared/subsession-delegation.ts | 5 + packages/harness/src/shared/types.ts | 1 + 15 files changed, 666 insertions(+), 28 deletions(-) diff --git a/.changeset/writable-project-subsessions.md b/.changeset/writable-project-subsessions.md index d1e48062..68c27628 100644 --- a/.changeset/writable-project-subsessions.md +++ b/.changeset/writable-project-subsessions.md @@ -2,8 +2,8 @@ "@sapiom/harness": minor --- -Add capability-scoped `project_subsession_delegate` support for creating or -reusing bounded batches of ordinary writable project sessions. Delegations use +Add capability-scoped `project_subsession_delegate` support for creating, +reusing, or releasing bounded batches of ordinary writable project sessions. Delegations use durable parent/key bindings, canonical request digests, transactional spawn and kickoff claims, exact focused-context references, readiness-gated delivery, restart recovery, nested common-tool composition, and real session IDs. @@ -19,11 +19,11 @@ Manual sessions remain outside coordinator ownership. Consumers should treat `uncertain` kickoff delivery as terminal until an exact persisted acknowledgement arrives, and should use a new request/delegation key when the corresponding canonical content changes. Nested delegation is bounded to four -levels and 64 live or resumable coordinator-owned sessions per project; callers -inspect and close sessions instead of blindly retrying when that bound is met. -Closing a delegated tab starts PTY termination before its private user-close -tombstone is persisted, so a storage error cannot leave the process running. -Request, binding, and acknowledged-delivery history use bounded retention so -long-lived projects do not dead-end on routine delegation or context refreshes. -Exited and failed bindings remain durable so their real Harness sessions can -still resume or recover. +levels and 64 live or resumable coordinator-owned sessions per project. A +parent can idempotently release its own child bindings by delegation key, +closing the exact coordinator-owned Harness session and recovering capacity +without granting access to manual or foreign sessions. Request, binding, and +acknowledged-delivery history use bounded retention so long-lived projects do +not dead-end on routine delegation, release, or context refreshes. Exited and +failed bindings remain durable for resume or recovery until explicitly +released. diff --git a/packages/harness/docs/shared-build-plan.md b/packages/harness/docs/shared-build-plan.md index 204b72db..a0407cee 100644 --- a/packages/harness/docs/shared-build-plan.md +++ b/packages/harness/docs/shared-build-plan.md @@ -95,14 +95,16 @@ path. ## Writable project subsessions Every ordinary project session discovers `project_subsession_delegate` beside -the shared map, plan, and brief tools. The operation creates or reuses one to -sixteen ordinary writable sessions. Each child receives the same common +the shared map, plan, and brief tools. The operation creates, reuses, or +releases one to sixteen ordinary writable sessions. Each child receives the same common project-agent prompt, coding capabilities, project tools, and delegation tool, so nested delegation follows the same path. An exact assignment, map node, or brief may focus the child, but focus never changes its tools or authority. Delegation is bounded to four levels and 64 live or resumable -coordinator-owned sessions per project. Reaching that bound directs the caller -to inspect and close sessions; blind retry cannot allocate another session. +coordinator-owned sessions per project. A parent can idempotently release its +own child bindings by delegation key to close their real Harness sessions and +recover capacity; it cannot name arbitrary session IDs or release another +parent's or a manual session. Callers provide both a request key and a delegation key. Identity is scoped by the private session capability to the trusted project and parent session. @@ -113,9 +115,11 @@ one durable transaction before the first process is spawned. Older request receipts compact into bounded key tombstones, and explicitly closed bindings compact into bounded ownership tombstones once no retained receipt references them. Exited and failed bindings remain available for the -coordinator's ordinary resume and recovery paths. The oldest tombstones expire -as the retention window advances, so routine delegation and focused-context -refreshes cannot permanently exhaust a project. +coordinator's ordinary resume and recovery paths until explicitly released. +Release receipts make partial retries converge before the closed binding is +compacted. The oldest tombstones expire as the retention window advances, so +routine delegation, release, and focused-context refreshes cannot permanently +exhaust a project. Proven acknowledged or unsent delivery epochs are likewise pruned when a newer focused-context delivery replaces them; ambiguous delivery evidence is retained. diff --git a/packages/harness/src/core/session-manager.test.ts b/packages/harness/src/core/session-manager.test.ts index 17a019da..1c5f46a6 100644 --- a/packages/harness/src/core/session-manager.test.ts +++ b/packages/harness/src/core/session-manager.test.ts @@ -459,6 +459,32 @@ describe("SessionManager", () => { expect(onSubsessionUserClosed).toHaveBeenCalledTimes(2); }); + it("closes only an exact coordinator-owned binding through the trusted path", async () => { + const { manager, spawns } = makeManager(); + const sessionId = "00000000-0000-4000-8000-000000000116"; + const input = delegatedCreate(sessionId); + await manager.createReserved( + sessionId, + { cwd: input.cwd, harness: input.harness }, + marker(sessionId), + input.trusted, + ); + + await expect( + manager.closeBound({ ...marker(sessionId), bindingId: "binding_foreign" }), + ).rejects.toBeInstanceOf(SubsessionBindingMismatchError); + expect(spawns[0]!.pty.kill).not.toHaveBeenCalled(); + + const closing = manager.closeBound(marker(sessionId)); + await vi.waitFor(() => + expect(spawns[0]!.pty.kill).toHaveBeenCalledTimes(1), + ); + spawns[0]!.emitExit(0); + await expect(closing).resolves.toBe(true); + await expect(manager.closeBound(marker(sessionId))).resolves.toBe(false); + expect(manager.wasSubsessionClosedByUser(marker(sessionId))).toBe(true); + }); + it("reports exact input write phases and kills only an exact runtime", async () => { const { manager, spawns } = makeManager(); const session = await manager.create({ diff --git a/packages/harness/src/core/session-manager.ts b/packages/harness/src/core/session-manager.ts index 2cf90faf..d799d73d 100644 --- a/packages/harness/src/core/session-manager.ts +++ b/packages/harness/src/core/session-manager.ts @@ -1918,6 +1918,27 @@ export class SessionManager { return killed; } + /** Close only when the caller proves the exact coordinator-owned binding. */ + async closeBound(expected: TrustedSubsessionBindingMarker): Promise { + const parsed = parseTrustedSubsessionBindingMarker( + expected, + expected.sessionId, + ); + if (!parsed) throw new SubsessionBindingMismatchError(); + const operation = async (): Promise => { + const current = this.subsessionBindings.get(parsed.sessionId); + if (!current || !sameSubsessionBinding(current, parsed)) + throw new SubsessionBindingMismatchError(); + return this.close(parsed.sessionId); + }; + const next = this.subsessionBindingQueue.catch(() => {}).then(operation); + this.subsessionBindingQueue = next.then( + () => undefined, + () => undefined, + ); + return next; + } + kill(id: string): Promise { const handle = this.ptys.get(id); if (!handle) { diff --git a/packages/harness/src/core/subsession-coordinator-store.test.ts b/packages/harness/src/core/subsession-coordinator-store.test.ts index 08e35bcb..2259e44a 100644 --- a/packages/harness/src/core/subsession-coordinator-store.test.ts +++ b/packages/harness/src/core/subsession-coordinator-store.test.ts @@ -30,6 +30,14 @@ const delegate = ( requestKey, operation: { kind: "delegate", delegations }, }); +const release = ( + requestKey = "release-1", + delegationKeys = ["research"], +) => ({ + schemaVersion: 1, + requestKey, + operation: { kind: "release", delegationKeys }, +}); describe("SubsessionCoordinatorStore", () => { const roots: string[] = []; @@ -118,6 +126,82 @@ describe("SubsessionCoordinatorStore", () => { expect(aggregate.bindings).toEqual(original.bindings); }); + it("reserves idempotent releases only for the trusted parent binding", async () => { + const root = await fixture(); + const store = new SubsessionCoordinatorStore(root); + const binding = ( + await store.reserveDelegations(identity, delegate(), target) + ).bindings[0]!; + + const first = await store.reserveReleases(identity, release()); + const replay = await store.reserveReleases(identity, release()); + expect(first).toMatchObject({ + replayed: false, + bindings: [{ state: "bound", binding: { bindingId: binding.bindingId } }], + }); + expect(replay).toEqual({ ...first, replayed: true }); + + const foreign = { ...identity, sessionId: "manual-session" }; + await expect( + store.reserveReleases(foreign, release("foreign-release")), + ).rejects.toMatchObject({ code: "binding_not_found" }); + expect((await store.read(projectId)).requestReceipts).toHaveLength(2); + }); + + it("retains a released binding tombstone while an active release receipt references it", async () => { + const root = await fixture(); + const store = new SubsessionCoordinatorStore(root, { + receiptRetentionLimit: 2, + historyTombstoneLimit: 1, + }); + const first = ( + await store.reserveDelegations(identity, delegate("request-1"), target) + ).bindings[0]!; + await store.closeBinding(identity, first.bindingId, first.sessionId); + const second = ( + await store.reserveDelegations( + identity, + delegate("request-2", [ + { delegationKey: "publisher", outcome: "Publish evidence" }, + ]), + target, + ) + ).bindings[0]!; + await store.reserveDelegations( + identity, + delegate("request-3", [ + { delegationKey: "writer", outcome: "Write evidence" }, + ]), + target, + ); + const releaseRequest = release("release-first"); + const released = await store.reserveReleases(identity, releaseRequest); + expect(released.bindings[0]).toMatchObject({ + state: "released", + binding: { bindingId: first.bindingId }, + }); + + await store.closeBinding(identity, second.bindingId, second.sessionId); + await store.reserveDelegations( + identity, + delegate("request-4", [ + { delegationKey: "editor", outcome: "Edit evidence" }, + ]), + target, + ); + + const aggregate = await store.read(projectId); + expect(aggregate.bindingTombstones).toContainEqual( + expect.objectContaining({ bindingId: first.bindingId }), + ); + expect(await store.reserveReleases(identity, releaseRequest)).toMatchObject({ + replayed: true, + bindings: [ + { state: "released", binding: { bindingId: first.bindingId } }, + ], + }); + }); + it("refreshes child context with an idempotent receipt and a new delivery epoch", async () => { const root = await fixture(); const store = new SubsessionCoordinatorStore(root); @@ -698,9 +782,78 @@ describe("SubsessionCoordinatorStore", () => { ), ).rejects.toMatchObject({ code: "live_session_limit_reached" }); expect((await store.read(projectId)).bindings).toHaveLength(2); + await store.reserveReleases( + identity, + release(`release-${sessionState}`), + ); + const closed = await store.closeBinding( + identity, + first.bindingId, + first.sessionId, + ); + expect(closed.sessionState).toBe("closed"); }, ); + it("reclaims released capacity so a sixty-fifth delegation can be reserved", async () => { + const root = await fixture(); + const store = new SubsessionCoordinatorStore(root, { + receiptRetentionLimit: 1, + }); + const bindings = []; + for (let batch = 0; batch < 4; batch += 1) { + const reserved = await store.reserveDelegations( + identity, + delegate( + `capacity-${batch}`, + Array.from({ length: 16 }, (_, index) => ({ + delegationKey: `child-${batch * 16 + index + 1}`, + outcome: `Task ${batch * 16 + index + 1}`, + })), + ), + target, + ); + bindings.push(...reserved.bindings); + } + await expect( + store.reserveDelegations( + identity, + delegate("capacity-65", [ + { delegationKey: "child-65", outcome: "Task 65" }, + ]), + target, + ), + ).rejects.toMatchObject({ code: "live_session_limit_reached" }); + + const released = await store.reserveReleases( + identity, + release("release-capacity", ["child-1"]), + ); + expect(released.bindings[0]).toMatchObject({ + state: "bound", + binding: { bindingId: bindings[0]!.bindingId }, + }); + await store.closeBinding( + identity, + bindings[0]!.bindingId, + bindings[0]!.sessionId, + ); + const sixtyFifth = await store.reserveDelegations( + identity, + delegate("capacity-65", [ + { delegationKey: "child-65", outcome: "Task 65" }, + ]), + target, + ); + + expect(sixtyFifth.bindings[0]!.delegationKey).toBe("child-65"); + const aggregate = await store.read(projectId); + expect(aggregate.bindings).toHaveLength(64); + expect(aggregate.bindingTombstones).toContainEqual( + expect.objectContaining({ bindingId: bindings[0]!.bindingId }), + ); + }); + it("prunes proven terminal deliveries so long-lived focused refresh stays writable", async () => { const root = await fixture(); const store = new SubsessionCoordinatorStore(root, { diff --git a/packages/harness/src/core/subsession-coordinator-store.ts b/packages/harness/src/core/subsession-coordinator-store.ts index 6e8db5ad..aaa25876 100644 --- a/packages/harness/src/core/subsession-coordinator-store.ts +++ b/packages/harness/src/core/subsession-coordinator-store.ts @@ -114,6 +114,7 @@ export interface SubsessionCoordinatorStoreEvent { | "subsession.binding_reserved" | "subsession.duplicate_prevented" | "subsession.spawn_claimed" + | "subsession.released" | "subsession.kickoff_claimed" | "subsession.kickoff_uncertain"; projectId: StudioProjectId; @@ -126,6 +127,22 @@ export interface ReservedDelegations { bindings: readonly SubsessionBindingRecord[]; } +export type ReleasableSubsessionBinding = + | Readonly<{ + state: "bound"; + binding: SubsessionBindingRecord; + }> + | Readonly<{ + state: "released"; + binding: SubsessionCoordinatorBindingTombstone; + }>; + +export interface ReservedReleases { + replayed: boolean; + requestDigest: CanonicalDelegationRequestDigest; + bindings: readonly ReleasableSubsessionBinding[]; +} + export type SpawnClaimResult = | Readonly<{ claimed: true; binding: SubsessionBindingRecord }> | Readonly<{ @@ -500,7 +517,9 @@ function parseReceipt( !identifier(value.parentSessionId) || !identifier(value.requestKey) || !digest(value.requestDigest) || - !["delegate", "refresh-focused-context"].includes(String(value.operation)) || + !["delegate", "refresh-focused-context", "release"].includes( + String(value.operation), + ) || !Array.isArray(value.bindingIds) || value.bindingIds.length > 16 || !value.bindingIds.every((entry) => identifier(entry, "binding")) || @@ -622,10 +641,14 @@ export function parseSubsessionCoordinatorAggregate( throw new SubsessionCoordinatorStoreError("malformed_state"); } if ( - requestReceipts.some(({ bindingIds: ids }) => + requestReceipts.some(({ operation, bindingIds: ids }) => ids.some( (bindingId) => - !bindings.some((binding) => binding.bindingId === bindingId), + !bindings.some((binding) => binding.bindingId === bindingId) && + (operation !== "release" || + !bindingTombstones.some( + (binding) => binding.bindingId === bindingId, + )), ), ) ) { @@ -760,9 +783,13 @@ export class SubsessionCoordinatorStore { }); } if (aggregate.bindingTombstones.length > historyLimit) { - aggregate.bindingTombstones.splice( - 0, - aggregate.bindingTombstones.length - historyLimit, + let remaining = aggregate.bindingTombstones.length - historyLimit; + aggregate.bindingTombstones = aggregate.bindingTombstones.filter( + ({ bindingId }) => { + if (remaining === 0 || referenced.has(bindingId)) return true; + remaining -= 1; + return false; + }, ); } if (reclaimable.length > 0) { @@ -1008,6 +1035,91 @@ export class SubsessionCoordinatorStore { }); } + reserveReleases( + identity: ProjectAgentSession, + rawRequest: unknown, + ): Promise { + const request = parseProjectSubsessionRequest(rawRequest, identity.projectId); + if (request.operation.kind !== "release") + throw new SubsessionCoordinatorStoreError("malformed_state"); + const operation = request.operation; + const requestDigest = computeCanonicalDelegationRequestDigest(request); + return this.transact(identity.projectId, async (aggregate) => { + const sameRequest = ( + receipt: Pick, + ) => + receipt.parentSessionId === identity.sessionId && + receipt.requestKey === request.requestKey; + const resolve = (bindingId: SubsessionBindingId): ReleasableSubsessionBinding => { + const binding = aggregate.bindings.find( + (entry) => entry.bindingId === bindingId, + ); + if (binding) { + if (binding.parentSessionId !== identity.sessionId) + throw new SubsessionCoordinatorStoreError("binding_scope_mismatch"); + return { state: "bound", binding }; + } + const released = aggregate.bindingTombstones.find( + (entry) => entry.bindingId === bindingId, + ); + if (!released || released.parentSessionId !== identity.sessionId) + throw new SubsessionCoordinatorStoreError("malformed_state"); + return { state: "released", binding: released }; + }; + const previous = aggregate.requestReceipts.find(sameRequest); + if (previous) { + if ( + previous.requestDigest !== requestDigest || + previous.operation !== "release" + ) { + throw new SubsessionCoordinatorStoreError("request_key_reused"); + } + return { + value: { + replayed: true, + requestDigest, + bindings: previous.bindingIds.map(resolve), + }, + }; + } + if (aggregate.requestTombstones.some(sameRequest)) + throw new SubsessionCoordinatorStoreError("request_key_expired"); + const bindings = operation.delegationKeys.map( + (delegationKey): ReleasableSubsessionBinding => { + const binding = aggregate.bindings.find( + (entry) => + entry.parentSessionId === identity.sessionId && + entry.delegationKey === delegationKey, + ); + if (binding) return { state: "bound", binding }; + const released = aggregate.bindingTombstones.find( + (entry) => + entry.parentSessionId === identity.sessionId && + entry.delegationKey === delegationKey, + ); + if (released) return { state: "released", binding: released }; + throw new SubsessionCoordinatorStoreError("binding_not_found"); + }, + ); + const now = this.now(); + aggregate.requestReceipts.push({ + parentSessionId: identity.sessionId, + requestKey: request.requestKey, + requestDigest, + operation: "release", + bindingIds: bindings.map(({ binding }) => binding.bindingId), + createdAt: now, + }); + this.compactTerminalHistory(aggregate); + aggregate.recordVersion += 1; + aggregate.updatedAt = now; + return { + value: { replayed: false, requestDigest, bindings }, + next: aggregate, + }; + }); + } + /** Server-only bridge from SessionManager's private two-sided marker. */ closeOwnedBinding(marker: Readonly<{ projectId: StudioProjectId; diff --git a/packages/harness/src/core/subsession-coordinator.test.ts b/packages/harness/src/core/subsession-coordinator.test.ts index a1a6509f..ed2c3852 100644 --- a/packages/harness/src/core/subsession-coordinator.test.ts +++ b/packages/harness/src/core/subsession-coordinator.test.ts @@ -322,6 +322,14 @@ describe("SubsessionCoordinator", () => { ], }, } as const; + const releaseRequest = { + schemaVersion: 1, + requestKey: "release-1", + operation: { + kind: "release", + delegationKeys: ["research"], + }, + } as const; it("creates one ordinary writable child and reuses it on retry", async () => { const { coordinator, caller, manager, spawnPty, telemetry, unsubscribe } = @@ -358,6 +366,120 @@ describe("SubsessionCoordinator", () => { expect(JSON.stringify(telemetry)).not.toContain("Run the focused tests"); }); + it("idempotently releases and closes the exact real child session", async () => { + const { coordinator, caller, manager, store, spawned, telemetry, unsubscribe } = + await fixture(); + const created = await coordinator.execute(caller, request); + const childId = created.results[0]!.sessionId!; + + const releasing = coordinator.execute(caller, releaseRequest); + await vi.waitFor(() => expect(spawned[1]!.pty.kill).toHaveBeenCalledTimes(1)); + spawned[1]!.emitExit(0); + const released = await releasing; + const replay = await coordinator.execute(caller, releaseRequest); + const aggregate = await store.read(projectId); + unsubscribe(); + + expect(released).toMatchObject({ + replayed: false, + results: [{ + delegationKey: "research", + sessionId: childId, + outcome: "released", + sessionState: "closed", + }], + }); + expect(replay).toMatchObject({ + replayed: true, + results: [{ sessionId: childId, outcome: "released" }], + }); + expect(manager.get(childId)).toMatchObject({ status: "exited" }); + expect(aggregate.bindings[0]).toMatchObject({ + sessionId: childId, + sessionState: "closed", + }); + expect(telemetry).toContainEqual( + expect.objectContaining({ + name: "subsession.released", + projectId, + sessionId: childId, + }), + ); + }); + + it.each(["exited", "failed"] as const)( + "releases an already-%s child without spawning or resuming it", + async (terminalState) => { + const { + coordinator, + caller, + manager, + store, + spawned, + spawnPty, + unsubscribe, + } = await fixture(); + const created = await coordinator.execute(caller, request); + const childId = created.results[0]!.sessionId!; + spawned[1]!.emitExit(0); + await manager.flush(); + const binding = (await store.read(projectId)).bindings[0]!; + await store.transitionSession(caller, binding.bindingId, { + expectedLifecycleEpoch: binding.lifecycleEpoch, + expectedSpawnEpoch: binding.spawnEpoch, + expectedRuntimeToken: binding.runtime?.runtimeToken ?? null, + state: terminalState, + }); + + const released = await coordinator.execute(caller, releaseRequest); + unsubscribe(); + + expect(released.results[0]).toMatchObject({ + sessionId: childId, + outcome: "released", + sessionState: "closed", + }); + expect(spawnPty).toHaveBeenCalledTimes(2); + }, + ); + + it("fails closed instead of releasing or killing a manual session", async () => { + const { coordinator, caller, manager, store, spawned, unsubscribe } = + await fixture(); + const manual = await manager.create({ + cwd: "/tmp/manual-project-session", + harness: "claude-code", + }); + const reserved = await store.reserveDelegations(caller, request, { + harness: "claude-code", + projectRoot: "/tmp/delegated-project-session", + }); + const release = await store.reserveReleases(caller, releaseRequest); + vi.spyOn(store, "reserveReleases").mockResolvedValueOnce({ + ...release, + bindings: [{ + state: "bound", + binding: { ...reserved.bindings[0]!, sessionId: manual.id }, + }], + }); + + const result = await coordinator.execute(caller, releaseRequest); + unsubscribe(); + + expect(result.results[0]).toMatchObject({ + outcome: "failed", + error: { + code: "binding_session_mismatch", + retryable: false, + }, + }); + expect(manager.get(manual.id)).toMatchObject({ status: "running" }); + expect(spawned[1]!.pty.kill).not.toHaveBeenCalled(); + expect((await store.read(projectId)).bindings[0]).toMatchObject({ + sessionState: "reserved", + }); + }); + it("converges independent coordinator instances on one child process", async () => { const { coordinator, newCoordinator, caller, manager, spawnPty, unsubscribe } = await fixture(); diff --git a/packages/harness/src/core/subsession-coordinator.ts b/packages/harness/src/core/subsession-coordinator.ts index c1158f2e..2a688fd4 100644 --- a/packages/harness/src/core/subsession-coordinator.ts +++ b/packages/harness/src/core/subsession-coordinator.ts @@ -54,6 +54,7 @@ export interface SubsessionCoordinatorEvent { | "subsession.requested" | "subsession.created" | "subsession.reused" + | "subsession.released" | "subsession.ready" | "subsession.failed" | "subsession.kickoff_submitted" @@ -103,6 +104,13 @@ type RefreshRequest = Omit & { kind: "refresh-focused-context" } >; }>; +type ReleaseRequest = Omit & + Readonly<{ + operation: Extract< + ProjectSubsessionRequest["operation"], + { kind: "release" } + >; + }>; const error = ( code: DelegationError["code"], @@ -195,6 +203,8 @@ export class SubsessionCoordinator { this.emit({ name: "subsession.requested", projectId: identity.projectId }); if (request.operation.kind === "refresh-focused-context") return this.refresh(identity, request as RefreshRequest); + if (request.operation.kind === "release") + return this.release(identity, request as ReleaseRequest); return this.delegate(identity, request as DelegateRequest); } @@ -266,6 +276,88 @@ export class SubsessionCoordinator { }; } + private async release( + identity: ProjectAgentSession, + request: ReleaseRequest, + ): Promise { + let reserved; + try { + reserved = await this.options.store.reserveReleases(identity, request); + } catch (cause) { + throw this.wholeCallError(cause); + } + const results: DelegationItemResult[] = []; + for (const target of reserved.bindings) { + if (target.state === "released") { + results.push(this.releasedResult(target.binding)); + continue; + } + const binding = target.binding; + const scopedIdentity = bindingIdentity(identity, binding); + try { + const privateMarker = this.options.sessionManager.getSubsessionBinding( + binding.sessionId, + ); + const session = this.options.sessionManager.get(binding.sessionId); + if (privateMarker) { + const expected = markerFor( + binding, + binding.runtime?.incarnation ?? privateMarker.incarnation, + ); + if (!this.options.sessionManager.matchesSubsessionBinding(expected)) + throw error( + "binding_session_mismatch", + false, + "inspect_session", + ); + await this.options.sessionManager.closeBound(expected); + } else if ( + session || + binding.runtime !== null || + !["reserved", "spawn-claimed"].includes(binding.sessionState) + ) { + throw error( + "binding_session_mismatch", + false, + "inspect_session", + ); + } + const closed = await this.options.store.closeBinding( + scopedIdentity, + binding.bindingId, + binding.sessionId, + ); + this.emit({ + name: "subsession.released", + projectId: binding.projectId, + sessionId: binding.sessionId, + }); + results.push(this.releasedResult(closed)); + } catch (cause) { + const detail = this.itemError(cause); + this.emit({ + name: + detail.code === "binding_session_mismatch" + ? "subsession.manual_session_protected" + : "subsession.failed", + projectId: binding.projectId, + sessionId: binding.sessionId, + code: detail.code, + }); + results.push(this.failedResult(binding, detail)); + } + } + return { + schemaVersion: 1, + requestKey: request.requestKey, + requestDigest: reserved.requestDigest, + replayed: reserved.replayed, + results: results.sort((left, right) => + left.delegationKey.localeCompare(right.delegationKey), + ), + }; + } + private async reconcileBinding( caller: ProjectAgentSession, initial: SubsessionBindingRecord, @@ -1132,6 +1224,23 @@ export class SubsessionCoordinator { }; } + private releasedResult( + binding: Pick< + SubsessionBindingRecord, + "delegationKey" | "bindingId" | "sessionId" + >, + ): DelegationItemResult { + return { + delegationKey: binding.delegationKey, + bindingId: binding.bindingId, + sessionId: binding.sessionId, + outcome: "released", + sessionState: "closed", + contextState: "none", + kickoffState: "pending", + }; + } + private failedResult( binding: SubsessionBindingRecord, detail: DelegationError, @@ -1184,6 +1293,10 @@ export class SubsessionCoordinator { return new SubsessionCoordinatorError( error("context_not_found", false, "reread"), ); + if (cause.code === "binding_not_found") + return new SubsessionCoordinatorError( + error("session_closed", false, "inspect_session"), + ); if (refresh && cause.code === "lifecycle_conflict") return new SubsessionCoordinatorError( error("context_refresh_conflict", false, "reread"), diff --git a/packages/harness/src/profiles/project-agent.ts b/packages/harness/src/profiles/project-agent.ts index 8eefa94f..5f69865c 100644 --- a/packages/harness/src/profiles/project-agent.ts +++ b/packages/harness/src/profiles/project-agent.ts @@ -12,7 +12,7 @@ Use agent_map_read when the current project architecture is relevant. When the w Keep internal implementation details local: library choices, ordinary implementation steps, incidental model or tool calls, and refactors that do not change a meaningful project boundary do not belong in the Agent Map. Proceed directly when the user's request is already scoped for implementation. -Focused assignments, map-node references, bootstrap context, and focused briefs are context only. They never grant or remove authority. Use project_subsession_delegate when decomposition improves delivery, and never relabel, close, or otherwise reconcile unrelated user-created sessions. +Focused assignments, map-node references, bootstrap context, and focused briefs are context only. They never grant or remove authority. Use project_subsession_delegate when decomposition improves delivery, and release your coordinator-owned child bindings when they are no longer needed. Never relabel, close, or otherwise reconcile unrelated user-created sessions. `; /** diff --git a/packages/harness/src/server/agent-map-mcp-tools.ts b/packages/harness/src/server/agent-map-mcp-tools.ts index 69615cf4..5e90035b 100644 --- a/packages/harness/src/server/agent-map-mcp-tools.ts +++ b/packages/harness/src/server/agent-map-mcp-tools.ts @@ -109,6 +109,10 @@ const projectSubsessionRequestSchema = z.object({ expectedContextDigest: digest, focus: delegationFocusSchema.nullable(), }).strict(), + z.object({ + kind: z.literal("release"), + delegationKeys: z.array(delegationKey).min(1).max(16), + }).strict(), ]), }).strict(); @@ -385,9 +389,9 @@ export function createAgentMapToolServer( server.registerTool( "project_subsession_delegate", { - description: "Create or reuse one or a bounded batch of ordinary writable project subsessions, or refresh exact focused context, using caller-owned idempotency keys.", + description: "Create, reuse, or release a bounded batch of ordinary writable project subsessions, or refresh exact focused context, using caller-owned idempotency keys.", inputSchema: projectSubsessionRequestSchema, - annotations: { readOnlyHint: false, destructiveHint: false, openWorldHint: false }, + annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: false }, }, async (request) => instrument("project_subsession_delegate", async () => { const result = await subsessionCoordinator.execute(identity, request); diff --git a/packages/harness/src/server/agent-map-mcp-wiring.test.ts b/packages/harness/src/server/agent-map-mcp-wiring.test.ts index 5906d531..02b1c95e 100644 --- a/packages/harness/src/server/agent-map-mcp-wiring.test.ts +++ b/packages/harness/src/server/agent-map-mcp-wiring.test.ts @@ -187,6 +187,14 @@ it("uses the actual ephemeral port and revokes private MCP launch authority on e name: "project_subsession_delegate", arguments: delegationArguments, }); + const released = await client.callTool({ + name: "project_subsession_delegate", + arguments: { + schemaVersion: 1, + requestKey: "release-child", + operation: { kind: "release", delegationKeys: ["child"] }, + }, + }); stopReadyBridge(); expect(delegated.isError).not.toBe(true); expect(delegated.structuredContent).toMatchObject({ @@ -200,6 +208,17 @@ it("uses the actual ephemeral port and revokes private MCP launch authority on e sessionId: (delegated.structuredContent as { results: Array<{ sessionId: string }> }).results[0]!.sessionId, }], }); + expect(released.structuredContent).toMatchObject({ + results: [{ outcome: "released", sessionState: "closed" }], + }); + expect(server.sessionManager.isLive( + (delegated.structuredContent as { results: Array<{ sessionId: string }> }) + .results[0]!.sessionId, + )).toBe(false); + expect(server.sessionManager.list().filter(({ id }) => + id === (nested.structuredContent as { results: Array<{ sessionId: string }> }) + .results[0]!.sessionId, + )).toHaveLength(1); expect(server.sessionManager.list()).toHaveLength(3); await client.close(); diff --git a/packages/harness/src/shared/subsession-delegation-codec.test.ts b/packages/harness/src/shared/subsession-delegation-codec.test.ts index bc89721d..0ff5c3e3 100644 --- a/packages/harness/src/shared/subsession-delegation-codec.test.ts +++ b/packages/harness/src/shared/subsession-delegation-codec.test.ts @@ -182,5 +182,43 @@ describe("subsession delegation codec", () => { }, }); }); -}); + it("canonicalizes a bounded release without accepting session ids", () => { + const release = parseProjectSubsessionRequest( + { + schemaVersion: 1, + requestKey: "release-1", + operation: { + kind: "release", + delegationKeys: ["writer", "research"], + }, + }, + projectId, + ); + + expect(release.operation).toEqual({ + kind: "release", + delegationKeys: ["research", "writer"], + }); + let duplicateError: unknown; + try { + parseProjectSubsessionRequest( + { + schemaVersion: 1, + requestKey: "release-2", + operation: { + kind: "release", + delegationKeys: ["research", "research"], + }, + }, + projectId, + ); + } catch (error) { + duplicateError = error; + } + expect(duplicateError).toMatchObject({ + code: "invalid_request", + issues: [{ code: "duplicate_delegation_key" }], + }); + }); +}); diff --git a/packages/harness/src/shared/subsession-delegation-codec.ts b/packages/harness/src/shared/subsession-delegation-codec.ts index d198bc4f..013952e7 100644 --- a/packages/harness/src/shared/subsession-delegation-codec.ts +++ b/packages/harness/src/shared/subsession-delegation-codec.ts @@ -281,6 +281,27 @@ export function parseProjectSubsessionRequest( "operation.focus", ), }; + } else if ( + value.operation.kind === "release" && + hasExactKeys(value.operation, ["kind", "delegationKeys"]) && + Array.isArray(value.operation.delegationKeys) + ) { + if ( + value.operation.delegationKeys.length < 1 || + value.operation.delegationKeys.length > PROJECT_SUBSESSION_DELEGATION_LIMIT + ) { + throw new SubsessionDelegationValidationError("capacity_exceeded", [ + { path: "operation.delegationKeys", code: "delegation_count" }, + ]); + } + if (!value.operation.delegationKeys.every(isKey)) + return invalid("operation.delegationKeys", "invalid_delegation_key"); + const delegationKeys = value.operation.delegationKeys + .map(normalizeText) + .sort((left, right) => left.localeCompare(right)); + if (new Set(delegationKeys).size !== delegationKeys.length) + return invalid("operation.delegationKeys", "duplicate_delegation_key"); + operation = { kind: "release", delegationKeys }; } else { return invalid("operation", "invalid_operation"); } @@ -324,4 +345,3 @@ export function computeSubsessionContextDigest( focus, ) as SubsessionContextDigest; } - diff --git a/packages/harness/src/shared/subsession-delegation.ts b/packages/harness/src/shared/subsession-delegation.ts index 128ad928..97e1cae5 100644 --- a/packages/harness/src/shared/subsession-delegation.ts +++ b/packages/harness/src/shared/subsession-delegation.ts @@ -72,6 +72,10 @@ export type ProjectSubsessionRequest = Readonly<{ expectedContextEpoch: number; expectedContextDigest: SubsessionContextDigest; focus: DelegationFocusRef | null; + }> + | Readonly<{ + kind: "release"; + delegationKeys: readonly string[]; }>; }>; @@ -147,6 +151,7 @@ export type DelegationItemOutcome = | "created" | "reused" | "already-running" + | "released" | "failed"; export type DelegationItemResult = Readonly<{ diff --git a/packages/harness/src/shared/types.ts b/packages/harness/src/shared/types.ts index 005f55d2..9ac32504 100644 --- a/packages/harness/src/shared/types.ts +++ b/packages/harness/src/shared/types.ts @@ -844,6 +844,7 @@ export type AnalyticsEventType = | "subsession.requested" | "subsession.created" | "subsession.reused" + | "subsession.released" | "subsession.ready" | "subsession.failed" | "subsession.kickoff_claimed" From fa5d2f978c660695f7596bb3aaf378e8523cb894 Mon Sep 17 00:00:00 2001 From: Yash Date: Fri, 4 Sep 2026 21:23:23 +0000 Subject: [PATCH 15/19] fix(harness): bound subsession release recovery Refs: SAP-3151 --- .changeset/writable-project-subsessions.md | 17 +- packages/harness/docs/shared-build-plan.md | 21 +- .../harness/src/core/session-manager.test.ts | 76 +++++++ packages/harness/src/core/session-manager.ts | 20 +- .../core/subsession-coordinator-store.test.ts | 190 +++++++++++++++++- .../src/core/subsession-coordinator-store.ts | 128 +++++++++--- .../src/core/subsession-coordinator.test.ts | 90 +++++++++ .../src/core/subsession-coordinator.ts | 101 ++++++++-- .../src/server/agent-map-mcp-wiring.test.ts | 4 + 9 files changed, 577 insertions(+), 70 deletions(-) diff --git a/.changeset/writable-project-subsessions.md b/.changeset/writable-project-subsessions.md index 68c27628..80e6af0e 100644 --- a/.changeset/writable-project-subsessions.md +++ b/.changeset/writable-project-subsessions.md @@ -19,11 +19,14 @@ Manual sessions remain outside coordinator ownership. Consumers should treat `uncertain` kickoff delivery as terminal until an exact persisted acknowledgement arrives, and should use a new request/delegation key when the corresponding canonical content changes. Nested delegation is bounded to four -levels and 64 live or resumable coordinator-owned sessions per project. A -parent can idempotently release its own child bindings by delegation key, -closing the exact coordinator-owned Harness session and recovering capacity -without granting access to manual or foreign sessions. Request, binding, and -acknowledged-delivery history use bounded retention so long-lived projects do -not dead-end on routine delegation, release, or context refreshes. Exited and -failed bindings remain durable for resume or recovery until explicitly +levels and 64 active or explicitly re-referenced coordinator-owned sessions per +project. A parent can idempotently release its own child bindings by delegation +key, closing the exact coordinator-owned Harness session and recovering +capacity without granting access to manual or foreign sessions; unknown keys +converge as already released. After the coordinator close is durable, private +SessionManager ownership proof is pruned so release churn remains bounded +across restart. Request, binding, and acknowledged-delivery history use bounded +retention so long-lived projects do not dead-end on routine delegation, release, +or context refreshes. Exited and failed bindings remain durable for resume or +recovery without holding an active slot until re-referenced, or until explicitly released. diff --git a/packages/harness/docs/shared-build-plan.md b/packages/harness/docs/shared-build-plan.md index a0407cee..4715f48f 100644 --- a/packages/harness/docs/shared-build-plan.md +++ b/packages/harness/docs/shared-build-plan.md @@ -100,11 +100,14 @@ releases one to sixteen ordinary writable sessions. Each child receives the same project-agent prompt, coding capabilities, project tools, and delegation tool, so nested delegation follows the same path. An exact assignment, map node, or brief may focus the child, but focus never changes its tools or authority. -Delegation is bounded to four levels and 64 live or resumable -coordinator-owned sessions per project. A parent can idempotently release its -own child bindings by delegation key to close their real Harness sessions and -recover capacity; it cannot name arbitrary session IDs or release another -parent's or a manual session. +Delegation is bounded to four levels and 64 active or explicitly re-referenced +coordinator-owned sessions per project. Dormant exited or failed bindings retain +their exact resume identity without holding an active slot until they are +re-referenced. A parent can idempotently release its own child bindings by +delegation key to close their real Harness sessions and recover capacity; it +cannot name arbitrary session IDs or release another parent's or a manual +session. Unknown or expired keys converge as already released without exposing +a session identity. Callers provide both a request key and a delegation key. Identity is scoped by the private session capability to the trusted project and parent session. @@ -117,9 +120,11 @@ closed bindings compact into bounded ownership tombstones once no retained receipt references them. Exited and failed bindings remain available for the coordinator's ordinary resume and recovery paths until explicitly released. Release receipts make partial retries converge before the closed binding is -compacted. The oldest tombstones expire as the retention window advances, so -routine delegation, release, and focused-context refreshes cannot permanently -exhaust a project. +compacted. Once the durable coordinator close succeeds, SessionManager prunes +the exact private ownership marker and close tombstone; a failed final cleanup +retains that proof for the next idempotent retry. The oldest tombstones expire +as the retention window advances, so routine delegation, release, and +focused-context refreshes cannot permanently exhaust a project. Proven acknowledged or unsent delivery epochs are likewise pruned when a newer focused-context delivery replaces them; ambiguous delivery evidence is retained. diff --git a/packages/harness/src/core/session-manager.test.ts b/packages/harness/src/core/session-manager.test.ts index 1c5f46a6..fe4af8d3 100644 --- a/packages/harness/src/core/session-manager.test.ts +++ b/packages/harness/src/core/session-manager.test.ts @@ -455,8 +455,84 @@ describe("SessionManager", () => { failCloseWrite = false; await expect(manager.close(sessionId)).resolves.toBe(false); + expect(writeSubsessionBindingRegistry).toHaveBeenCalledTimes(4); + expect(onSubsessionUserClosed).toHaveBeenCalledTimes(2); + }); + + it("prunes durably closed binding proof across release churn and restart", async () => { + const onSubsessionUserClosed = vi.fn(async () => {}); + const { manager, spawns } = makeManager({ onSubsessionUserClosed }); + + for (let index = 0; index < 70; index += 1) { + const sessionId = `00000000-0000-4000-8000-${String(index).padStart(12, "0")}`; + const input = delegatedCreate(sessionId); + await manager.createReserved( + sessionId, + { cwd: input.cwd, harness: input.harness }, + marker(sessionId), + input.trusted, + ); + const closing = manager.closeBound(marker(sessionId)); + spawns[index]!.emitExit(0); + await closing; + expect(manager.getSubsessionBinding(sessionId)).toBeNull(); + } + + expect(onSubsessionUserClosed).toHaveBeenCalledTimes(70); + expect( + JSON.parse( + await readFile(`${sessionsPath}.subsession-bindings.json`, "utf8"), + ), + ).toEqual({ version: 1, markers: {}, closedSessionIds: [] }); + const { manager: restarted } = makeManager({ onSubsessionUserClosed }); + await restarted.init(); + expect(restarted.getSubsessionBinding( + "00000000-0000-4000-8000-000000000000", + )).toBeNull(); + }); + + it("retains exact binding proof when final cleanup fails and prunes it after restart", async () => { + let writeCount = 0; + const writeSubsessionBindingRegistry = vi.fn( + async (file: string, serialized: string) => { + writeCount += 1; + if (writeCount === 3) + throw new Error("injected cleanup persistence failure"); + await writeFile(file, serialized, "utf8"); + }, + ); + const onSubsessionUserClosed = vi.fn(async () => {}); + const { manager, spawns } = makeManager({ + writeSubsessionBindingRegistry, + onSubsessionUserClosed, + }); + const sessionId = "00000000-0000-4000-8000-000000000117"; + const input = delegatedCreate(sessionId); + await manager.createReserved( + sessionId, + { cwd: input.cwd, harness: input.harness }, + marker(sessionId), + input.trusted, + ); + + const closing = manager.closeBound(marker(sessionId)); + spawns[0]!.emitExit(0); + await expect(closing).rejects.toThrow("injected cleanup persistence failure"); + expect(manager.getSubsessionBinding(sessionId)).toEqual(marker(sessionId)); + expect(manager.wasSubsessionClosedByUser(marker(sessionId))).toBe(true); + + const { manager: restarted } = makeManager({ onSubsessionUserClosed }); + await restarted.init(); + expect(restarted.getSubsessionBinding(sessionId)).toEqual(marker(sessionId)); + await expect(restarted.closeBound(marker(sessionId))).resolves.toBe(false); + expect(restarted.getSubsessionBinding(sessionId)).toBeNull(); expect(writeSubsessionBindingRegistry).toHaveBeenCalledTimes(3); expect(onSubsessionUserClosed).toHaveBeenCalledTimes(2); + expect( + JSON.parse( + await readFile(`${sessionsPath}.subsession-bindings.json`, "utf8"), + ), + ).toEqual({ version: 1, markers: {}, closedSessionIds: [] }); }); it("closes only an exact coordinator-owned binding through the trusted path", async () => { diff --git a/packages/harness/src/core/session-manager.ts b/packages/harness/src/core/session-manager.ts index d799d73d..e2f1fbfd 100644 --- a/packages/harness/src/core/session-manager.ts +++ b/packages/harness/src/core/session-manager.ts @@ -1901,6 +1901,7 @@ export class SessionManager { // in-memory tombstone on failure and let a later close retry persistence. const termination = this.kill(id); let persistenceError: unknown; + let coordinatorCloseRecorded = false; if (binding) { try { await this.persistSubsessionBindings(); @@ -1908,12 +1909,29 @@ export class SessionManager { persistenceError = error; } try { - await this.onSubsessionUserClosed?.(binding); + if (this.onSubsessionUserClosed) { + await this.onSubsessionUserClosed(binding); + coordinatorCloseRecorded = true; + } } catch (error) { persistenceError ??= error; } } const killed = await termination; + if (binding && persistenceError === undefined && coordinatorCloseRecorded) { + const current = this.subsessionBindings.get(id); + if (current && sameSubsessionBinding(current, binding)) { + this.subsessionBindings.delete(id); + this.userClosedSubsessions.delete(id); + try { + await this.persistSubsessionBindings(); + } catch (error) { + this.subsessionBindings.set(id, binding); + this.userClosedSubsessions.add(id); + persistenceError = error; + } + } + } if (persistenceError !== undefined) throw persistenceError; return killed; } diff --git a/packages/harness/src/core/subsession-coordinator-store.test.ts b/packages/harness/src/core/subsession-coordinator-store.test.ts index 2259e44a..2adac779 100644 --- a/packages/harness/src/core/subsession-coordinator-store.test.ts +++ b/packages/harness/src/core/subsession-coordinator-store.test.ts @@ -16,7 +16,11 @@ const identity: ProjectAgentSession = { userId: "user-1", sessionId: "parent-session-1", }; -const target = { harness: "codex" as const, projectRoot: "/project/root" }; +const target = { + harness: "codex" as const, + projectRoot: "/project/root", + ownerId: "coordinator-1", +}; const delegate = ( requestKey = "request-1", @@ -144,8 +148,33 @@ describe("SubsessionCoordinatorStore", () => { const foreign = { ...identity, sessionId: "manual-session" }; await expect( store.reserveReleases(foreign, release("foreign-release")), - ).rejects.toMatchObject({ code: "binding_not_found" }); - expect((await store.read(projectId)).requestReceipts).toHaveLength(2); + ).resolves.toMatchObject({ + replayed: false, + bindings: [{ state: "absent", delegationKey: "research" }], + }); + expect((await store.read(projectId)).requestReceipts).toHaveLength(3); + }); + + it("reserves known and unknown release keys independently and replays both", async () => { + const root = await fixture(); + const store = new SubsessionCoordinatorStore(root); + const binding = ( + await store.reserveDelegations(identity, delegate(), target) + ).bindings[0]!; + const request = release("mixed-release", ["missing", "research"]); + + const first = await store.reserveReleases(identity, request); + const replay = await store.reserveReleases(identity, request); + + expect(first.bindings).toEqual([ + { state: "absent", delegationKey: "missing" }, + { state: "bound", binding }, + ]); + expect(replay).toEqual({ ...first, replayed: true }); + expect((await store.read(projectId)).requestReceipts.at(-1)).toMatchObject({ + operation: "release", + bindingIds: [binding.bindingId], + }); }); it("retains a released binding tombstone while an active release receipt references it", async () => { @@ -662,6 +691,22 @@ describe("SubsessionCoordinatorStore", () => { sessionId: first.sessionId, }), ); + await expect( + store.closeOwnedBinding({ + projectId, + parentSessionId: identity.sessionId, + bindingId: first.bindingId, + sessionId: first.sessionId, + }), + ).resolves.toBeUndefined(); + await expect( + store.closeOwnedBinding({ + projectId, + parentSessionId: identity.sessionId, + bindingId: first.bindingId, + sessionId: "foreign-session", + }), + ).rejects.toMatchObject({ code: "binding_not_found" }); await expect( store.reserveDelegations(identity, delegate(), target), ).rejects.toMatchObject({ code: "request_key_expired" }); @@ -715,7 +760,7 @@ describe("SubsessionCoordinatorStore", () => { }); it.each(["exited", "failed"] as const)( - "keeps a %s binding resumable and counted toward live capacity after receipt churn", + "keeps a %s binding resumable and charges capacity only when it is re-referenced", async (sessionState) => { const root = await fixture(); const store = new SubsessionCoordinatorStore(root, { @@ -766,7 +811,7 @@ describe("SubsessionCoordinatorStore", () => { expect(replay.bindings[0]).toMatchObject({ bindingId: first.bindingId, sessionId: first.sessionId, - sessionState, + sessionState: "spawn-claimed", }); expect(aggregate.bindings).toContainEqual(replay.bindings[0]); expect(aggregate.bindingTombstones).not.toContainEqual( @@ -795,6 +840,141 @@ describe("SubsessionCoordinatorStore", () => { }, ); + it("lets a new parent delegate after an old parent's descendants become dormant", async () => { + const root = await fixture(); + const store = new SubsessionCoordinatorStore(root, { liveSessionLimit: 2 }); + const dormant = ( + await store.reserveDelegations(identity, delegate("old-request"), target) + ).bindings[0]!; + const claim = await store.claimSpawn(identity, dormant.bindingId, { + ownerId: "coordinator-1", + expectedLifecycleEpoch: dormant.lifecycleEpoch, + expectedSpawnEpoch: dormant.spawnEpoch, + }); + if (!claim.claimed || !claim.binding.spawnClaim) + throw new Error("spawn claim was not acquired"); + const starting = await store.attachSpawnedRuntime( + identity, + dormant.bindingId, + { + claimId: claim.binding.spawnClaim.claimId, + spawnEpoch: claim.binding.spawnEpoch, + runtimeToken: "runtime-dormant", + incarnation: 1, + }, + ); + await store.transitionSession(identity, dormant.bindingId, { + expectedLifecycleEpoch: starting.lifecycleEpoch, + expectedSpawnEpoch: starting.spawnEpoch, + expectedRuntimeToken: "runtime-dormant", + state: "exited", + }); + + const newParent = { ...identity, sessionId: "parent-session-2" }; + const active = await store.reserveDelegations( + newParent, + delegate("new-request", [ + { delegationKey: "publisher", outcome: "Publish evidence" }, + { delegationKey: "writer", outcome: "Write evidence" }, + ]), + target, + ); + + expect(active.bindings).toHaveLength(2); + expect((await store.read(projectId)).bindings).toContainEqual( + expect.objectContaining({ + bindingId: dormant.bindingId, + sessionState: "exited", + }), + ); + await expect( + store.reserveDelegations( + newParent, + delegate("new-request-2", [ + { delegationKey: "editor", outcome: "Edit evidence" }, + ]), + target, + ), + ).rejects.toMatchObject({ code: "live_session_limit_reached" }); + }); + + it("fences a dormant resume racing a new parent's active reservation", async () => { + const root = await fixture(); + const firstStore = new SubsessionCoordinatorStore(root, { + liveSessionLimit: 1, + }); + const secondStore = new SubsessionCoordinatorStore(root, { + liveSessionLimit: 1, + }); + const dormant = ( + await firstStore.reserveDelegations( + identity, + delegate("old-request"), + target, + ) + ).bindings[0]!; + const claim = await firstStore.claimSpawn(identity, dormant.bindingId, { + ownerId: "coordinator-1", + expectedLifecycleEpoch: dormant.lifecycleEpoch, + expectedSpawnEpoch: dormant.spawnEpoch, + }); + if (!claim.claimed || !claim.binding.spawnClaim) + throw new Error("spawn claim was not acquired"); + const starting = await firstStore.attachSpawnedRuntime( + identity, + dormant.bindingId, + { + claimId: claim.binding.spawnClaim.claimId, + spawnEpoch: claim.binding.spawnEpoch, + runtimeToken: "runtime-dormant-race", + incarnation: 1, + }, + ); + await firstStore.transitionSession(identity, dormant.bindingId, { + expectedLifecycleEpoch: starting.lifecycleEpoch, + expectedSpawnEpoch: starting.spawnEpoch, + expectedRuntimeToken: "runtime-dormant-race", + state: "exited", + }); + + const newParent = { ...identity, sessionId: "parent-session-2" }; + const results = await Promise.allSettled([ + firstStore.reserveDelegations( + identity, + delegate("resume-request"), + target, + ), + secondStore.reserveDelegations( + newParent, + delegate("new-request", [ + { delegationKey: "publisher", outcome: "Publish evidence" }, + ]), + { ...target, ownerId: "coordinator-2" }, + ), + ]); + + expect(results.filter(({ status }) => status === "fulfilled")).toHaveLength(1); + expect(results.filter(({ status }) => status === "rejected")).toHaveLength(1); + expect(results.find(({ status }) => status === "rejected")).toMatchObject({ + reason: { code: "live_session_limit_reached" }, + }); + const aggregate = await firstStore.read(projectId); + expect( + aggregate.bindings.filter(({ sessionState }) => + [ + "reserved", + "spawn-claimed", + "starting", + "awaiting-ready", + "ready", + ].includes(sessionState), + ), + ).toHaveLength(1); + expect(aggregate.bindings).toContainEqual( + expect.objectContaining({ bindingId: dormant.bindingId }), + ); + }); + it("reclaims released capacity so a sixty-fifth delegation can be reserved", async () => { const root = await fixture(); const store = new SubsessionCoordinatorStore(root, { diff --git a/packages/harness/src/core/subsession-coordinator-store.ts b/packages/harness/src/core/subsession-coordinator-store.ts index aaa25876..36fe2567 100644 --- a/packages/harness/src/core/subsession-coordinator-store.ts +++ b/packages/harness/src/core/subsession-coordinator-store.ts @@ -114,7 +114,6 @@ export interface SubsessionCoordinatorStoreEvent { | "subsession.binding_reserved" | "subsession.duplicate_prevented" | "subsession.spawn_claimed" - | "subsession.released" | "subsession.kickoff_claimed" | "subsession.kickoff_uncertain"; projectId: StudioProjectId; @@ -135,6 +134,10 @@ export type ReleasableSubsessionBinding = | Readonly<{ state: "released"; binding: SubsessionCoordinatorBindingTombstone; + }> + | Readonly<{ + state: "absent"; + delegationKey: string; }>; export interface ReservedReleases { @@ -1074,11 +1077,19 @@ export class SubsessionCoordinatorStore { ) { throw new SubsessionCoordinatorStoreError("request_key_reused"); } + const resolved = previous.bindingIds.map(resolve); return { value: { replayed: true, requestDigest, - bindings: previous.bindingIds.map(resolve), + bindings: operation.delegationKeys.map( + (delegationKey) => + resolved.find( + (entry) => + entry.state !== "absent" && + entry.binding.delegationKey === delegationKey, + ) ?? { state: "absent", delegationKey }, + ), }, }; } @@ -1098,7 +1109,7 @@ export class SubsessionCoordinatorStore { entry.delegationKey === delegationKey, ); if (released) return { state: "released", binding: released }; - throw new SubsessionCoordinatorStoreError("binding_not_found"); + return { state: "absent", delegationKey }; }, ); const now = this.now(); @@ -1107,7 +1118,9 @@ export class SubsessionCoordinatorStore { requestKey: request.requestKey, requestDigest, operation: "release", - bindingIds: bindings.map(({ binding }) => binding.bindingId), + bindingIds: bindings.flatMap((entry) => + entry.state === "absent" ? [] : [entry.binding.bindingId], + ), createdAt: now, }); this.compactTerminalHistory(aggregate); @@ -1126,16 +1139,41 @@ export class SubsessionCoordinatorStore { parentSessionId: string; bindingId: string; sessionId: string; - }>): Promise { - return this.closeBinding( - { - projectId: marker.projectId, - userId: "studio-subsession-coordinator", - sessionId: marker.parentSessionId, - }, - marker.bindingId as SubsessionBindingId, - marker.sessionId, - ); + }>): Promise { + return this.transact(marker.projectId, async (aggregate) => { + const binding = aggregate.bindings.find( + ({ bindingId }) => bindingId === marker.bindingId, + ); + if (!binding) { + const tombstone = aggregate.bindingTombstones.find( + ({ bindingId }) => bindingId === marker.bindingId, + ); + if ( + tombstone?.parentSessionId === marker.parentSessionId && + tombstone.sessionId === marker.sessionId + ) { + return { value: undefined }; + } + throw new SubsessionCoordinatorStoreError("binding_not_found"); + } + if ( + binding.parentSessionId !== marker.parentSessionId || + binding.sessionId !== marker.sessionId + ) { + throw new SubsessionCoordinatorStoreError("binding_scope_mismatch"); + } + if (binding.sessionState === "closed") return { value: undefined }; + const now = this.now(); + binding.sessionState = "closed"; + binding.lifecycleEpoch += 1; + binding.spawnClaim = null; + binding.runtime = null; + binding.updatedAt = now; + this.compactTerminalHistory(aggregate); + aggregate.recordVersion += 1; + aggregate.updatedAt = now; + return { value: undefined, next: aggregate }; + }); } refreshFocusedContext( @@ -1248,7 +1286,11 @@ export class SubsessionCoordinatorStore { async reserveDelegations( identity: ProjectAgentSession, rawRequest: unknown, - target: Readonly<{ harness: HarnessKind; projectRoot: string }>, + target: Readonly<{ + harness: HarnessKind; + projectRoot: string; + ownerId: string; + }>, ): Promise { parseProjectAgentActorRef({ userId: identity.userId, @@ -1260,13 +1302,32 @@ export class SubsessionCoordinatorStore { if ( !["claude-code", "codex"].includes(target.harness) || !path.isAbsolute(target.projectRoot) || - target.projectRoot.includes("\0") + target.projectRoot.includes("\0") || + !identifier(target.ownerId) ) { throw new SubsessionCoordinatorStoreError("malformed_state"); } const requestDigest = computeCanonicalDelegationRequestDigest(request); const operation = request.operation; return this.transact(identity.projectId, async (aggregate) => { + const now = this.now(); + const activeCount = () => + aggregate.bindings.filter(({ sessionState }) => + [ + "reserved", + "spawn-claimed", + "starting", + "awaiting-ready", + "ready", + ].includes(sessionState), + ).length; + const activateDormant = (binding: MutableBinding): void => { + binding.spawnEpoch += 1; + binding.lifecycleEpoch += 1; + binding.spawnClaim = this.claim(now, target.ownerId) as MutableClaim; + binding.sessionState = "spawn-claimed"; + binding.updatedAt = now; + }; const sameRequest = ( receipt: Pick, ): boolean => @@ -1293,32 +1354,39 @@ export class SubsessionCoordinatorStore { throw new SubsessionCoordinatorStoreError("malformed_state"); return binding; }); + const dormant = bindings.filter(({ sessionState }) => + ["exited", "failed"].includes(sessionState), + ); + if ( + activeCount() + dormant.length > + (this.options.liveSessionLimit ?? + PROJECT_SUBSESSION_LIVE_SESSION_LIMIT) + ) { + throw new SubsessionCoordinatorStoreError( + "live_session_limit_reached", + ); + } + for (const binding of dormant) activateDormant(binding); this.emit({ name: "subsession.duplicate_prevented", projectId: identity.projectId, count: bindings.length, }); + if (dormant.length === 0) + return { value: { replayed: true, requestDigest, bindings } }; + aggregate.recordVersion += 1; + aggregate.updatedAt = now; return { value: { replayed: true, requestDigest, bindings }, + next: aggregate, }; } if (aggregate.requestTombstones.some(sameRequest)) throw new SubsessionCoordinatorStoreError("request_key_expired"); this.compactTerminalHistory(aggregate); - const now = this.now(); const bindings: SubsessionBindingRecord[] = []; let created = 0; - const live = aggregate.bindings.filter(({ sessionState }) => - [ - "reserved", - "spawn-claimed", - "starting", - "awaiting-ready", - "ready", - "exited", - "failed", - ].includes(sessionState), - ).length; + const live = activeCount(); const parentBinding = aggregate.bindings.find( ({ sessionId }) => sessionId === identity.sessionId, ); @@ -1348,6 +1416,10 @@ export class SubsessionCoordinatorStore { ); if (existing.sessionState === "closed") throw new SubsessionCoordinatorStoreError("session_closed"); + if (["exited", "failed"].includes(existing.sessionState)) { + additionalLive += 1; + activateDormant(existing); + } bindings.push(existing); continue; } diff --git a/packages/harness/src/core/subsession-coordinator.test.ts b/packages/harness/src/core/subsession-coordinator.test.ts index ed2c3852..6eb763b8 100644 --- a/packages/harness/src/core/subsession-coordinator.test.ts +++ b/packages/harness/src/core/subsession-coordinator.test.ts @@ -176,6 +176,10 @@ describe("SubsessionCoordinator", () => { async function fixture( resumable = false, childIdentityState?: "ready" | "ambiguous", + managerOptions: Pick< + ConstructorParameters[0], + "writeSubsessionBindingRegistry" + > = {}, ) { const root = await fs.mkdtemp(path.join(os.tmpdir(), "subsession-service-")); roots.push(root); @@ -206,6 +210,7 @@ describe("SubsessionCoordinator", () => { await closeStore.current?.closeOwnedBinding(marker); }, resolveAgentMapIdentity: async (_sessionId, _cwd, persisted) => persisted, + ...managerOptions, }); managers.push(manager); await manager.init(); @@ -394,6 +399,7 @@ describe("SubsessionCoordinator", () => { results: [{ sessionId: childId, outcome: "released" }], }); expect(manager.get(childId)).toMatchObject({ status: "exited" }); + expect(manager.getSubsessionBinding(childId)).toBeNull(); expect(aggregate.bindings[0]).toMatchObject({ sessionId: childId, sessionState: "closed", @@ -407,6 +413,89 @@ describe("SubsessionCoordinator", () => { ); }); + it("finishes private binding cleanup when a release is retried after a partial failure", async () => { + let writeCount = 0; + let failCleanup = true; + const writeSubsessionBindingRegistry = vi.fn( + async (file: string, serialized: string) => { + writeCount += 1; + if (failCleanup && writeCount === 3) + throw new Error("injected cleanup persistence failure"); + await fs.writeFile(file, serialized, "utf8"); + }, + ); + const { coordinator, caller, manager, spawned, unsubscribe } = await fixture( + false, + undefined, + { writeSubsessionBindingRegistry }, + ); + const created = await coordinator.execute(caller, request); + const childId = created.results[0]!.sessionId!; + + const releasing = coordinator.execute(caller, releaseRequest); + await vi.waitFor(() => expect(spawned[1]!.pty.kill).toHaveBeenCalledTimes(1)); + spawned[1]!.emitExit(0); + const partial = await releasing; + expect(partial.results[0]).toMatchObject({ + sessionId: childId, + outcome: "failed", + }); + expect(manager.getSubsessionBinding(childId)).not.toBeNull(); + + failCleanup = false; + const retried = await coordinator.execute(caller, releaseRequest); + unsubscribe(); + + expect(retried).toMatchObject({ + replayed: true, + results: [{ sessionId: childId, outcome: "released" }], + }); + expect(manager.getSubsessionBinding(childId)).toBeNull(); + expect(writeSubsessionBindingRegistry).toHaveBeenCalledTimes(5); + }); + + it("releases known children in a mixed batch and treats unknown keys as already released", async () => { + const { coordinator, caller, spawned, unsubscribe } = await fixture(); + const created = await coordinator.execute(caller, request); + const childId = created.results[0]!.sessionId!; + const mixed = { + schemaVersion: 1, + requestKey: "release-mixed", + operation: { + kind: "release", + delegationKeys: ["missing", "research"], + }, + } as const; + + const releasing = coordinator.execute(caller, mixed); + await vi.waitFor(() => expect(spawned[1]!.pty.kill).toHaveBeenCalledTimes(1)); + spawned[1]!.emitExit(0); + const released = await releasing; + const replay = await coordinator.execute(caller, mixed); + unsubscribe(); + + expect(released.results).toEqual([ + expect.objectContaining({ + delegationKey: "missing", + bindingId: null, + sessionId: null, + outcome: "released", + }), + expect.objectContaining({ + delegationKey: "research", + sessionId: childId, + outcome: "released", + }), + ]); + expect(replay).toMatchObject({ + replayed: true, + results: [ + { delegationKey: "missing", outcome: "released" }, + { delegationKey: "research", sessionId: childId, outcome: "released" }, + ], + }); + }); + it.each(["exited", "failed"] as const)( "releases an already-%s child without spawning or resuming it", async (terminalState) => { @@ -453,6 +542,7 @@ describe("SubsessionCoordinator", () => { const reserved = await store.reserveDelegations(caller, request, { harness: "claude-code", projectRoot: "/tmp/delegated-project-session", + ownerId: "coordinator-test", }); const release = await store.reserveReleases(caller, releaseRequest); vi.spyOn(store, "reserveReleases").mockResolvedValueOnce({ diff --git a/packages/harness/src/core/subsession-coordinator.ts b/packages/harness/src/core/subsession-coordinator.ts index 2a688fd4..923368a1 100644 --- a/packages/harness/src/core/subsession-coordinator.ts +++ b/packages/harness/src/core/subsession-coordinator.ts @@ -233,6 +233,7 @@ export class SubsessionCoordinator { reserved = await this.options.store.reserveDelegations(identity, request, { harness: caller.harness, projectRoot: caller.cwd, + ownerId: this.ownerId, }); } catch (cause) { throw this.wholeCallError(cause); @@ -288,8 +289,56 @@ export class SubsessionCoordinator { } const results: DelegationItemResult[] = []; for (const target of reserved.bindings) { + if (target.state === "absent") { + results.push({ + delegationKey: target.delegationKey, + bindingId: null, + sessionId: null, + outcome: "released", + sessionState: "closed", + contextState: "none", + kickoffState: "pending", + }); + continue; + } if (target.state === "released") { - results.push(this.releasedResult(target.binding)); + const binding = target.binding; + try { + const privateMarker = + this.options.sessionManager.getSubsessionBinding(binding.sessionId); + if (privateMarker) { + if ( + privateMarker.projectId !== identity.projectId || + privateMarker.parentSessionId !== binding.parentSessionId || + privateMarker.bindingId !== binding.bindingId || + privateMarker.sessionId !== binding.sessionId + ) { + throw error( + "binding_session_mismatch", + false, + "inspect_session", + ); + } + await this.options.sessionManager.closeBound(privateMarker); + } + results.push(this.releasedResult(binding)); + } catch (cause) { + const detail = this.itemError(cause); + this.emit({ + name: + detail.code === "binding_session_mismatch" + ? "subsession.manual_session_protected" + : "subsession.failed", + projectId: identity.projectId, + sessionId: binding.sessionId, + code: detail.code, + }); + results.push({ + ...this.releasedResult(binding), + outcome: "failed", + error: detail, + }); + } continue; } const binding = target.binding; @@ -311,6 +360,8 @@ export class SubsessionCoordinator { "inspect_session", ); await this.options.sessionManager.closeBound(expected); + } else if (binding.sessionState === "closed") { + // The durable coordinator close won before private marker cleanup. } else if ( session || binding.runtime !== null || @@ -674,15 +725,19 @@ export class SubsessionCoordinator { throw error("session_unreachable", true, "inspect_session"); } - let claim = await this.options.store.claimSpawn( - identity, - binding.bindingId, - { - ownerId: this.ownerId, - expectedLifecycleEpoch: binding.lifecycleEpoch, - expectedSpawnEpoch: binding.spawnEpoch, - }, - ); + let claim = + binding.sessionState === "spawn-claimed" && + binding.spawnClaim?.ownerId === this.ownerId + ? { claimed: true as const, binding } + : await this.options.store.claimSpawn( + identity, + binding.bindingId, + { + ownerId: this.ownerId, + expectedLifecycleEpoch: binding.lifecycleEpoch, + expectedSpawnEpoch: binding.spawnEpoch, + }, + ); if (!claim.claimed) { if (claim.reason !== "expired-requires-inspection") return { binding: claim.binding, created: false, reused: true }; @@ -786,17 +841,21 @@ export class SubsessionCoordinator { } let claim; if (binding.sessionState === "spawn-claimed" && binding.spawnClaim) { - if (binding.spawnClaim.expiresAt > new Date().toISOString()) return binding; - claim = await this.options.store.takeoverExpiredSpawnClaim( - identity, - binding.bindingId, - { - ownerId: this.ownerId, - expiredClaimId: binding.spawnClaim.claimId, - expectedLifecycleEpoch: binding.lifecycleEpoch, - expectedSpawnEpoch: binding.spawnEpoch, - }, - ); + if (binding.spawnClaim.ownerId === this.ownerId) { + claim = { claimed: true as const, binding }; + } else { + if (binding.spawnClaim.expiresAt > new Date().toISOString()) return binding; + claim = await this.options.store.takeoverExpiredSpawnClaim( + identity, + binding.bindingId, + { + ownerId: this.ownerId, + expiredClaimId: binding.spawnClaim.claimId, + expectedLifecycleEpoch: binding.lifecycleEpoch, + expectedSpawnEpoch: binding.spawnEpoch, + }, + ); + } } else { claim = await this.options.store.claimSpawn( identity, diff --git a/packages/harness/src/server/agent-map-mcp-wiring.test.ts b/packages/harness/src/server/agent-map-mcp-wiring.test.ts index 02b1c95e..5346b931 100644 --- a/packages/harness/src/server/agent-map-mcp-wiring.test.ts +++ b/packages/harness/src/server/agent-map-mcp-wiring.test.ts @@ -215,6 +215,10 @@ it("uses the actual ephemeral port and revokes private MCP launch authority on e (delegated.structuredContent as { results: Array<{ sessionId: string }> }) .results[0]!.sessionId, )).toBe(false); + expect(server.sessionManager.getSubsessionBinding( + (delegated.structuredContent as { results: Array<{ sessionId: string }> }) + .results[0]!.sessionId, + )).toBeNull(); expect(server.sessionManager.list().filter(({ id }) => id === (nested.structuredContent as { results: Array<{ sessionId: string }> }) .results[0]!.sessionId, From 7efe577da3a2615ca3b9826bcd3b518a63dae4e8 Mon Sep 17 00:00:00 2001 From: Yash Date: Fri, 4 Sep 2026 21:45:01 +0000 Subject: [PATCH 16/19] fix(harness): reclaim orphaned dormant subsessions Refs: SAP-3151 --- .changeset/writable-project-subsessions.md | 6 +- packages/harness/docs/shared-build-plan.md | 30 ++- .../core/subsession-coordinator-store.test.ts | 114 +++++++++++ .../src/core/subsession-coordinator-store.ts | 183 +++++++++++++++++- .../src/core/subsession-coordinator.test.ts | 156 ++++++++++++++- .../src/core/subsession-coordinator.ts | 67 ++++++- .../harness/src/profiles/project-agent.ts | 2 +- .../harness/src/server/agent-map-mcp-tools.ts | 6 +- .../harness/src/server/agent-map-mcp.test.ts | 14 ++ .../subsession-delegation-codec.test.ts | 37 ++++ .../src/shared/subsession-delegation-codec.ts | 11 ++ .../src/shared/subsession-delegation.ts | 9 + 12 files changed, 607 insertions(+), 28 deletions(-) diff --git a/.changeset/writable-project-subsessions.md b/.changeset/writable-project-subsessions.md index 80e6af0e..d1776da7 100644 --- a/.changeset/writable-project-subsessions.md +++ b/.changeset/writable-project-subsessions.md @@ -29,4 +29,8 @@ across restart. Request, binding, and acknowledged-delivery history use bounded retention so long-lived projects do not dead-end on routine delegation, release, or context refreshes. Exited and failed bindings remain durable for resume or recovery without holding an active slot until re-referenced, or until explicitly -released. +released. Any current project agent may explicitly reclaim up to sixteen dormant +coordinator-owned bindings whose original parent is absent or exited, without +supplying raw session IDs. This destructive recovery compacts coordinator and +private ownership state while retaining the ordinary Harness session history; +the released binding is no longer automatically resumable. diff --git a/packages/harness/docs/shared-build-plan.md b/packages/harness/docs/shared-build-plan.md index 4715f48f..b83cf837 100644 --- a/packages/harness/docs/shared-build-plan.md +++ b/packages/harness/docs/shared-build-plan.md @@ -115,19 +115,29 @@ Identical retries converge on the same durable binding and real Harness session ID; changing canonical request or binding content under an existing key fails explicitly. All binding IDs and session IDs for a bounded batch are reserved in one durable transaction before the first process is spawned. -Older request receipts compact into bounded key tombstones, and explicitly -closed bindings compact into bounded ownership tombstones once no retained -receipt references them. Exited and failed bindings remain available for the -coordinator's ordinary resume and recovery paths until explicitly released. -Release receipts make partial retries converge before the closed binding is -compacted. Once the durable coordinator close succeeds, SessionManager prunes -the exact private ownership marker and close tombstone; a failed final cleanup -retains that proof for the next idempotent retry. The oldest tombstones expire -as the retention window advances, so routine delegation, release, and -focused-context refreshes cannot permanently exhaust a project. +Older request receipts compact into bounded key tombstones. User-closed +bindings compact into bounded ownership tombstones once no retained receipt +references them; an explicit release finalizes immediately to the same +tombstone while its receipt retains deterministic replay. Exited and failed +bindings remain available for the coordinator's ordinary resume and recovery +paths until explicitly released. Once the durable coordinator close succeeds, +SessionManager prunes the exact private ownership marker and close tombstone; a +failed final cleanup retains that proof for the next idempotent retry. The +oldest tombstones expire as the retention window advances, so routine +delegation, release, and focused-context refreshes cannot permanently exhaust a +project. Proven acknowledged or unsent delivery epochs are likewise pruned when a newer focused-context delivery replaces them; ambiguous delivery evidence is retained. +If exited or failed bindings outlive a dead or unreachable original parent and +fill durable binding history, any current project session may explicitly invoke +the bounded `release-dormant` operation. The coordinator selects at most sixteen +eligible records inside the capability-derived project; the request accepts no +session IDs and never selects active bindings or manual sessions. This operation +is destructive: it retains the ordinary Harness conversation/session history, +but compacts the coordinator binding and ends automatic resume through that +binding. Request receipts make the sweep idempotent and restart-safe. + The coordinator waits for canonical adapter readiness and exact transcript identity, then uses fenced spawn and delivery epochs to submit one kickoff. Delivery states distinguish pending, claimed, submitted without acknowledgement, diff --git a/packages/harness/src/core/subsession-coordinator-store.test.ts b/packages/harness/src/core/subsession-coordinator-store.test.ts index 2adac779..9c4df54c 100644 --- a/packages/harness/src/core/subsession-coordinator-store.test.ts +++ b/packages/harness/src/core/subsession-coordinator-store.test.ts @@ -42,6 +42,11 @@ const release = ( requestKey, operation: { kind: "release", delegationKeys }, }); +const releaseDormant = (requestKey = "release-dormant-1", limit = 16) => ({ + schemaVersion: 1, + requestKey, + operation: { kind: "release-dormant", limit }, +}); describe("SubsessionCoordinatorStore", () => { const roots: string[] = []; @@ -975,6 +980,115 @@ describe("SubsessionCoordinatorStore", () => { ); }); + it("reclaims a bounded dormant binding at history capacity and preserves replay", async () => { + const root = await fixture(); + const store = new SubsessionCoordinatorStore(root, { + bindingLimit: 2, + liveSessionLimit: 2, + }); + const dormant = ( + await store.reserveDelegations( + identity, + delegate("old-request", [ + { delegationKey: "research", outcome: "Collect evidence" }, + { delegationKey: "publisher", outcome: "Publish evidence" }, + ]), + target, + ) + ).bindings; + for (const [index, binding] of dormant.entries()) { + const claim = await store.claimSpawn(identity, binding.bindingId, { + ownerId: "coordinator-1", + expectedLifecycleEpoch: binding.lifecycleEpoch, + expectedSpawnEpoch: binding.spawnEpoch, + }); + if (!claim.claimed || !claim.binding.spawnClaim) + throw new Error("spawn claim was not acquired"); + const starting = await store.attachSpawnedRuntime( + identity, + binding.bindingId, + { + claimId: claim.binding.spawnClaim.claimId, + spawnEpoch: claim.binding.spawnEpoch, + runtimeToken: `runtime-history-${index}`, + incarnation: 1, + }, + ); + await store.transitionSession(identity, binding.bindingId, { + expectedLifecycleEpoch: starting.lifecycleEpoch, + expectedSpawnEpoch: starting.spawnEpoch, + expectedRuntimeToken: `runtime-history-${index}`, + state: index === 0 ? "exited" : "failed", + }); + } + + const newParent = { ...identity, sessionId: "parent-session-2" }; + await expect( + store.reserveDelegations( + newParent, + delegate("new-request", [ + { delegationKey: "writer", outcome: "Write evidence" }, + ]), + target, + ), + ).rejects.toMatchObject({ code: "history_quota_exceeded" }); + + const request = releaseDormant("sweep-at-cap", 1); + const reserved = await store.reserveDormantReleases( + newParent, + request, + [dormant[0]!.bindingId], + ); + expect(reserved.bindings).toEqual([ + { + state: "bound", + binding: expect.objectContaining({ + bindingId: dormant[0]!.bindingId, + sessionState: "closed", + }), + }, + ]); + await store.closeBinding( + identity, + dormant[0]!.bindingId, + dormant[0]!.sessionId, + ); + await store.finalizeReleasedBinding( + identity, + dormant[0]!.bindingId, + dormant[0]!.sessionId, + ); + const replay = await store.reserveDormantReleases( + newParent, + request, + [], + ); + expect(replay).toMatchObject({ + replayed: true, + bindings: [ + { + state: "released", + binding: { bindingId: dormant[0]!.bindingId }, + }, + ], + }); + + const created = await store.reserveDelegations( + newParent, + delegate("new-request", [ + { delegationKey: "writer", outcome: "Write evidence" }, + ]), + target, + ); + expect(created.bindings[0]).toMatchObject({ delegationKey: "writer" }); + expect((await store.read(projectId)).bindings).toContainEqual( + expect.objectContaining({ + bindingId: dormant[1]!.bindingId, + sessionState: "failed", + }), + ); + }); + it("reclaims released capacity so a sixty-fifth delegation can be reserved", async () => { const root = await fixture(); const store = new SubsessionCoordinatorStore(root, { diff --git a/packages/harness/src/core/subsession-coordinator-store.ts b/packages/harness/src/core/subsession-coordinator-store.ts index 36fe2567..4e7c1e90 100644 --- a/packages/harness/src/core/subsession-coordinator-store.ts +++ b/packages/harness/src/core/subsession-coordinator-store.ts @@ -520,7 +520,12 @@ function parseReceipt( !identifier(value.parentSessionId) || !identifier(value.requestKey) || !digest(value.requestDigest) || - !["delegate", "refresh-focused-context", "release"].includes( + ![ + "delegate", + "refresh-focused-context", + "release", + "release-dormant", + ].includes( String(value.operation), ) || !Array.isArray(value.bindingIds) || @@ -644,14 +649,13 @@ export function parseSubsessionCoordinatorAggregate( throw new SubsessionCoordinatorStoreError("malformed_state"); } if ( - requestReceipts.some(({ operation, bindingIds: ids }) => + requestReceipts.some(({ bindingIds: ids }) => ids.some( (bindingId) => !bindings.some((binding) => binding.bindingId === bindingId) && - (operation !== "release" || - !bindingTombstones.some( - (binding) => binding.bindingId === bindingId, - )), + !bindingTombstones.some( + (binding) => binding.bindingId === bindingId, + ), ), ) ) { @@ -703,6 +707,7 @@ export class SubsessionCoordinatorStore { claimTtlMs?: number; receiptRetentionLimit?: number; historyTombstoneLimit?: number; + bindingLimit?: number; liveSessionLimit?: number; maxDelegationDepth?: number; onEvent?: (event: SubsessionCoordinatorStoreEvent) => void | Promise; @@ -729,6 +734,16 @@ export class SubsessionCoordinatorStore { return (this.options.generateId ?? randomUUID)(); } + private bindingLimit(): number { + return Math.max( + 1, + Math.min( + this.options.bindingLimit ?? SUBSESSION_COORDINATOR_BINDING_LIMIT, + SUBSESSION_COORDINATOR_BINDING_LIMIT, + ), + ); + } + private compactTerminalHistory(aggregate: MutableAggregate): void { const historyLimit = Math.max( 1, @@ -1038,6 +1053,51 @@ export class SubsessionCoordinatorStore { }); } + /** Compacts an exact durably closed binding while release receipts retain replay. */ + finalizeReleasedBinding( + identity: ProjectAgentSession, + bindingId: SubsessionBindingId, + expectedSessionId: string, + ): Promise { + return this.transact(identity.projectId, async (aggregate) => { + const existing = aggregate.bindingTombstones.find( + (entry) => entry.bindingId === bindingId, + ); + if (existing) { + if ( + existing.parentSessionId !== identity.sessionId || + existing.sessionId !== expectedSessionId + ) { + throw new SubsessionCoordinatorStoreError("binding_scope_mismatch"); + } + return { value: existing }; + } + const binding = this.scopedBinding(aggregate, identity, bindingId); + if (binding.sessionId !== expectedSessionId) + throw new SubsessionCoordinatorStoreError("binding_scope_mismatch"); + if (binding.sessionState !== "closed") + throw new SubsessionCoordinatorStoreError("lifecycle_conflict"); + const tombstone: SubsessionCoordinatorBindingTombstone = { + bindingId: binding.bindingId, + parentSessionId: binding.parentSessionId, + parentBindingId: binding.parentBindingId, + delegationDepth: binding.delegationDepth, + delegationKey: binding.delegationKey, + bindingDigest: binding.bindingDigest, + sessionId: binding.sessionId, + closedAt: binding.updatedAt, + }; + aggregate.bindings = aggregate.bindings.filter( + (entry) => entry.bindingId !== bindingId, + ); + aggregate.bindingTombstones.push(tombstone); + this.compactTerminalHistory(aggregate); + aggregate.recordVersion += 1; + aggregate.updatedAt = this.now(); + return { value: tombstone, next: aggregate }; + }); + } + reserveReleases( identity: ProjectAgentSession, rawRequest: unknown, @@ -1133,6 +1193,113 @@ export class SubsessionCoordinatorStore { }); } + /** + * Reserves an explicit project-scoped cleanup of dormant coordinator-owned + * bindings. Candidate IDs are selected by the trusted coordinator after it + * proves each original parent is absent or exited; they are never accepted + * from the public request. + */ + reserveDormantReleases( + identity: ProjectAgentSession, + rawRequest: unknown, + candidateBindingIds: readonly SubsessionBindingId[], + ): Promise { + const request = parseProjectSubsessionRequest(rawRequest, identity.projectId); + if (request.operation.kind !== "release-dormant") + throw new SubsessionCoordinatorStoreError("malformed_state"); + if ( + candidateBindingIds.length > request.operation.limit || + new Set(candidateBindingIds).size !== candidateBindingIds.length || + !candidateBindingIds.every((bindingId) => + identifier(bindingId, "binding"), + ) + ) { + throw new SubsessionCoordinatorStoreError("malformed_state"); + } + const requestDigest = computeCanonicalDelegationRequestDigest(request); + return this.transact(identity.projectId, async (aggregate) => { + const sameRequest = ( + receipt: Pick, + ) => + receipt.parentSessionId === identity.sessionId && + receipt.requestKey === request.requestKey; + const resolve = ( + bindingId: SubsessionBindingId, + ): ReleasableSubsessionBinding => { + const binding = aggregate.bindings.find( + (entry) => entry.bindingId === bindingId, + ); + if (binding) return { state: "bound", binding }; + const released = aggregate.bindingTombstones.find( + (entry) => entry.bindingId === bindingId, + ); + if (released) return { state: "released", binding: released }; + throw new SubsessionCoordinatorStoreError("malformed_state"); + }; + const previous = aggregate.requestReceipts.find(sameRequest); + if (previous) { + if ( + previous.requestDigest !== requestDigest || + previous.operation !== "release-dormant" + ) { + throw new SubsessionCoordinatorStoreError("request_key_reused"); + } + return { + value: { + replayed: true, + requestDigest, + bindings: previous.bindingIds.map(resolve), + }, + }; + } + if (aggregate.requestTombstones.some(sameRequest)) + throw new SubsessionCoordinatorStoreError("request_key_expired"); + this.compactTerminalHistory(aggregate); + const now = this.now(); + const bindings: ReleasableSubsessionBinding[] = []; + for (const bindingId of candidateBindingIds) { + const binding = aggregate.bindings.find( + (entry) => entry.bindingId === bindingId, + ); + if (binding) { + if (["exited", "failed"].includes(binding.sessionState)) { + // The explicit destructive boundary and request receipt commit in + // the same transaction. A concurrent resume must lose this fence + // before any exact private ownership marker is removed. + binding.sessionState = "closed"; + binding.lifecycleEpoch += 1; + binding.spawnClaim = null; + binding.runtime = null; + binding.updatedAt = now; + bindings.push({ state: "bound", binding }); + } + continue; + } + const released = aggregate.bindingTombstones.find( + (entry) => entry.bindingId === bindingId, + ); + if (released) bindings.push({ state: "released", binding: released }); + } + aggregate.requestReceipts.push({ + parentSessionId: identity.sessionId, + requestKey: request.requestKey, + requestDigest, + operation: "release-dormant", + bindingIds: bindings.flatMap((entry) => + entry.state === "absent" ? [] : [entry.binding.bindingId], + ), + createdAt: now, + }); + this.compactTerminalHistory(aggregate); + aggregate.recordVersion += 1; + aggregate.updatedAt = now; + return { + value: { replayed: false, requestDigest, bindings }, + next: aggregate, + }; + }); + } + /** Server-only bridge from SessionManager's private two-sided marker. */ closeOwnedBinding(marker: Readonly<{ projectId: StudioProjectId; @@ -1433,9 +1600,7 @@ export class SubsessionCoordinatorStore { throw new SubsessionCoordinatorStoreError("delegation_key_reused"); throw new SubsessionCoordinatorStoreError("session_closed"); } - if ( - aggregate.bindings.length >= SUBSESSION_COORDINATOR_BINDING_LIMIT - ) { + if (aggregate.bindings.length >= this.bindingLimit()) { throw new SubsessionCoordinatorStoreError("history_quota_exceeded"); } additionalLive += 1; diff --git a/packages/harness/src/core/subsession-coordinator.test.ts b/packages/harness/src/core/subsession-coordinator.test.ts index 6eb763b8..9095f14c 100644 --- a/packages/harness/src/core/subsession-coordinator.test.ts +++ b/packages/harness/src/core/subsession-coordinator.test.ts @@ -180,6 +180,9 @@ describe("SubsessionCoordinator", () => { ConstructorParameters[0], "writeSubsessionBindingRegistry" > = {}, + storeOptions: NonNullable< + ConstructorParameters[1] + > = {}, ) { const root = await fs.mkdtemp(path.join(os.tmpdir(), "subsession-service-")); roots.push(root); @@ -278,6 +281,7 @@ describe("SubsessionCoordinator", () => { }; const store = new SubsessionCoordinatorStore( path.join(root, "agent-map"), + storeOptions, ); closeStore.current = store; const telemetry: unknown[] = []; @@ -335,6 +339,11 @@ describe("SubsessionCoordinator", () => { delegationKeys: ["research"], }, } as const; + const dormantReleaseRequest = { + schemaVersion: 1, + requestKey: "release-dormant-1", + operation: { kind: "release-dormant", limit: 1 }, + } as const; it("creates one ordinary writable child and reuses it on retry", async () => { const { coordinator, caller, manager, spawnPty, telemetry, unsubscribe } = @@ -400,9 +409,8 @@ describe("SubsessionCoordinator", () => { }); expect(manager.get(childId)).toMatchObject({ status: "exited" }); expect(manager.getSubsessionBinding(childId)).toBeNull(); - expect(aggregate.bindings[0]).toMatchObject({ + expect(aggregate.bindingTombstones[0]).toMatchObject({ sessionId: childId, - sessionState: "closed", }); expect(telemetry).toContainEqual( expect.objectContaining({ @@ -496,6 +504,107 @@ describe("SubsessionCoordinator", () => { }); }); + it("lets a new project agent reclaim a dead parent's dormant child at history capacity", async () => { + const { coordinator, caller, manager, store, spawned, unsubscribe } = + await fixture(false, undefined, {}, { bindingLimit: 1 }); + const created = await coordinator.execute(caller, request); + const childId = created.results[0]!.sessionId!; + spawned[1]!.emitExit(0); + await manager.flush(); + const binding = (await store.read(projectId)).bindings[0]!; + await store.transitionSession(caller, binding.bindingId, { + expectedLifecycleEpoch: binding.lifecycleEpoch, + expectedSpawnEpoch: binding.spawnEpoch, + expectedRuntimeToken: binding.runtime?.runtimeToken ?? null, + state: "exited", + }); + + const closingParent = manager.close(caller.sessionId); + spawned[0]!.emitExit(0); + await closingParent; + const manual = await manager.create({ + cwd: "/tmp/manual-dormant-session", + harness: "claude-code", + }); + spawned[2]!.emitExit(0); + await manager.flush(); + const nextParent = await manager.create( + { cwd: manager.get(caller.sessionId)!.cwd, harness: "claude-code" }, + { + agentMapIdentity: (sessionId) => ({ + projectId, + userId: caller.userId, + sessionId, + }), + }, + ); + manager.setReady(nextParent.id, manager.getRuntimeEpoch(nextParent.id)!); + const nextCaller = nextParent.agentMapIdentity!; + await expect( + coordinator.execute(nextCaller, { + ...request, + requestKey: "blocked-before-dormant-release", + operation: { + ...request.operation, + delegations: [{ + delegationKey: "writer", + outcome: "Write evidence", + }], + }, + }), + ).rejects.toMatchObject({ + detail: { code: "capacity_exceeded", retryable: false }, + }); + + const released = await coordinator.execute( + nextCaller, + dormantReleaseRequest, + ); + const replay = await coordinator.execute(nextCaller, dormantReleaseRequest); + expect(released.results).toEqual([ + expect.objectContaining({ + delegationKey: "research", + sessionId: childId, + outcome: "released", + }), + ]); + expect(replay).toMatchObject({ + replayed: true, + results: [{ sessionId: childId, outcome: "released" }], + }); + expect(manager.get(childId)).toMatchObject({ status: "exited" }); + expect(manager.getSubsessionBinding(childId)).toBeNull(); + expect(manager.get(manual.id)).toMatchObject({ status: "exited" }); + expect(manager.getSubsessionBinding(manual.id)).toBeNull(); + await expect( + coordinator.execute( + { ...nextCaller, projectId: "project_foreign" }, + { ...dormantReleaseRequest, requestKey: "foreign-sweep" }, + ), + ).rejects.toMatchObject({ + detail: { code: "capability_scope_mismatch" }, + }); + + const next = await coordinator.execute(nextCaller, { + ...request, + requestKey: "after-dormant-release", + operation: { + ...request.operation, + delegations: [{ + delegationKey: "writer", + outcome: "Write evidence", + }], + }, + }); + unsubscribe(); + + expect(next.results[0]).toMatchObject({ + delegationKey: "writer", + outcome: "created", + }); + expect((await store.read(projectId)).bindings).toHaveLength(1); + }); + it.each(["exited", "failed"] as const)( "releases an already-%s child without spawning or resuming it", async (terminalState) => { @@ -588,6 +697,49 @@ describe("SubsessionCoordinator", () => { expect(spawnPty).toHaveBeenCalledTimes(2); }); + it("atomically renews an expired self-owned spawn claim across coordinators", async () => { + const { + newCoordinator, + caller, + manager, + store, + spawnPty, + unsubscribe, + } = await fixture(false, undefined, {}, { claimTtlMs: 500 }); + const parent = manager.get(caller.sessionId)!; + const binding = ( + await store.reserveDelegations(caller, request, { + harness: parent.harness, + projectRoot: parent.cwd, + ownerId: "coordinator-self", + }) + ).bindings[0]!; + const original = await store.claimSpawn(caller, binding.bindingId, { + ownerId: "coordinator-self", + expectedLifecycleEpoch: binding.lifecycleEpoch, + expectedSpawnEpoch: binding.spawnEpoch, + }); + if (!original.claimed) throw new Error("spawn claim was not acquired"); + await new Promise((resolve) => setTimeout(resolve, 550)); + + const self = newCoordinator("coordinator-self"); + const other = newCoordinator("coordinator-other"); + const [first, second] = await Promise.all([ + self.execute(caller, request), + other.execute(caller, request), + ]); + const aggregate = await store.read(projectId); + unsubscribe(); + + expect(first.results[0]!.sessionId).toBe(second.results[0]!.sessionId); + expect(spawnPty).toHaveBeenCalledTimes(2); + expect(aggregate.bindings[0]).toMatchObject({ + bindingId: binding.bindingId, + spawnEpoch: 2, + sessionState: "ready", + }); + }); + it("acknowledges only the exact persisted kickoff marker", async () => { const { coordinator, caller, manager, store, spawned, unsubscribe } = await fixture(); diff --git a/packages/harness/src/core/subsession-coordinator.ts b/packages/harness/src/core/subsession-coordinator.ts index 923368a1..6fda3299 100644 --- a/packages/harness/src/core/subsession-coordinator.ts +++ b/packages/harness/src/core/subsession-coordinator.ts @@ -43,6 +43,7 @@ import type { import { SubsessionCoordinatorStore, SubsessionCoordinatorStoreError, + type ReservedReleases, } from "./subsession-coordinator-store.js"; const DEFAULT_READINESS_TIMEOUT_MS = 30_000; @@ -111,6 +112,13 @@ type ReleaseRequest = Omit & { kind: "release" } >; }>; +type DormantReleaseRequest = Omit & + Readonly<{ + operation: Extract< + ProjectSubsessionRequest["operation"], + { kind: "release-dormant" } + >; + }>; const error = ( code: DelegationError["code"], @@ -205,6 +213,8 @@ export class SubsessionCoordinator { return this.refresh(identity, request as RefreshRequest); if (request.operation.kind === "release") return this.release(identity, request as ReleaseRequest); + if (request.operation.kind === "release-dormant") + return this.releaseDormant(identity, request as DormantReleaseRequest); return this.delegate(identity, request as DelegateRequest); } @@ -287,6 +297,46 @@ export class SubsessionCoordinator { } catch (cause) { throw this.wholeCallError(cause); } + return this.releaseReserved(identity, request.requestKey, reserved); + } + + private async releaseDormant( + identity: ProjectAgentSession, + request: DormantReleaseRequest, + ): Promise { + let reserved; + try { + const aggregate = await this.options.store.read(identity.projectId); + const candidateBindingIds = aggregate.bindings + .filter(({ sessionState }) => + ["exited", "failed"].includes(sessionState), + ) + .filter(({ parentSessionId }) => { + const parent = this.options.sessionManager.get(parentSessionId); + return !parent || parent.status === "exited"; + }) + .sort((left, right) => + left.updatedAt.localeCompare(right.updatedAt) || + left.bindingId.localeCompare(right.bindingId), + ) + .slice(0, request.operation.limit) + .map(({ bindingId }) => bindingId); + reserved = await this.options.store.reserveDormantReleases( + identity, + request, + candidateBindingIds, + ); + } catch (cause) { + throw this.wholeCallError(cause); + } + return this.releaseReserved(identity, request.requestKey, reserved); + } + + private async releaseReserved( + identity: ProjectAgentSession, + requestKey: string, + reserved: ReservedReleases, + ): Promise { const results: DelegationItemResult[] = []; for (const target of reserved.bindings) { if (target.state === "absent") { @@ -373,7 +423,12 @@ export class SubsessionCoordinator { "inspect_session", ); } - const closed = await this.options.store.closeBinding( + await this.options.store.closeBinding( + scopedIdentity, + binding.bindingId, + binding.sessionId, + ); + const closed = await this.options.store.finalizeReleasedBinding( scopedIdentity, binding.bindingId, binding.sessionId, @@ -400,7 +455,7 @@ export class SubsessionCoordinator { } return { schemaVersion: 1, - requestKey: request.requestKey, + requestKey, requestDigest: reserved.requestDigest, replayed: reserved.replayed, results: results.sort((left, right) => @@ -727,7 +782,8 @@ export class SubsessionCoordinator { let claim = binding.sessionState === "spawn-claimed" && - binding.spawnClaim?.ownerId === this.ownerId + binding.spawnClaim?.ownerId === this.ownerId && + binding.spawnClaim.expiresAt > new Date().toISOString() ? { claimed: true as const, binding } : await this.options.store.claimSpawn( identity, @@ -841,7 +897,10 @@ export class SubsessionCoordinator { } let claim; if (binding.sessionState === "spawn-claimed" && binding.spawnClaim) { - if (binding.spawnClaim.ownerId === this.ownerId) { + if ( + binding.spawnClaim.ownerId === this.ownerId && + binding.spawnClaim.expiresAt > new Date().toISOString() + ) { claim = { claimed: true as const, binding }; } else { if (binding.spawnClaim.expiresAt > new Date().toISOString()) return binding; diff --git a/packages/harness/src/profiles/project-agent.ts b/packages/harness/src/profiles/project-agent.ts index 5f69865c..a8d4cf42 100644 --- a/packages/harness/src/profiles/project-agent.ts +++ b/packages/harness/src/profiles/project-agent.ts @@ -12,7 +12,7 @@ Use agent_map_read when the current project architecture is relevant. When the w Keep internal implementation details local: library choices, ordinary implementation steps, incidental model or tool calls, and refactors that do not change a meaningful project boundary do not belong in the Agent Map. Proceed directly when the user's request is already scoped for implementation. -Focused assignments, map-node references, bootstrap context, and focused briefs are context only. They never grant or remove authority. Use project_subsession_delegate when decomposition improves delivery, and release your coordinator-owned child bindings when they are no longer needed. Never relabel, close, or otherwise reconcile unrelated user-created sessions. +Focused assignments, map-node references, bootstrap context, and focused briefs are context only. They never grant or remove authority. Use project_subsession_delegate when decomposition improves delivery, and release your coordinator-owned child bindings when they are no longer needed. If dormant bindings from dead or unreachable parents exhaust durable history, use its bounded release-dormant operation to reclaim only those coordinator-owned records. That operation is destructive and preserves ordinary session history, but ends automatic resume through the released binding. Never relabel, close, or otherwise reconcile unrelated user-created sessions. `; /** diff --git a/packages/harness/src/server/agent-map-mcp-tools.ts b/packages/harness/src/server/agent-map-mcp-tools.ts index 5e90035b..9a2a33e3 100644 --- a/packages/harness/src/server/agent-map-mcp-tools.ts +++ b/packages/harness/src/server/agent-map-mcp-tools.ts @@ -113,6 +113,10 @@ const projectSubsessionRequestSchema = z.object({ kind: z.literal("release"), delegationKeys: z.array(delegationKey).min(1).max(16), }).strict(), + z.object({ + kind: z.literal("release-dormant"), + limit: z.number().int().min(1).max(16), + }).strict(), ]), }).strict(); @@ -389,7 +393,7 @@ export function createAgentMapToolServer( server.registerTool( "project_subsession_delegate", { - description: "Create, reuse, or release a bounded batch of ordinary writable project subsessions, or refresh exact focused context, using caller-owned idempotency keys.", + description: "Create, reuse, or release a bounded batch of ordinary writable project subsessions, reclaim bounded dormant bindings whose parents are unreachable, or refresh exact focused context, using caller-owned idempotency keys.", inputSchema: projectSubsessionRequestSchema, annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: false }, }, diff --git a/packages/harness/src/server/agent-map-mcp.test.ts b/packages/harness/src/server/agent-map-mcp.test.ts index 8ac50126..c2f50302 100644 --- a/packages/harness/src/server/agent-map-mcp.test.ts +++ b/packages/harness/src/server/agent-map-mcp.test.ts @@ -151,6 +151,20 @@ describe("Agent Map Streamable HTTP MCP", () => { replayed: false, }, }); + await expect(client.callTool({ + name: "project_subsession_delegate", + arguments: { + schemaVersion: 1, + requestKey: "release-dormant-one", + operation: { kind: "release-dormant", limit: 1 }, + }, + })).resolves.toMatchObject({ + structuredContent: { + schemaVersion: 1, + requestKey: "release-dormant-one", + results: [], + }, + }); const validate = tools.tools.find( ({ name }) => name === "agent_map_validate", )!; diff --git a/packages/harness/src/shared/subsession-delegation-codec.test.ts b/packages/harness/src/shared/subsession-delegation-codec.test.ts index 0ff5c3e3..4952b10c 100644 --- a/packages/harness/src/shared/subsession-delegation-codec.test.ts +++ b/packages/harness/src/shared/subsession-delegation-codec.test.ts @@ -221,4 +221,41 @@ describe("subsession delegation codec", () => { issues: [{ code: "duplicate_delegation_key" }], }); }); + + it("accepts only a bounded server-selected dormant release", () => { + expect( + parseProjectSubsessionRequest( + { + schemaVersion: 1, + requestKey: "release-dormant-1", + operation: { kind: "release-dormant", limit: 16 }, + }, + projectId, + ).operation, + ).toEqual({ kind: "release-dormant", limit: 16 }); + expect(() => + parseProjectSubsessionRequest( + { + schemaVersion: 1, + requestKey: "release-dormant-2", + operation: { + kind: "release-dormant", + limit: 1, + sessionIds: ["manual-session"], + }, + }, + projectId, + ), + ).toThrowError(SubsessionDelegationValidationError); + expect(() => + parseProjectSubsessionRequest( + { + schemaVersion: 1, + requestKey: "release-dormant-3", + operation: { kind: "release-dormant", limit: 17 }, + }, + projectId, + ), + ).toThrowError(SubsessionDelegationValidationError); + }); }); diff --git a/packages/harness/src/shared/subsession-delegation-codec.ts b/packages/harness/src/shared/subsession-delegation-codec.ts index 013952e7..2c3e9aa9 100644 --- a/packages/harness/src/shared/subsession-delegation-codec.ts +++ b/packages/harness/src/shared/subsession-delegation-codec.ts @@ -302,6 +302,17 @@ export function parseProjectSubsessionRequest( if (new Set(delegationKeys).size !== delegationKeys.length) return invalid("operation.delegationKeys", "duplicate_delegation_key"); operation = { kind: "release", delegationKeys }; + } else if ( + value.operation.kind === "release-dormant" && + hasExactKeys(value.operation, ["kind", "limit"]) && + Number.isSafeInteger(value.operation.limit) && + (value.operation.limit as number) >= 1 && + (value.operation.limit as number) <= PROJECT_SUBSESSION_DELEGATION_LIMIT + ) { + operation = { + kind: "release-dormant", + limit: value.operation.limit as number, + }; } else { return invalid("operation", "invalid_operation"); } diff --git a/packages/harness/src/shared/subsession-delegation.ts b/packages/harness/src/shared/subsession-delegation.ts index 97e1cae5..cfcf8de3 100644 --- a/packages/harness/src/shared/subsession-delegation.ts +++ b/packages/harness/src/shared/subsession-delegation.ts @@ -76,6 +76,15 @@ export type ProjectSubsessionRequest = Readonly<{ | Readonly<{ kind: "release"; delegationKeys: readonly string[]; + }> + | Readonly<{ + /** + * Explicitly releases at most `limit` dormant coordinator bindings + * whose original parent session is no longer reachable. Selection is + * server-side and project-scoped; callers never provide session IDs. + */ + kind: "release-dormant"; + limit: number; }>; }>; From 78ea86d89627d69105b83520da773174b95685df Mon Sep 17 00:00:00 2001 From: Yash Date: Fri, 4 Sep 2026 21:57:05 +0000 Subject: [PATCH 17/19] fix(harness): make dormant recovery project-scoped Refs: SAP-3151 --- .changeset/writable-project-subsessions.md | 9 ++++--- packages/harness/docs/shared-build-plan.md | 20 ++++++++------ .../core/subsession-coordinator-store.test.ts | 27 +++++++++++++++++++ .../src/core/subsession-coordinator-store.ts | 7 ++--- .../src/core/subsession-coordinator.test.ts | 25 +++++++++++------ .../src/core/subsession-coordinator.ts | 13 ++++----- .../harness/src/profiles/project-agent.ts | 2 +- .../harness/src/server/agent-map-mcp-tools.ts | 2 +- .../src/shared/subsession-delegation.ts | 7 ++--- 9 files changed, 77 insertions(+), 35 deletions(-) diff --git a/.changeset/writable-project-subsessions.md b/.changeset/writable-project-subsessions.md index d1776da7..01cbef8c 100644 --- a/.changeset/writable-project-subsessions.md +++ b/.changeset/writable-project-subsessions.md @@ -30,7 +30,10 @@ retention so long-lived projects do not dead-end on routine delegation, release, or context refreshes. Exited and failed bindings remain durable for resume or recovery without holding an active slot until re-referenced, or until explicitly released. Any current project agent may explicitly reclaim up to sixteen dormant -coordinator-owned bindings whose original parent is absent or exited, without -supplying raw session IDs. This destructive recovery compacts coordinator and +coordinator-owned bindings in its project without supplying raw session IDs. +Each child is atomically rechecked as exited or failed; parent liveness is +intentionally irrelevant. This destructive recovery compacts coordinator and private ownership state while retaining the ordinary Harness session history; -the released binding is no longer automatically resumable. +the released binding is no longer automatically resumable. Durable-history +capacity failures identify `release_dormant` in their recovery field, while an +all-active live cap does not suggest dormant cleanup. diff --git a/packages/harness/docs/shared-build-plan.md b/packages/harness/docs/shared-build-plan.md index b83cf837..a583f0a0 100644 --- a/packages/harness/docs/shared-build-plan.md +++ b/packages/harness/docs/shared-build-plan.md @@ -129,14 +129,18 @@ project. Proven acknowledged or unsent delivery epochs are likewise pruned when a newer focused-context delivery replaces them; ambiguous delivery evidence is retained. -If exited or failed bindings outlive a dead or unreachable original parent and -fill durable binding history, any current project session may explicitly invoke -the bounded `release-dormant` operation. The coordinator selects at most sixteen -eligible records inside the capability-derived project; the request accepts no -session IDs and never selects active bindings or manual sessions. This operation -is destructive: it retains the ordinary Harness conversation/session history, -but compacts the coordinator binding and ends automatic resume through that -binding. Request receipts make the sweep idempotent and restart-safe. +If exited or failed bindings fill durable binding history, any current project +session may explicitly invoke the bounded `release-dormant` operation. The +coordinator selects at most sixteen eligible records inside the +capability-derived project; the request accepts no session IDs and never selects +active bindings or manual sessions. Parent liveness is intentionally irrelevant: +this explicit project-wide destructive operation relinquishes dormant delegation +resume identity even when the original parent is active. It retains the ordinary +Harness conversation/session history, but compacts the coordinator binding and +ends automatic resume through that binding. Request receipts make the sweep +idempotent and restart-safe. Durable-history capacity errors expose the explicit +`release_dormant` recovery code; an all-active live cap continues to require +session inspection instead of suggesting an inapplicable dormant cleanup. The coordinator waits for canonical adapter readiness and exact transcript identity, then uses fenced spawn and delivery epochs to submit one kickoff. diff --git a/packages/harness/src/core/subsession-coordinator-store.test.ts b/packages/harness/src/core/subsession-coordinator-store.test.ts index 9c4df54c..0ce7e37d 100644 --- a/packages/harness/src/core/subsession-coordinator-store.test.ts +++ b/packages/harness/src/core/subsession-coordinator-store.test.ts @@ -1089,6 +1089,33 @@ describe("SubsessionCoordinatorStore", () => { ); }); + it("atomically excludes an active child from project-wide dormant release", async () => { + const root = await fixture(); + const store = new SubsessionCoordinatorStore(root); + const binding = ( + await store.reserveDelegations( + identity, + delegate("parent-race", [ + { delegationKey: "research", outcome: "Collect evidence" }, + ]), + target, + ) + ).bindings[0]!; + const reserved = await store.reserveDormantReleases( + { ...identity, sessionId: "new-parent" }, + releaseDormant("active-child", 1), + [binding.bindingId], + ); + + expect(reserved.bindings).toEqual([]); + expect((await store.read(projectId)).bindings).toContainEqual( + expect.objectContaining({ + bindingId: binding.bindingId, + sessionState: "reserved", + }), + ); + }); + it("reclaims released capacity so a sixty-fifth delegation can be reserved", async () => { const root = await fixture(); const store = new SubsessionCoordinatorStore(root, { diff --git a/packages/harness/src/core/subsession-coordinator-store.ts b/packages/harness/src/core/subsession-coordinator-store.ts index 4e7c1e90..6abdc250 100644 --- a/packages/harness/src/core/subsession-coordinator-store.ts +++ b/packages/harness/src/core/subsession-coordinator-store.ts @@ -1195,9 +1195,10 @@ export class SubsessionCoordinatorStore { /** * Reserves an explicit project-scoped cleanup of dormant coordinator-owned - * bindings. Candidate IDs are selected by the trusted coordinator after it - * proves each original parent is absent or exited; they are never accepted - * from the public request. + * bindings. Candidate IDs are selected by the trusted coordinator and never + * accepted from the public request. The transaction rechecks the child state; + * parent liveness is intentionally irrelevant to this explicit project-wide + * destructive operation. */ reserveDormantReleases( identity: ProjectAgentSession, diff --git a/packages/harness/src/core/subsession-coordinator.test.ts b/packages/harness/src/core/subsession-coordinator.test.ts index 9095f14c..2c146e7e 100644 --- a/packages/harness/src/core/subsession-coordinator.test.ts +++ b/packages/harness/src/core/subsession-coordinator.test.ts @@ -504,7 +504,7 @@ describe("SubsessionCoordinator", () => { }); }); - it("lets a new project agent reclaim a dead parent's dormant child at history capacity", async () => { + it("lets a new project agent reclaim a dormant child of an active parent at history capacity", async () => { const { coordinator, caller, manager, store, spawned, unsubscribe } = await fixture(false, undefined, {}, { bindingLimit: 1 }); const created = await coordinator.execute(caller, request); @@ -519,9 +519,6 @@ describe("SubsessionCoordinator", () => { state: "exited", }); - const closingParent = manager.close(caller.sessionId); - spawned[0]!.emitExit(0); - await closingParent; const manual = await manager.create({ cwd: "/tmp/manual-dormant-session", harness: "claude-code", @@ -553,7 +550,11 @@ describe("SubsessionCoordinator", () => { }, }), ).rejects.toMatchObject({ - detail: { code: "capacity_exceeded", retryable: false }, + detail: { + code: "capacity_exceeded", + retryable: false, + recovery: "release_dormant", + }, }); const released = await coordinator.execute( @@ -574,6 +575,7 @@ describe("SubsessionCoordinator", () => { }); expect(manager.get(childId)).toMatchObject({ status: "exited" }); expect(manager.getSubsessionBinding(childId)).toBeNull(); + expect(manager.get(caller.sessionId)?.status).not.toBe("exited"); expect(manager.get(manual.id)).toMatchObject({ status: "exited" }); expect(manager.getSubsessionBinding(manual.id)).toBeNull(); await expect( @@ -596,12 +598,19 @@ describe("SubsessionCoordinator", () => { }], }, }); + const activeSweep = await coordinator.execute(nextCaller, { + ...dormantReleaseRequest, + requestKey: "active-and-manual-exclusion", + }); unsubscribe(); expect(next.results[0]).toMatchObject({ delegationKey: "writer", outcome: "created", }); + expect(activeSweep.results).toEqual([]); + expect(manager.get(next.results[0]!.sessionId!)?.status).not.toBe("exited"); + expect(manager.get(manual.id)).toMatchObject({ status: "exited" }); expect((await store.read(projectId)).bindings).toHaveLength(1); }); @@ -874,7 +883,7 @@ describe("SubsessionCoordinator", () => { unsubscribe(); }); - it("does not advise retry for unreclaimable active history exhaustion", async () => { + it("directs dormant history exhaustion to bounded dormant release", async () => { const { coordinator, caller, store, unsubscribe } = await fixture(); vi.spyOn(store, "reserveDelegations").mockRejectedValueOnce( new SubsessionCoordinatorStoreError("history_quota_exceeded"), @@ -883,13 +892,13 @@ describe("SubsessionCoordinator", () => { detail: { code: "capacity_exceeded", retryable: false, - recovery: "none", + recovery: "release_dormant", }, }); unsubscribe(); }); - it("directs resumable-session capacity exhaustion to session inspection", async () => { + it("directs genuinely live-session capacity exhaustion to session inspection", async () => { const { coordinator, caller, store, unsubscribe } = await fixture(); vi.spyOn(store, "reserveDelegations").mockRejectedValueOnce( new SubsessionCoordinatorStoreError("live_session_limit_reached"), diff --git a/packages/harness/src/core/subsession-coordinator.ts b/packages/harness/src/core/subsession-coordinator.ts index 6fda3299..fac59d07 100644 --- a/packages/harness/src/core/subsession-coordinator.ts +++ b/packages/harness/src/core/subsession-coordinator.ts @@ -311,10 +311,6 @@ export class SubsessionCoordinator { .filter(({ sessionState }) => ["exited", "failed"].includes(sessionState), ) - .filter(({ parentSessionId }) => { - const parent = this.options.sessionManager.get(parentSessionId); - return !parent || parent.status === "exited"; - }) .sort((left, right) => left.updatedAt.localeCompare(right.updatedAt) || left.bindingId.localeCompare(right.bindingId), @@ -1391,10 +1387,11 @@ export class SubsessionCoordinator { return new SubsessionCoordinatorError( error("capacity_exceeded", false, "inspect_session"), ); - if ( - cause.code === "delegation_depth_exceeded" || - cause.code === "history_quota_exceeded" - ) { + if (cause.code === "history_quota_exceeded") + return new SubsessionCoordinatorError( + error("capacity_exceeded", false, "release_dormant"), + ); + if (cause.code === "delegation_depth_exceeded") { return new SubsessionCoordinatorError( error("capacity_exceeded", false, "none"), ); diff --git a/packages/harness/src/profiles/project-agent.ts b/packages/harness/src/profiles/project-agent.ts index a8d4cf42..c36ffc0b 100644 --- a/packages/harness/src/profiles/project-agent.ts +++ b/packages/harness/src/profiles/project-agent.ts @@ -12,7 +12,7 @@ Use agent_map_read when the current project architecture is relevant. When the w Keep internal implementation details local: library choices, ordinary implementation steps, incidental model or tool calls, and refactors that do not change a meaningful project boundary do not belong in the Agent Map. Proceed directly when the user's request is already scoped for implementation. -Focused assignments, map-node references, bootstrap context, and focused briefs are context only. They never grant or remove authority. Use project_subsession_delegate when decomposition improves delivery, and release your coordinator-owned child bindings when they are no longer needed. If dormant bindings from dead or unreachable parents exhaust durable history, use its bounded release-dormant operation to reclaim only those coordinator-owned records. That operation is destructive and preserves ordinary session history, but ends automatic resume through the released binding. Never relabel, close, or otherwise reconcile unrelated user-created sessions. +Focused assignments, map-node references, bootstrap context, and focused briefs are context only. They never grant or remove authority. Use project_subsession_delegate when decomposition improves delivery, and release your coordinator-owned child bindings when they are no longer needed. If dormant coordinator bindings exhaust durable history, use its bounded project-wide release-dormant operation. It releases only bindings atomically rechecked as exited or failed, regardless of parent liveness. The operation is destructive and preserves ordinary session history, but ends automatic resume through each released binding. Never relabel, close, or otherwise reconcile unrelated user-created sessions. `; /** diff --git a/packages/harness/src/server/agent-map-mcp-tools.ts b/packages/harness/src/server/agent-map-mcp-tools.ts index 9a2a33e3..031a335f 100644 --- a/packages/harness/src/server/agent-map-mcp-tools.ts +++ b/packages/harness/src/server/agent-map-mcp-tools.ts @@ -393,7 +393,7 @@ export function createAgentMapToolServer( server.registerTool( "project_subsession_delegate", { - description: "Create, reuse, or release a bounded batch of ordinary writable project subsessions, reclaim bounded dormant bindings whose parents are unreachable, or refresh exact focused context, using caller-owned idempotency keys.", + description: "Create, reuse, or release a bounded batch of ordinary writable project subsessions, reclaim a bounded project-wide set of coordinator-owned dormant bindings, or refresh exact focused context, using caller-owned idempotency keys.", inputSchema: projectSubsessionRequestSchema, annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: false }, }, diff --git a/packages/harness/src/shared/subsession-delegation.ts b/packages/harness/src/shared/subsession-delegation.ts index cfcf8de3..ae281cb4 100644 --- a/packages/harness/src/shared/subsession-delegation.ts +++ b/packages/harness/src/shared/subsession-delegation.ts @@ -79,9 +79,9 @@ export type ProjectSubsessionRequest = Readonly<{ }> | Readonly<{ /** - * Explicitly releases at most `limit` dormant coordinator bindings - * whose original parent session is no longer reachable. Selection is - * server-side and project-scoped; callers never provide session IDs. + * Explicitly releases at most `limit` dormant coordinator bindings in + * the current project. Selection is server-side; callers never provide + * session IDs. */ kind: "release-dormant"; limit: number; @@ -124,6 +124,7 @@ export type DelegationRecovery = | "inspect_session" | "new_request_key" | "new_delegation_key" + | "release_dormant" | "reduce_request"; export type DelegationError = Readonly<{ From a66e4d8602af1efd20f2d9c814d2dee2a5f2d050 Mon Sep 17 00:00:00 2001 From: Yash Date: Fri, 4 Sep 2026 22:07:20 +0000 Subject: [PATCH 18/19] fix(harness): expire dormant delegation identities Refs: SAP-3151 --- .changeset/writable-project-subsessions.md | 6 +- packages/harness/docs/shared-build-plan.md | 19 +++--- .../core/subsession-coordinator-store.test.ts | 24 ++++---- .../src/core/subsession-coordinator-store.ts | 60 +++++++++++++++---- .../src/core/subsession-coordinator.test.ts | 45 ++++++++++---- .../harness/src/profiles/project-agent.ts | 2 +- 6 files changed, 111 insertions(+), 45 deletions(-) diff --git a/.changeset/writable-project-subsessions.md b/.changeset/writable-project-subsessions.md index 01cbef8c..15ddcf4e 100644 --- a/.changeset/writable-project-subsessions.md +++ b/.changeset/writable-project-subsessions.md @@ -36,4 +36,8 @@ intentionally irrelevant. This destructive recovery compacts coordinator and private ownership state while retaining the ordinary Harness session history; the released binding is no longer automatically resumable. Durable-history capacity failures identify `release_dormant` in their recovery field, while an -all-active live cap does not suggest dormant cleanup. +all-active live cap does not suggest dormant cleanup. Delegation retries converge +until this explicit project-wide eviction boundary. Eviction expires request +receipts that referenced the released binding, so an old retry returns bounded +`request_key_expired` / `new_request_key` recovery; a fresh request key may +atomically create one new binding and real session for the same delegation key. diff --git a/packages/harness/docs/shared-build-plan.md b/packages/harness/docs/shared-build-plan.md index a583f0a0..6f0c322f 100644 --- a/packages/harness/docs/shared-build-plan.md +++ b/packages/harness/docs/shared-build-plan.md @@ -112,9 +112,10 @@ a session identity. Callers provide both a request key and a delegation key. Identity is scoped by the private session capability to the trusted project and parent session. Identical retries converge on the same durable binding and real Harness session -ID; changing canonical request or binding content under an existing key fails -explicitly. All binding IDs and session IDs for a bounded batch are reserved in -one durable transaction before the first process is spawned. +ID until an explicit project-wide dormant eviction; changing canonical request +or binding content under an existing key otherwise fails explicitly. All binding +IDs and session IDs for a bounded batch are reserved in one durable transaction +before the first process is spawned. Older request receipts compact into bounded key tombstones. User-closed bindings compact into bounded ownership tombstones once no retained receipt references them; an explicit release finalizes immediately to the same @@ -137,10 +138,14 @@ active bindings or manual sessions. Parent liveness is intentionally irrelevant: this explicit project-wide destructive operation relinquishes dormant delegation resume identity even when the original parent is active. It retains the ordinary Harness conversation/session history, but compacts the coordinator binding and -ends automatic resume through that binding. Request receipts make the sweep -idempotent and restart-safe. Durable-history capacity errors expose the explicit -`release_dormant` recovery code; an all-active live cap continues to require -session inspection instead of suggesting an inapplicable dormant cleanup. +ends automatic resume through that binding. The sweep remains idempotent and +restart-safe, while prior request receipts referencing an evicted binding become +bounded expiry tombstones. Retrying one of those keys returns +`request_key_expired` with `new_request_key`; a fresh request key may atomically +create one new binding/session for the same delegation key. Durable-history +capacity errors expose the explicit `release_dormant` recovery code; an +all-active live cap continues to require session inspection instead of suggesting +an inapplicable dormant cleanup. The coordinator waits for canonical adapter readiness and exact transcript identity, then uses fenced spawn and delivery epochs to submit one kickoff. diff --git a/packages/harness/src/core/subsession-coordinator-store.test.ts b/packages/harness/src/core/subsession-coordinator-store.test.ts index 0ce7e37d..9c4492f2 100644 --- a/packages/harness/src/core/subsession-coordinator-store.test.ts +++ b/packages/harness/src/core/subsession-coordinator-store.test.ts @@ -1041,23 +1041,23 @@ describe("SubsessionCoordinatorStore", () => { ); expect(reserved.bindings).toEqual([ { - state: "bound", + state: "released", binding: expect.objectContaining({ bindingId: dormant[0]!.bindingId, - sessionState: "closed", + disposition: "dormant-evicted", }), }, ]); - await store.closeBinding( - identity, - dormant[0]!.bindingId, - dormant[0]!.sessionId, - ); - await store.finalizeReleasedBinding( - identity, - dormant[0]!.bindingId, - dormant[0]!.sessionId, - ); + await expect( + store.reserveDelegations( + identity, + delegate("old-request", [ + { delegationKey: "research", outcome: "Collect evidence" }, + { delegationKey: "publisher", outcome: "Publish evidence" }, + ]), + target, + ), + ).rejects.toMatchObject({ code: "request_key_expired" }); const replay = await store.reserveDormantReleases( newParent, request, diff --git a/packages/harness/src/core/subsession-coordinator-store.ts b/packages/harness/src/core/subsession-coordinator-store.ts index 6abdc250..352eec44 100644 --- a/packages/harness/src/core/subsession-coordinator-store.ts +++ b/packages/harness/src/core/subsession-coordinator-store.ts @@ -92,6 +92,7 @@ export type SubsessionCoordinatorBindingTombstone = Readonly<{ delegationKey: string; bindingDigest: string; sessionId: string; + disposition: "terminal" | "dormant-evicted"; closedAt: string; }>; @@ -552,6 +553,7 @@ function parseBindingTombstone( "delegationKey", "bindingDigest", "sessionId", + "disposition", "closedAt", ]) || !identifier(value.bindingId, "binding") || @@ -564,6 +566,7 @@ function parseBindingTombstone( !identifier(value.delegationKey) || !digest(value.bindingDigest) || !identifier(value.sessionId) || + !["terminal", "dormant-evicted"].includes(String(value.disposition)) || !timestamp(value.closedAt) ) { throw new SubsessionCoordinatorStoreError("malformed_state"); @@ -631,16 +634,18 @@ export function parseSubsessionCoordinatorAggregate( `${parentSessionId}\0${delegationKey}`, ); const allBindings = [...bindings, ...bindingTombstones]; - const allBindingKeys = [ + const terminalBindingKeys = [ ...bindingKeys, ...bindingTombstones.map( - ({ parentSessionId, delegationKey }) => - `${parentSessionId}\0${delegationKey}`, + ({ disposition, parentSessionId, delegationKey }) => + disposition === "terminal" + ? `${parentSessionId}\0${delegationKey}` + : null, ), - ]; + ].filter((key): key is string => key !== null); if ( new Set(requestKeys).size !== requestKeys.length || - new Set(allBindingKeys).size !== allBindings.length || + new Set(terminalBindingKeys).size !== terminalBindingKeys.length || new Set(allBindings.map(({ bindingId }) => bindingId)).size !== allBindings.length || new Set(allBindings.map(({ sessionId }) => sessionId)).size !== @@ -797,6 +802,7 @@ export class SubsessionCoordinatorStore { delegationKey: binding.delegationKey, bindingDigest: binding.bindingDigest, sessionId: binding.sessionId, + disposition: "terminal", closedAt: binding.updatedAt, }); } @@ -1085,6 +1091,7 @@ export class SubsessionCoordinatorStore { delegationKey: binding.delegationKey, bindingDigest: binding.bindingDigest, sessionId: binding.sessionId, + disposition: "terminal", closedAt: binding.updatedAt, }; aggregate.bindings = aggregate.bindings.filter( @@ -1258,6 +1265,7 @@ export class SubsessionCoordinatorStore { this.compactTerminalHistory(aggregate); const now = this.now(); const bindings: ReleasableSubsessionBinding[] = []; + const evictedBindingIds = new Set(); for (const bindingId of candidateBindingIds) { const binding = aggregate.bindings.find( (entry) => entry.bindingId === bindingId, @@ -1267,12 +1275,20 @@ export class SubsessionCoordinatorStore { // The explicit destructive boundary and request receipt commit in // the same transaction. A concurrent resume must lose this fence // before any exact private ownership marker is removed. - binding.sessionState = "closed"; - binding.lifecycleEpoch += 1; - binding.spawnClaim = null; - binding.runtime = null; - binding.updatedAt = now; - bindings.push({ state: "bound", binding }); + const tombstone: SubsessionCoordinatorBindingTombstone = { + bindingId: binding.bindingId, + parentSessionId: binding.parentSessionId, + parentBindingId: binding.parentBindingId, + delegationDepth: binding.delegationDepth, + delegationKey: binding.delegationKey, + bindingDigest: binding.bindingDigest, + sessionId: binding.sessionId, + disposition: "dormant-evicted", + closedAt: now, + }; + aggregate.bindingTombstones.push(tombstone); + evictedBindingIds.add(binding.bindingId); + bindings.push({ state: "released", binding: tombstone }); } continue; } @@ -1281,6 +1297,25 @@ export class SubsessionCoordinatorStore { ); if (released) bindings.push({ state: "released", binding: released }); } + if (evictedBindingIds.size > 0) { + const retainedReceipts: SubsessionCoordinatorRequestReceipt[] = []; + for (const receipt of aggregate.requestReceipts) { + if ( + receipt.operation !== "release-dormant" && + receipt.bindingIds.some((bindingId) => + evictedBindingIds.has(bindingId), + ) + ) { + aggregate.requestTombstones.push(receipt); + } else { + retainedReceipts.push(receipt); + } + } + aggregate.requestReceipts = retainedReceipts; + aggregate.bindings = aggregate.bindings.filter( + ({ bindingId }) => !evictedBindingIds.has(bindingId), + ); + } aggregate.requestReceipts.push({ parentSessionId: identity.sessionId, requestKey: request.requestKey, @@ -1594,7 +1629,8 @@ export class SubsessionCoordinatorStore { const terminal = aggregate.bindingTombstones.find( (entry) => entry.parentSessionId === identity.sessionId && - entry.delegationKey === delegation.delegationKey, + entry.delegationKey === delegation.delegationKey && + entry.disposition === "terminal", ); if (terminal) { if (terminal.bindingDigest !== bindingDigest) diff --git a/packages/harness/src/core/subsession-coordinator.test.ts b/packages/harness/src/core/subsession-coordinator.test.ts index 2c146e7e..4a414203 100644 --- a/packages/harness/src/core/subsession-coordinator.test.ts +++ b/packages/harness/src/core/subsession-coordinator.test.ts @@ -504,8 +504,16 @@ describe("SubsessionCoordinator", () => { }); }); - it("lets a new project agent reclaim a dormant child of an active parent at history capacity", async () => { - const { coordinator, caller, manager, store, spawned, unsubscribe } = + it("expires the old request and lets an active parent recreate a sibling-evicted dormant child", async () => { + const { + coordinator, + caller, + manager, + store, + spawned, + spawnPty, + unsubscribe, + } = await fixture(false, undefined, {}, { bindingLimit: 1 }); const created = await coordinator.execute(caller, request); const childId = created.results[0]!.sessionId!; @@ -587,17 +595,19 @@ describe("SubsessionCoordinator", () => { detail: { code: "capability_scope_mismatch" }, }); - const next = await coordinator.execute(nextCaller, { - ...request, - requestKey: "after-dormant-release", - operation: { - ...request.operation, - delegations: [{ - delegationKey: "writer", - outcome: "Write evidence", - }], + await expect(coordinator.execute(caller, request)).rejects.toMatchObject({ + detail: { + code: "request_key_expired", + retryable: false, + recovery: "new_request_key", }, }); + const recreatedRequest = { + ...request, + requestKey: "after-dormant-release", + } as const; + const next = await coordinator.execute(caller, recreatedRequest); + const nextReplay = await coordinator.execute(caller, recreatedRequest); const activeSweep = await coordinator.execute(nextCaller, { ...dormantReleaseRequest, requestKey: "active-and-manual-exclusion", @@ -605,10 +615,21 @@ describe("SubsessionCoordinator", () => { unsubscribe(); expect(next.results[0]).toMatchObject({ - delegationKey: "writer", + delegationKey: "research", outcome: "created", }); + expect(next.results[0]!.sessionId).not.toBe(childId); + expect(nextReplay).toMatchObject({ + replayed: true, + results: [{ + delegationKey: "research", + sessionId: next.results[0]!.sessionId, + outcome: "reused", + }], + }); expect(activeSweep.results).toEqual([]); + expect(spawnPty).toHaveBeenCalledTimes(5); + expect(manager.get(caller.sessionId)?.status).not.toBe("exited"); expect(manager.get(next.results[0]!.sessionId!)?.status).not.toBe("exited"); expect(manager.get(manual.id)).toMatchObject({ status: "exited" }); expect((await store.read(projectId)).bindings).toHaveLength(1); diff --git a/packages/harness/src/profiles/project-agent.ts b/packages/harness/src/profiles/project-agent.ts index c36ffc0b..a03bbe0e 100644 --- a/packages/harness/src/profiles/project-agent.ts +++ b/packages/harness/src/profiles/project-agent.ts @@ -12,7 +12,7 @@ Use agent_map_read when the current project architecture is relevant. When the w Keep internal implementation details local: library choices, ordinary implementation steps, incidental model or tool calls, and refactors that do not change a meaningful project boundary do not belong in the Agent Map. Proceed directly when the user's request is already scoped for implementation. -Focused assignments, map-node references, bootstrap context, and focused briefs are context only. They never grant or remove authority. Use project_subsession_delegate when decomposition improves delivery, and release your coordinator-owned child bindings when they are no longer needed. If dormant coordinator bindings exhaust durable history, use its bounded project-wide release-dormant operation. It releases only bindings atomically rechecked as exited or failed, regardless of parent liveness. The operation is destructive and preserves ordinary session history, but ends automatic resume through each released binding. Never relabel, close, or otherwise reconcile unrelated user-created sessions. +Focused assignments, map-node references, bootstrap context, and focused briefs are context only. They never grant or remove authority. Use project_subsession_delegate when decomposition improves delivery, and release your coordinator-owned child bindings when they are no longer needed. If dormant coordinator bindings exhaust durable history, use its bounded project-wide release-dormant operation. It releases only bindings atomically rechecked as exited or failed, regardless of parent liveness. The operation is destructive and preserves ordinary session history, but ends automatic resume through each released binding and expires its prior request keys. A later delegation of the same key requires a fresh request key and creates a fresh binding/session. Never relabel, close, or otherwise reconcile unrelated user-created sessions. `; /** From ca747232e0b775b5be5c69c5427c778b3774dd4f Mon Sep 17 00:00:00 2001 From: Yash Date: Fri, 4 Sep 2026 22:20:27 +0000 Subject: [PATCH 19/19] fix(harness): report dormant eviction outcomes Refs: SAP-3151 --- .changeset/writable-project-subsessions.md | 4 ++ packages/harness/docs/shared-build-plan.md | 5 +- .../core/subsession-coordinator-store.test.ts | 2 +- .../src/core/subsession-coordinator-store.ts | 7 +- .../src/core/subsession-coordinator.test.ts | 65 ++++++++++++++++++- .../src/core/subsession-coordinator.ts | 11 +++- 6 files changed, 88 insertions(+), 6 deletions(-) diff --git a/.changeset/writable-project-subsessions.md b/.changeset/writable-project-subsessions.md index 15ddcf4e..71e3b271 100644 --- a/.changeset/writable-project-subsessions.md +++ b/.changeset/writable-project-subsessions.md @@ -41,3 +41,7 @@ until this explicit project-wide eviction boundary. Eviction expires request receipts that referenced the released binding, so an old retry returns bounded `request_key_expired` / `new_request_key` recovery; a fresh request key may atomically create one new binding and real session for the same delegation key. +Dormant eviction emits content-free release telemetry when it commits. If later +private-marker cleanup fails, the result remains truthfully `released`, includes +the bounded cleanup error, and retains exact proof for idempotent cleanup after +the indicated recovery. diff --git a/packages/harness/docs/shared-build-plan.md b/packages/harness/docs/shared-build-plan.md index 6f0c322f..f989b1c7 100644 --- a/packages/harness/docs/shared-build-plan.md +++ b/packages/harness/docs/shared-build-plan.md @@ -145,7 +145,10 @@ bounded expiry tombstones. Retrying one of those keys returns create one new binding/session for the same delegation key. Durable-history capacity errors expose the explicit `release_dormant` recovery code; an all-active live cap continues to require session inspection instead of suggesting -an inapplicable dormant cleanup. +an inapplicable dormant cleanup. A bounded private-marker cleanup error may +accompany an already-`released` result because eviction is durable first. Exact +cleanup proof remains available so the same sweep can finish after the indicated +recovery or inspection without changing the release outcome. The coordinator waits for canonical adapter readiness and exact transcript identity, then uses fenced spawn and delivery epochs to submit one kickoff. diff --git a/packages/harness/src/core/subsession-coordinator-store.test.ts b/packages/harness/src/core/subsession-coordinator-store.test.ts index 9c4492f2..e4a6ccb3 100644 --- a/packages/harness/src/core/subsession-coordinator-store.test.ts +++ b/packages/harness/src/core/subsession-coordinator-store.test.ts @@ -1041,7 +1041,7 @@ describe("SubsessionCoordinatorStore", () => { ); expect(reserved.bindings).toEqual([ { - state: "released", + state: "evicted", binding: expect.objectContaining({ bindingId: dormant[0]!.bindingId, disposition: "dormant-evicted", diff --git a/packages/harness/src/core/subsession-coordinator-store.ts b/packages/harness/src/core/subsession-coordinator-store.ts index 352eec44..d739264e 100644 --- a/packages/harness/src/core/subsession-coordinator-store.ts +++ b/packages/harness/src/core/subsession-coordinator-store.ts @@ -132,6 +132,11 @@ export type ReleasableSubsessionBinding = state: "bound"; binding: SubsessionBindingRecord; }> + | Readonly<{ + /** This request atomically committed the dormant eviction. */ + state: "evicted"; + binding: SubsessionCoordinatorBindingTombstone; + }> | Readonly<{ state: "released"; binding: SubsessionCoordinatorBindingTombstone; @@ -1288,7 +1293,7 @@ export class SubsessionCoordinatorStore { }; aggregate.bindingTombstones.push(tombstone); evictedBindingIds.add(binding.bindingId); - bindings.push({ state: "released", binding: tombstone }); + bindings.push({ state: "evicted", binding: tombstone }); } continue; } diff --git a/packages/harness/src/core/subsession-coordinator.test.ts b/packages/harness/src/core/subsession-coordinator.test.ts index 4a414203..75c6560f 100644 --- a/packages/harness/src/core/subsession-coordinator.test.ts +++ b/packages/harness/src/core/subsession-coordinator.test.ts @@ -28,11 +28,13 @@ import { import { compileCanonicalWorkstreamBriefs } from "./agent-brief-compiler.js"; import { createEmptyProjectPlanningAggregate } from "./agent-map-aggregate-migration.js"; import type { EventReader } from "./collector/store.js"; +import { SubsessionBindingMismatchError } from "./errors.js"; import { IngestCredentialRegistry } from "./ingest-credentials.js"; import { SessionManager, type PtySpawnFn } from "./session-manager.js"; import { SubsessionCoordinator, SubsessionCoordinatorError, + type SubsessionCoordinatorEvent, } from "./subsession-coordinator.js"; import { SubsessionCoordinatorStore, @@ -284,7 +286,7 @@ describe("SubsessionCoordinator", () => { storeOptions, ); closeStore.current = store; - const telemetry: unknown[] = []; + const telemetry: SubsessionCoordinatorEvent[] = []; const planningStore = { read: vi.fn(async () => { throw new Error("no focused context expected"); @@ -512,6 +514,7 @@ describe("SubsessionCoordinator", () => { store, spawned, spawnPty, + telemetry, unsubscribe, } = await fixture(false, undefined, {}, { bindingLimit: 1 }); @@ -586,6 +589,13 @@ describe("SubsessionCoordinator", () => { expect(manager.get(caller.sessionId)?.status).not.toBe("exited"); expect(manager.get(manual.id)).toMatchObject({ status: "exited" }); expect(manager.getSubsessionBinding(manual.id)).toBeNull(); + expect(telemetry).toContainEqual( + expect.objectContaining({ + name: "subsession.released", + projectId, + sessionId: childId, + }), + ); await expect( coordinator.execute( { ...nextCaller, projectId: "project_foreign" }, @@ -635,6 +645,59 @@ describe("SubsessionCoordinator", () => { expect((await store.read(projectId)).bindings).toHaveLength(1); }); + it("reports a committed dormant eviction truthfully when private cleanup must retry", async () => { + const { coordinator, caller, manager, store, spawned, telemetry, unsubscribe } = + await fixture(); + const created = await coordinator.execute(caller, request); + const childId = created.results[0]!.sessionId!; + spawned[1]!.emitExit(0); + await manager.flush(); + const binding = (await store.read(projectId)).bindings[0]!; + await store.transitionSession(caller, binding.bindingId, { + expectedLifecycleEpoch: binding.lifecycleEpoch, + expectedSpawnEpoch: binding.spawnEpoch, + expectedRuntimeToken: binding.runtime?.runtimeToken ?? null, + state: "exited", + }); + const closeBound = vi + .spyOn(manager, "closeBound") + .mockRejectedValueOnce(new SubsessionBindingMismatchError()); + + const released = await coordinator.execute(caller, dormantReleaseRequest); + expect(released.results[0]).toMatchObject({ + delegationKey: "research", + sessionId: childId, + outcome: "released", + sessionState: "closed", + error: { + code: "binding_session_mismatch", + retryable: false, + recovery: "inspect_session", + }, + }); + expect((await store.read(projectId)).bindingTombstones[0]).toMatchObject({ + sessionId: childId, + disposition: "dormant-evicted", + }); + expect(manager.getSubsessionBinding(childId)).not.toBeNull(); + + closeBound.mockRestore(); + const replay = await coordinator.execute(caller, dormantReleaseRequest); + unsubscribe(); + + expect(replay).toMatchObject({ + replayed: true, + results: [{ sessionId: childId, outcome: "released" }], + }); + expect(manager.getSubsessionBinding(childId)).toBeNull(); + expect( + telemetry.filter( + (event) => + event.name === "subsession.released" && event.sessionId === childId, + ), + ).toHaveLength(1); + }); + it.each(["exited", "failed"] as const)( "releases an already-%s child without spawning or resuming it", async (terminalState) => { diff --git a/packages/harness/src/core/subsession-coordinator.ts b/packages/harness/src/core/subsession-coordinator.ts index fac59d07..255a906b 100644 --- a/packages/harness/src/core/subsession-coordinator.ts +++ b/packages/harness/src/core/subsession-coordinator.ts @@ -347,8 +347,16 @@ export class SubsessionCoordinator { }); continue; } - if (target.state === "released") { + if (target.state === "released" || target.state === "evicted") { const binding = target.binding; + const newlyEvicted = target.state === "evicted"; + if (newlyEvicted) { + this.emit({ + name: "subsession.released", + projectId: identity.projectId, + sessionId: binding.sessionId, + }); + } try { const privateMarker = this.options.sessionManager.getSubsessionBinding(binding.sessionId); @@ -381,7 +389,6 @@ export class SubsessionCoordinator { }); results.push({ ...this.releasedResult(binding), - outcome: "failed", error: detail, }); }