diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a52d96a0..9833483c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -662,6 +662,10 @@ jobs: with: bun-version: 1.4.0 + - uses: denoland/setup-deno@e95548e56dfa95d4e1a28d6f422fafe75c4c26fb # v2.0.3 + with: + deno-version: v2.9.5 + - run: bun install - name: Test diff --git a/packages/test-agent/README.md b/packages/test-agent/README.md index 9a3444521..1d7c227ec 100644 --- a/packages/test-agent/README.md +++ b/packages/test-agent/README.md @@ -97,6 +97,42 @@ the unnamed session, not every named session. Each test and each working directory receive independent scenario state, even when they use the same mapping. A test may finish with later scenario stages unused. +## Exercise model and effort selection + +Declare fictional choices on `TestAgent.Scenario` and keep the workflow's +ordinary Session configuration: + +```md + + + + + Review this change. + + + +``` + +In `agents/review.md`, observe what the prompt actually consumed: + +```md + +Reviewed with the requested configuration. +``` + +The IDs are exact strings supplied by the test. Each nonempty catalog has a +selected member and unique options. A model remembers its own effort selection +when the session switches away and back. Omitted or null catalogs represent +unavailable choices; ordinary unconfigured Sessions still run. A missing +WhenPrompt constraint skips that check, while null asserts unavailable choices. +A mismatch leaves the stage active and fails the test. The same catalog syntax +works in nested Execution declarations; separate Tests and children start from +their own defaults. No real provider or credentials are needed. + ## Use a failure to fix the workflow If the document sends this instead: diff --git a/packages/test-agent/mod.ts b/packages/test-agent/mod.ts index 139641adf..a5010424f 100644 --- a/packages/test-agent/mod.ts +++ b/packages/test-agent/mod.ts @@ -58,3 +58,5 @@ export { runTestAgentWorker } from "./src/worker/run.ts"; export { installChildTestAgent, testAgentChildDeclaration } from "./src/child-configuration.ts"; export type { PlanProviderAssembly, PlanProviderPolicy } from "./src/child-configuration.ts"; export type { ChildTestAgentInstallation } from "./src/child-configuration.ts"; + +export type { ScenarioModels, ScenarioModel, ScenarioEfforts } from "@executablemd/testing"; diff --git a/packages/test-agent/src/SessionConfiguration.defaults.md b/packages/test-agent/src/SessionConfiguration.defaults.md new file mode 100644 index 000000000..45ed704b8 --- /dev/null +++ b/packages/test-agent/src/SessionConfiguration.defaults.md @@ -0,0 +1,5 @@ + +declared defaults + + +changed defaults diff --git a/packages/test-agent/src/SessionConfiguration.refusal-behavior.md b/packages/test-agent/src/SessionConfiguration.refusal-behavior.md new file mode 100644 index 000000000..34a19fc06 --- /dev/null +++ b/packages/test-agent/src/SessionConfiguration.refusal-behavior.md @@ -0,0 +1,5 @@ + +restored + + +still restored diff --git a/packages/test-agent/src/SessionConfiguration.refusals.md b/packages/test-agent/src/SessionConfiguration.refusals.md new file mode 100644 index 000000000..b272af456 --- /dev/null +++ b/packages/test-agent/src/SessionConfiguration.refusals.md @@ -0,0 +1,27 @@ + + + + + + +not started + + +not started + +restored + +not started + +still restored + + + + + +not started + +unavailable + + + diff --git a/packages/test-agent/src/SessionConfiguration.review.md b/packages/test-agent/src/SessionConfiguration.review.md new file mode 100644 index 000000000..47a587d49 --- /dev/null +++ b/packages/test-agent/src/SessionConfiguration.review.md @@ -0,0 +1,14 @@ + +small low + +review deep + +small high + +review brief + +stored high + +same high + +restored high diff --git a/packages/test-agent/src/SessionConfiguration.test.md b/packages/test-agent/src/SessionConfiguration.test.md new file mode 100644 index 000000000..c2e495735 --- /dev/null +++ b/packages/test-agent/src/SessionConfiguration.test.md @@ -0,0 +1,48 @@ +# Scenario model and effort selection + + + + + + + + + +restored + +configured + +remember high +other model +stored effort +same model +restored high + + + + + +defaults +changed + + + +defaults + + + +unavailable + + + + + +defaultschanged'} as="first"> + + + +defaults'} as="second"> + + + + diff --git a/packages/test-agent/src/SessionConfiguration.unavailable.md b/packages/test-agent/src/SessionConfiguration.unavailable.md new file mode 100644 index 000000000..96c24b043 --- /dev/null +++ b/packages/test-agent/src/SessionConfiguration.unavailable.md @@ -0,0 +1,2 @@ + +choices unavailable diff --git a/packages/test-agent/src/child-configuration.ts b/packages/test-agent/src/child-configuration.ts index 811136e90..eaf512546 100644 --- a/packages/test-agent/src/child-configuration.ts +++ b/packages/test-agent/src/child-configuration.ts @@ -56,6 +56,8 @@ import { import type { ScenarioDeclaration } from "./components.ts"; import { TEST_AGENT_PROVIDER } from "./provider.ts"; +import { normalizeModels } from "./configuration.ts"; + const TEST_AGENT = "TestAgent"; const SCENARIO = "TestAgent.Scenario"; @@ -107,6 +109,13 @@ function open(collect: { refuse(`<${SCENARIO}> requires a "src" prop.`); return ""; } + let models; + try { + models = normalizeModels(props.models); + } catch (error) { + refuse(`<${SCENARIO}> ${error instanceof Error ? error.message : String(error)}`); + return ""; + } const read = yield* readScenarioSource(src); if (!read.ok) { refuse(`<${SCENARIO}> cannot read the behavior document "${src}": ${read.error.message}`); @@ -128,6 +137,7 @@ function open(collect: { session, rootDir: read.value.rootDir, document: read.value.document, + ...(models === undefined ? {} : { models }), }); return ""; }; @@ -266,6 +276,7 @@ export function* installChildTestAgent( rootDir: scenario.rootDir, document: { path: scenario.document.path, source: scenario.document.source }, duplicate: false, + models: normalizeModels(scenario.models), }); } const partition = yield* provisionPartition({ diff --git a/packages/test-agent/src/components.ts b/packages/test-agent/src/components.ts index 842ce996c..ded902667 100644 --- a/packages/test-agent/src/components.ts +++ b/packages/test-agent/src/components.ts @@ -38,6 +38,8 @@ import type { ScenarioHandle, TestAgentControllerInternals } from "./controller. import { createControlledExecutableObserver } from "./executable-observer.ts"; import { createDeterministicSessionCoordinator } from "./session-coordinator.ts"; import type { AcpxProviderDependencies } from "@executablemd/acp"; +import { normalizeModels, MODELS_PROPS } from "./configuration.ts"; +import type { ScenarioModels } from "@executablemd/testing"; import { duplicateScenario, missingScenario } from "./child-configuration.ts"; import { TEST_AGENT_CLIENT_NATIVE, TEST_AGENT_PROVIDER, useTestAgentProvider } from "./provider.ts"; import type { SessionRouting } from "./provider.ts"; @@ -47,6 +49,7 @@ export const DEFAULT_TEST_AGENT = "test"; /** One `` mapping, before any worker exists. */ export interface ScenarioDeclaration { + models?: ScenarioModels; agent: string; sessionName: string; rootDir: string; @@ -196,6 +199,7 @@ export function* provisionPartition(options: { try { scenario = yield* controller.useScenario({ document: declared.document, + models: declared.models, rootDir: declared.rootDir, }); } catch (error) { @@ -414,6 +418,7 @@ export function* Scenario(props: Record): Operation { return reported.message; } + const models = normalizeModels(props.models); const read = yield* readScenarioSource(src); if (!read.ok) { // The read's own failure, unwrapped: a wrapper's scenario is read while a @@ -436,6 +441,7 @@ export function* Scenario(props: Record): Operation { rootDir: read.value.rootDir, document: read.value.document, duplicate: false, + ...(models === undefined ? {} : { models }), }); return ""; } @@ -497,6 +503,7 @@ export const SCENARIO_PROPS: PropsSchema = { type: "object", properties: { src: { type: "string" }, + models: MODELS_PROPS, agent: { type: "string" }, session: { type: "string" }, }, diff --git a/packages/test-agent/src/configuration.ts b/packages/test-agent/src/configuration.ts new file mode 100644 index 000000000..4364d34b9 --- /dev/null +++ b/packages/test-agent/src/configuration.ts @@ -0,0 +1,163 @@ +import { z } from "zod"; +import type { SessionConfigOption } from "@agentclientprotocol/sdk"; +import type { ScenarioModels } from "@executablemd/testing"; +import type { PropsSchema } from "@executablemd/core"; + +const id = z.string().min(1); +const efforts = z + .object({ selected: id, options: z.array(id).min(1) }) + .strict() + .refine( + (value) => + new Set(value.options).size === value.options.length && + value.options.includes(value.selected), + "effort selections must be unique members of their options", + ); +export const modelsSchema = z + .object({ + selected: id, + options: z.array(z.object({ id, efforts: efforts.nullish() }).strict()).min(1), + }) + .strict() + .refine( + (value) => + new Set(value.options.map((option) => option.id)).size === value.options.length && + value.options.some((option) => option.id === value.selected), + "model selections must be unique members of their options", + ); + +export function normalizeModels(value: unknown): ScenarioModels | undefined { + if (value === null || value === undefined) { + return undefined; + } + const parsed = modelsSchema.parse(value); + return Object.freeze({ + selected: parsed.selected, + options: Object.freeze( + parsed.options.map((model) => + Object.freeze({ + id: model.id, + ...(model.efforts == null + ? {} + : { + efforts: Object.freeze({ + selected: model.efforts.selected, + options: Object.freeze([...model.efforts.options]), + }), + }), + }), + ), + ), + }); +} + +export function selectConfiguration( + models: ScenarioModels | undefined, + configId: string, + value: string, +): ScenarioModels { + if (models === undefined) { + throw new Error("Configuration choices are unavailable."); + } + if (configId === "model" && models.options.some((model) => model.id === value)) { + return { ...models, selected: value }; + } + const selected = models.options.find((model) => model.id === models.selected); + const efforts = selected?.efforts; + if (configId === "effort" && efforts?.options.includes(value)) { + return { + ...models, + options: models.options.map((model) => + model.id !== selected?.id ? model : { ...model, efforts: { ...efforts, selected: value } }, + ), + }; + } + throw new Error("Unsupported configuration choice."); +} + +export interface ConfigurationSnapshot { + model: string | null; + effort: string | null; +} +export function configurationSnapshot(models: ScenarioModels | undefined): ConfigurationSnapshot { + return { + model: models?.selected ?? null, + effort: + models?.options.find((model) => model.id === models.selected)?.efforts?.selected ?? null, + }; +} + +export function configOptions( + models: ScenarioModels | undefined, +): SessionConfigOption[] | undefined { + if (models === undefined) { + return undefined; + } + const options: SessionConfigOption[] = [ + { + id: "model", + name: "model", + category: "model", + type: "select", + currentValue: models.selected, + options: models.options.map((model) => ({ value: model.id, name: model.id })), + }, + ]; + const selected = models.options.find((model) => model.id === models.selected); + if (selected?.efforts != null) { + options.push({ + id: "effort", + name: "effort", + category: "thought_level", + type: "select", + currentValue: selected.efforts.selected, + options: selected.efforts.options.map((value) => ({ value, name: value })), + }); + } + return options; +} + +export const MODELS_PROPS: PropsSchema = { + anyOf: [ + { type: "null" }, + { + type: "object", + properties: { + selected: { type: "string", minLength: 1 }, + options: { + type: "array", + minItems: 1, + items: { + type: "object", + properties: { + id: { type: "string", minLength: 1 }, + efforts: { + anyOf: [ + { type: "null" }, + { + type: "object", + properties: { + selected: { type: "string", minLength: 1 }, + options: { + type: "array", + minItems: 1, + uniqueItems: true, + items: { type: "string", minLength: 1 }, + }, + }, + required: ["selected", "options"], + additionalProperties: false, + }, + ], + }, + }, + required: ["id"], + additionalProperties: false, + }, + }, + }, + required: ["selected", "options"], + additionalProperties: false, + }, + ], +}; diff --git a/packages/test-agent/src/controller.ts b/packages/test-agent/src/controller.ts index f27327c79..dca0ceafc 100644 --- a/packages/test-agent/src/controller.ts +++ b/packages/test-agent/src/controller.ts @@ -24,6 +24,8 @@ import type { NativeLaunchOutcome, NativeLaunchRequest } from "@executablemd/run import type { DurableEvent } from "@executablemd/durable-streams"; import { encodeMessage, formatRoute, parseWorkerMessage, PROBE_INSTANCE } from "./protocol.ts"; import type { ControllerMessage, WorkerMessage } from "./protocol.ts"; +import { normalizeModels, selectConfiguration } from "./configuration.ts"; +import type { ScenarioModels } from "@executablemd/testing"; import { useLineServer } from "./net.ts"; import type { LineSocket } from "./net.ts"; @@ -41,6 +43,7 @@ export interface TestAgentController { useScenario(options: { document: { path: string; source: string }; rootDir: string; + models?: ScenarioModels | null; }): Operation; } @@ -103,6 +106,7 @@ export interface ScenarioFailure { * handle. */ export interface ScenarioRecord extends ScenarioHandle { + models?: ScenarioModels; id: string; /** The real directory Markdown dependencies are served from. */ rootDir: string; @@ -132,6 +136,7 @@ export interface TestAgentControllerInternals extends TestAgentController { useScenario(options: { document: { path: string; source: string }; rootDir: string; + models?: ScenarioModels | null; }): Operation; getScenarioRecord(id: string): ScenarioRecord | undefined; } @@ -276,6 +281,7 @@ export function useTestAgentController(): Operation { return resource(function* (provide) { const id = randomUUID(); @@ -418,6 +437,7 @@ export function useTestAgentController(): Operation = z.lazy(() => z.union([ z.null(), @@ -27,6 +30,7 @@ export type WireDurableEvent = DurableEvent; export type WorkerMessage = | { t: "attach"; token: string; instance: string } | { t: "journal"; seq: number; event: WireDurableEvent } + | { t: "configuration"; configId: string; value: string } | { t: "read"; path: string } | { t: "stat"; path: string } | { t: "session"; systemPrompt?: string } @@ -51,7 +55,9 @@ export type ControllerMessage = * already exists rather than minting a replacement for it. */ nativeSessionId: string; + models?: ScenarioModels; } + | { t: "configuration"; models: ScenarioModels } | { t: "ack"; seq: number } | { t: "recorded" } | { t: "read"; path: string; source?: string; missing: boolean } @@ -92,6 +98,11 @@ const durableEvent: z.ZodType = z.union([ const workerMessage: z.ZodType = z.discriminatedUnion("t", [ z.object({ t: z.literal("attach"), token: z.string(), instance: z.string() }), z.object({ t: z.literal("journal"), seq: z.number().int().nonnegative(), event: durableEvent }), + z.object({ + t: z.literal("configuration"), + configId: z.string().min(1), + value: z.string().min(1), + }), z.object({ t: z.literal("read"), path: z.string() }), z.object({ t: z.literal("stat"), path: z.string() }), z.object({ t: z.literal("session"), systemPrompt: z.string().optional() }), @@ -112,11 +123,13 @@ const configMessage: z.ZodType = z.discriminatedUnion("mode", doc: z.object({ path: z.string(), source: z.string() }), journal: z.array(durableEvent), nativeSessionId: z.string(), + models: modelsSchema.optional(), }), ]); const controllerMessage: z.ZodType = z.union([ configMessage, + z.object({ t: z.literal("configuration"), models: modelsSchema }), z.object({ t: z.literal("ack"), seq: z.number().int().nonnegative() }), z.object({ t: z.literal("recorded") }), z.object({ diff --git a/packages/test-agent/src/worker/acp-server.ts b/packages/test-agent/src/worker/acp-server.ts index a332afb36..d2674513c 100644 --- a/packages/test-agent/src/worker/acp-server.ts +++ b/packages/test-agent/src/worker/acp-server.ts @@ -37,6 +37,8 @@ export interface WorkerAgent { */ startSession(instructions: string | undefined): Operation; runTurn(text: string): Operation; + configuration?(): Operation; + setConfiguration?(configId: string, value: string): Operation; cancel(): void; } @@ -129,7 +131,13 @@ function useAcpConnection( handle(function* () { yield* worker.ready(); const nativeSessionId = yield* worker.startSession(systemPromptOf(ctx.params._meta)); - return { sessionId: randomUUID(), _meta: { agentSessionId: nativeSessionId } }; + const configOptions = + worker.configuration === undefined ? [] : ((yield* worker.configuration()) ?? []); + return { + sessionId: randomUUID(), + _meta: { agentSessionId: nativeSessionId }, + ...(configOptions.length === 0 ? {} : { configOptions }), + }; }), ) .onRequest("session/load", (ctx) => @@ -141,7 +149,22 @@ function useAcpConnection( // conversation than the one that was named — which is exactly what // attaching by name exists to rule out. The scenario's own identity // stands in only when nothing was named. - return { _meta: { agentSessionId: ctx.params.sessionId ?? nativeSessionId } }; + const configOptions = + worker.configuration === undefined ? [] : ((yield* worker.configuration()) ?? []); + return { + _meta: { agentSessionId: ctx.params.sessionId ?? nativeSessionId }, + ...(configOptions.length === 0 ? {} : { configOptions }), + }; + }), + ) + .onRequest("session/set_config_option", (ctx) => + handle(function* () { + if (worker.setConfiguration === undefined || typeof ctx.params.value !== "string") { + throw new acp.RequestError(-32602, "Configuration choices are unavailable."); + } + return { + configOptions: yield* worker.setConfiguration(ctx.params.configId, ctx.params.value), + }; }), ) .onRequest("session/close", () => ({})) diff --git a/packages/test-agent/src/worker/bridge.ts b/packages/test-agent/src/worker/bridge.ts index a41b8770b..848bfc921 100644 --- a/packages/test-agent/src/worker/bridge.ts +++ b/packages/test-agent/src/worker/bridge.ts @@ -10,6 +10,8 @@ import { createChannel, withResolvers } from "effection"; import type { Channel, Operation, Result, Subscription } from "effection"; import type { Captures } from "@executablemd/core"; +import type { ConfigurationSnapshot } from "../configuration.ts"; + export type BridgeEvent = | { kind: "output"; text: string } | { kind: "suspended"; stage: string } @@ -18,12 +20,13 @@ export type BridgeEvent = export interface PromptOffer { text: string; + configuration?: ConfigurationSnapshot; respond(outcome: Result): void; } export interface TurnBridge { events: Channel; - offer(text: string): Operation>; + offer(text: string, configuration?: ConfigurationSnapshot): Operation>; nextOffer(): Operation; } @@ -41,9 +44,13 @@ export function createTurnBridge(): TurnBridge { return { events, - *offer(text) { + *offer(text, configuration) { const outcome = withResolvers>(); - const offer: PromptOffer = { text, respond: outcome.resolve }; + const offer: PromptOffer = { + text, + ...(configuration === undefined ? {} : { configuration: { ...configuration } }), + respond: outcome.resolve, + }; const waiter = waiters.shift(); if (waiter) { waiter(offer); diff --git a/packages/test-agent/src/worker/run.ts b/packages/test-agent/src/worker/run.ts index ad749b776..3200ff73c 100644 --- a/packages/test-agent/src/worker/run.ts +++ b/packages/test-agent/src/worker/run.ts @@ -23,6 +23,8 @@ import { InMemoryStream } from "@executablemd/durable-streams"; import type { DurableEvent } from "@executablemd/durable-streams"; import { encodeMessage, parseControllerMessage, parseRoute, PROBE_INSTANCE } from "../protocol.ts"; import type { ControllerMessage, WorkerMessage } from "../protocol.ts"; +import { configOptions, configurationSnapshot } from "../configuration.ts"; +import type { ConfigurationSnapshot } from "../configuration.ts"; import { createTurnBridge, collectTurn } from "./bridge.ts"; import type { Captures } from "@executablemd/core"; import { PromptMismatchError } from "@executablemd/core"; @@ -58,7 +60,7 @@ interface ScenarioRuntime { task: Task; turnEvents: Subscription; ready: Operation; - offer(text: string): Operation>; + offer(text: string, configuration: ConfigurationSnapshot): Operation>; flush(): Operation; isExhausted(): boolean; markExhausted(): void; @@ -103,6 +105,7 @@ export function* runTestAgentWorker(options: { connect: string }): Operation bridge.offer(text), + offer: (text, configuration) => bridge.offer(text, configuration), flush, isExhausted: () => exhausted, markExhausted: () => { @@ -261,6 +264,7 @@ export function* runTestAgentWorker(options: { connect: string }): Operation void) | undefined; + let configurationPending = false; function* restart(): Operation { yield* current.task.halt(); @@ -287,7 +291,36 @@ export function* runTestAgentWorker(options: { connect: string }): Operation { + if (configurationPending) { + throw new RequestError(-32602, "Scenario configuration is still being acknowledged."); + } + const configuration = configurationSnapshot(models); if (current.isExhausted()) { yield* report({ t: "turn-failure", kind: "exhausted", actual: text }); throw new RequestError( @@ -302,10 +335,8 @@ export function* runTestAgentWorker(options: { connect: string }): Operation { - const match = yield* runtime.offer(text); + const match = yield* runtime.offer(text, configuration); if (!match.ok) { - // Only matchPrompt fails an offer, so the mismatch detail is - // always there; `text` is the prompt it compared against. const mismatch = match.error instanceof PromptMismatchError ? match.error : undefined; const failure: WorkerMessage = { diff --git a/packages/test-agent/src/worker/when-prompt.ts b/packages/test-agent/src/worker/when-prompt.ts index 0529ab36d..6dad0d891 100644 --- a/packages/test-agent/src/worker/when-prompt.ts +++ b/packages/test-agent/src/worker/when-prompt.ts @@ -7,12 +7,13 @@ * advances to the active matcher without re-matching. */ -import { scoped } from "effection"; +import { scoped, Err } from "effection"; import type { Operation } from "effection"; import { createDurableOperation } from "@executablemd/durable-streams"; import type { Json, Workflow } from "@executablemd/durable-streams"; import { env, + PromptMismatchError, hasContent, getExpansion, matchPrompt, @@ -32,14 +33,22 @@ import type { import type { TurnBridge } from "./bridge.ts"; +import { z } from "zod"; +import type { ConfigurationSnapshot } from "../configuration.ts"; + +const snapshotSchema = z + .object({ model: z.string().min(1).nullable(), effort: z.string().min(1).nullable() }) + .strict(); + const WHEN_PROMPT = "when_prompt"; interface StageRecord { prompt: string; captures: Record; + configuration?: ConfigurationSnapshot; } -function parseStageRecord(value: unknown): StageRecord | undefined { +function parseStageRecord(value: unknown, constrained: boolean): StageRecord | undefined { if (typeof value !== "object" || value === null || Array.isArray(value)) { return undefined; } @@ -60,7 +69,17 @@ function parseStageRecord(value: unknown): StageRecord | undefined { } parsed[name] = text; } - return { prompt, captures: parsed }; + const snapshot = snapshotSchema.safeParse( + "configuration" in value ? value.configuration : undefined, + ); + if (constrained && !snapshot.success) { + return undefined; + } + return { + prompt, + captures: parsed, + ...(constrained && snapshot.success ? { configuration: snapshot.data } : {}), + }; } function configError(message: string): ErrorSegment { @@ -84,13 +103,36 @@ function* awaitMatch( bridge: TurnBridge, template: ParsedTemplate, bindings: Record, + constraints: { model?: string | null; effort?: string | null }, ): Operation { while (true) { const offer = yield* bridge.nextOffer(); - const outcome = matchPrompt(template, offer.text, bindings); + let outcome = matchPrompt(template, offer.text, bindings); + const constrained = Object.keys(constraints).length > 0; + if ( + outcome.ok && + constrained && + (!offer.configuration || + Object.entries(constraints).some( + ([key, value]) => offer.configuration?.[key === "model" ? "model" : "effort"] !== value, + )) + ) { + outcome = Err( + new PromptMismatchError( + "mismatch", + JSON.stringify(constraints), + JSON.stringify(offer.configuration), + "Prompt configuration does not match constraints.", + ), + ); + } offer.respond(outcome); if (outcome.ok) { - return { prompt: offer.text, captures: outcome.value }; + return { + prompt: offer.text, + captures: outcome.value, + ...(constrained ? { configuration: offer.configuration } : {}), + }; } } } @@ -98,6 +140,7 @@ function* awaitMatch( function* persistStage( identity: { name: string; input: string; position?: Readonly }, live: () => Operation, + constrained: boolean, ): Workflow { const stored = yield createDurableOperation( { @@ -108,10 +151,16 @@ function* persistStage( }, function* (): Operation { const record = yield* live(); - return { prompt: record.prompt, captures: record.captures }; + return { + prompt: record.prompt, + captures: record.captures, + ...(record.configuration === undefined + ? {} + : { configuration: { ...record.configuration } }), + }; }, ); - const parsed = parseStageRecord(stored); + const parsed = parseStageRecord(stored, constrained); if (!parsed) { throw new Error(`journaled when_prompt "${identity.name}" has an unexpected shape`); } @@ -160,6 +209,14 @@ export function* installWhenPromptComponent(bridge: TurnBridge): Operation return reported.message; } + const constraints: { model?: string | null; effort?: string | null } = {}; + if (props.model !== undefined) { + constraints.model = typeof props.model === "string" ? props.model : null; + } + if (props.effort !== undefined) { + constraints.effort = typeof props.effort === "string" ? props.effort : null; + } + const constrained = Object.keys(constraints).length > 0; const expansion = yield* getExpansion(); const location = formatLocation(expansion); const ordinal = ordinals.get(location) ?? 0; @@ -172,14 +229,31 @@ export function* installWhenPromptComponent(bridge: TurnBridge): Operation // follows all of that stage's output through one ordered channel, // so the collector never loses the final chunk. const record = yield* persistStage( - { name: `when:${location}#${ordinal}`, input: source, position: expansion.position }, + { + name: `when:${location}#${ordinal}`, + input: constrained + ? JSON.stringify({ template: source, configuration: constraints }) + : source, + position: expansion.position, + }, () => scoped(function* () { yield* bridge.events.send({ kind: "suspended", stage: source }); - return yield* awaitMatch(bridge, parsed.value, currentEnv.values); + return yield* awaitMatch(bridge, parsed.value, currentEnv.values, constraints); }), + constrained, ); + if ( + constrained && + (!record.configuration || + Object.entries(constraints).some( + ([key, value]) => record.configuration?.[key === "model" ? "model" : "effort"] !== value, + )) + ) { + throw new Error("Journaled prompt configuration does not satisfy its recorded constraints."); + } + // Returned rather than bound here: `as` is the engine's, and a return binds // by reference under it. A template with captures invoked without `as` // discards them — the component cannot see whether `as` was written. @@ -201,6 +275,10 @@ export function* installWhenPromptComponent(bridge: TurnBridge): Operation const WHEN_PROMPT_PROPS: PropsSchema = { type: "object", - properties: { template: { type: "string" } }, + properties: { + template: { type: "string" }, + model: { anyOf: [{ type: "string", minLength: 1 }, { type: "null" }] }, + effort: { anyOf: [{ type: "string", minLength: 1 }, { type: "null" }] }, + }, additionalProperties: false, }; diff --git a/packages/test-agent/tests/behavior-engine.test.ts b/packages/test-agent/tests/behavior-engine.test.ts index 4a129dc25..3a5e7184d 100644 --- a/packages/test-agent/tests/behavior-engine.test.ts +++ b/packages/test-agent/tests/behavior-engine.test.ts @@ -16,6 +16,7 @@ import * as path from "node:path"; import { Component, DocumentOutput, execute } from "@executablemd/core"; import { InMemoryStream } from "@executablemd/durable-streams"; import { collectTurn, createTurnBridge } from "../src/worker/bridge.ts"; +import type { ConfigurationSnapshot } from "../src/configuration.ts"; import type { BridgeEvent } from "../src/worker/bridge.ts"; import type { Captures } from "@executablemd/core"; import { installWhenPromptComponent } from "../src/worker/when-prompt.ts"; @@ -36,7 +37,7 @@ const REVIEW = [ interface Engine { turnEvents: Subscription; - offer(text: string): Operation>; + offer(text: string, configuration?: ConfigurationSnapshot): Operation>; } function* useEngine(source: string): Operation { @@ -74,7 +75,7 @@ function* useEngine(source: string): Operation { } }); - return { turnEvents, offer: (text: string) => bridge.offer(text) }; + return { turnEvents, offer: (text, configuration) => bridge.offer(text, configuration) }; } /** @@ -89,6 +90,22 @@ function withEngine(source: string, body: (engine: Engine) => Operation): } describe("Tier BE — behavior engine", () => { + it("matches text and actual configuration as one offer without consuming mismatches", function* () { + const engine = yield* useEngine( + '\n\n{r.subject}\n\n\n', + ); + yield* collectTurn(engine.turnEvents); + expect((yield* engine.offer("Review core", { model: "A", effort: "deep" })).ok).toBe(false); + expect((yield* engine.offer("Review core", { model: "B", effort: "brief" })).ok).toBe(false); + expect((yield* engine.offer("Review core")).ok).toBe(false); + expect(yield* engine.offer("Review core", { model: "B", effort: "deep" })).toEqual( + Ok({ subject: "core" }), + ); + expect((yield* collectTurn(engine.turnEvents)).text).toContain("core"); + expect((yield* engine.offer("done", { model: "B", effort: null })).ok).toBe(false); + expect((yield* engine.offer("done", { model: null, effort: null })).ok).toBe(true); + expect((yield* collectTurn(engine.turnEvents)).end).toBe("eof"); + }); it("BE1: a matched prompt advances the stage, exposes captures, and reaches the second stage", function* () { const engine = yield* useEngine(REVIEW); diff --git a/packages/test-agent/tests/components.test.ts b/packages/test-agent/tests/components.test.ts index f4d142255..88a2874fb 100644 --- a/packages/test-agent/tests/components.test.ts +++ b/packages/test-agent/tests/components.test.ts @@ -27,10 +27,14 @@ import type { AnswerConfiguration } from "@executablemd/core/host"; import { executeInstalled } from "@executablemd/core/host"; import { API } from "@executablemd/runtime"; import { InMemoryStream } from "@executablemd/durable-streams"; -import { installTestingComponents, useTesting } from "@executablemd/testing"; +import { + installTestingComponents, + testHarnessInstallation, + useTesting, +} from "@executablemd/testing"; import type { TestResult } from "@executablemd/testing"; import { installTestAgentComponents } from "../src/components.ts"; -import { installChildTestAgent } from "../src/child-configuration.ts"; +import { installChildTestAgent, testAgentChildDeclaration } from "../src/child-configuration.ts"; import { useCommand } from "./command.ts"; import { cliBase } from "@executablemd/test-support/launch"; import type { Json } from "@executablemd/core"; @@ -44,6 +48,7 @@ interface Run { } interface RunOptions { + child?: boolean; session?: boolean; /** * Search these for repository components, relative to the fixture directory. @@ -107,7 +112,21 @@ function* runDoc(files: Record, options?: RunOptions): Operation ? { includes: options.includes.map((d) => path.join(dir, d)) } : {}), }, - [{ components: agentIdentityComponents() }], + [ + { components: agentIdentityComponents() }, + ...(options?.child + ? [ + testHarnessInstallation( + { + *runChild() { + throw new Error("invalid catalog reached child provisioning"); + }, + }, + [testAgentChildDeclaration()], + ), + ] + : []), + ], ); const subscription = yield* execution.output; let next = yield* subscription.next(); @@ -127,6 +146,48 @@ const TWO_STAGES = describe("Tier TV — TestAgent components", { sanitizeOps: false, sanitizeResources: false }, () => { beforeAll(() => useTempFileCompiler()); + it("rejects malformed catalogs equivalently in wrapper and nested declarations before provisioning", function* () { + const invalid = [ + { selected: "A", options: [] }, + { selected: "A", options: [{ id: "A" }, { id: "A" }] }, + { selected: "missing", options: [{ id: "A" }] }, + { selected: "A", options: [{ id: "" }] }, + { selected: "A", options: [{ id: "A", efforts: { selected: "low", options: [] } }] }, + { selected: "A", options: [{ id: "A", efforts: { selected: "missing", options: ["low"] } }] }, + { + selected: "A", + options: [{ id: "A", efforts: { selected: "low", options: ["low", "low"] } }], + }, + ]; + for (const models of invalid) { + const declaration = ``; + const wrapper = yield* runDoc({ + "hi.md": HI, + "doc.md": `\n${declaration}\nnever reached\n\n`, + }); + expect(wrapper.result.ok).toBe(false); + const nested = yield* runDoc( + { + "hi.md": HI, + "doc.md": `\n\n\n${declaration}\n\n\n\n`, + }, + { child: true }, + ); + expect(nested.results.map((result) => result.status)).toEqual(["fail"]); + expect(nested.output).not.toContain("invalid catalog reached child provisioning"); + } + }); + + it("fails the owning Test when actual prompt configuration violates a constraint", function* () { + const run = yield* runDoc({ + "hi.md": '\n\nwrong\n', + "doc.md": + '\n\n\nhi\n\n\n', + }); + expect(run.results.map((result) => result.status)).toEqual(["fail"]); + expect(run.output).toContain("Prompt configuration does not match"); + }); + it("TV1: outside an active testing session is a configuration error", function* () { const run = yield* runDoc( { "doc.md": "\nbody\n\n" }, diff --git a/packages/test-agent/tests/controller.test.ts b/packages/test-agent/tests/controller.test.ts index f2acf68ec..e9b6153d7 100644 --- a/packages/test-agent/tests/controller.test.ts +++ b/packages/test-agent/tests/controller.test.ts @@ -51,7 +51,82 @@ function* useClient(route: string): Operation { }; } +const MODELS = { + selected: "A", + options: [ + { id: "A", efforts: { selected: "low", options: ["low", "high"] } }, + { id: "B", efforts: { selected: "brief", options: ["brief", "deep"] } }, + { id: "none" }, + ], +}; + describe("Tier TC — controller", () => { + it("acknowledges exact selections per scenario, retaining each model's effort", function* () { + const controller = yield* useTestAgentController(); + const scenario = yield* controller.useScenario({ + document: { path: "config.md", source: '' }, + rootDir: os.tmpdir(), + models: MODELS, + }); + const sibling = yield* controller.useScenario({ + document: { path: "config.md", source: '' }, + rootDir: os.tmpdir(), + models: MODELS, + }); + yield* scoped(function* () { + const parsed = parseRoute(scenario.route); + if (!parsed.ok) { + throw parsed.error; + } + const client = yield* useClient(scenario.route); + client.send({ t: "attach", token: parsed.value.token, instance: scenario.id }); + expect(yield* client.next()).toMatchObject({ t: "config", models: MODELS }); + for (const [configId, value] of [ + ["effort", "high"], + ["model", "B"], + ["effort", "deep"], + ["model", "A"], + ["model", "A"], + ]) { + client.send({ t: "configuration", configId, value }); + expect((yield* client.next()).t).toBe("configuration"); + } + const record = controller.getScenarioRecord(scenario.id); + expect(record?.models?.selected).toBe("A"); + expect(record?.models?.options[0].efforts?.selected).toBe("high"); + expect(record?.models?.options[1].efforts?.selected).toBe("deep"); + const before = JSON.stringify(record?.models); + for (const [configId, value] of [ + ["model", "unknown"], + ["effort", "deep"], + ["other", "A"], + ]) { + client.send({ t: "configuration", configId, value }); + expect((yield* client.next()).t).toBe("error"); + expect(JSON.stringify(record?.models)).toBe(before); + } + expect(controller.getScenarioRecord(sibling.id)?.models).toEqual(MODELS); + }); + yield* scoped(function* () { + const parsed = parseRoute(scenario.route); + if (!parsed.ok) { + throw parsed.error; + } + const client = yield* useClient(scenario.route); + client.send({ t: "attach", token: parsed.value.token, instance: scenario.id }); + expect(yield* client.next()).toMatchObject({ + t: "config", + models: { + selected: "A", + options: [ + { id: "A", efforts: { selected: "high" } }, + { id: "B", efforts: { selected: "deep" } }, + { id: "none" }, + ], + }, + }); + }); + }); it("TC1: probe attach configures probe mode; bad tokens are rejected", function* () { const controller = yield* useTestAgentController(); const parsed = parseRoute(controller.probeRoute); diff --git a/packages/test-agent/tests/protocol.test.ts b/packages/test-agent/tests/protocol.test.ts index 7f1feef9e..52f5bedba 100644 --- a/packages/test-agent/tests/protocol.test.ts +++ b/packages/test-agent/tests/protocol.test.ts @@ -14,6 +14,26 @@ import { } from "../src/protocol.ts"; describe("Tier TP — controller protocol", () => { + it("validates configuration requests and acknowledged catalogs without widening legacy config", function* () { + const models = { + selected: "A", + options: [{ id: "A", efforts: { selected: "low", options: ["low", "high"] } }], + }; + const request = { t: "configuration", configId: "model", value: "A" }; + expect(parseWorkerMessage(JSON.stringify(request))).toMatchObject({ ok: true, value: request }); + expect(parseWorkerMessage(JSON.stringify({ ...request, value: "" })).ok).toBe(false); + const reply = { t: "configuration", models }; + expect(parseControllerMessage(JSON.stringify(reply))).toMatchObject({ ok: true, value: reply }); + for (const malformed of [ + { selected: "A", options: [] }, + { selected: "X", options: [{ id: "A" }] }, + { selected: "A", options: [{ id: "A" }, { id: "A" }] }, + ]) { + expect( + parseControllerMessage(JSON.stringify({ t: "configuration", models: malformed })).ok, + ).toBe(false); + } + }); it("TP1: worker messages round-trip through encode and parse", function* () { const line = encodeMessage({ t: "attach", token: "tok", instance: "inst" }); const result = parseWorkerMessage(line.trimEnd()); diff --git a/packages/test-agent/tests/session-configuration.test.ts b/packages/test-agent/tests/session-configuration.test.ts new file mode 100644 index 000000000..311f6d829 --- /dev/null +++ b/packages/test-agent/tests/session-configuration.test.ts @@ -0,0 +1,228 @@ +import { describe, it } from "@executablemd/test-support/bdd"; +import { executeInstalled } from "@executablemd/core/host"; +import { expect } from "@executablemd/test-support/expect"; +import { scoped } from "effection"; +import type { Operation, Result } from "effection"; +import * as path from "node:path"; +import { + agentIdentityComponents, + content, + registerComponents, + inlineSource, + installAgentComponents, +} from "@executablemd/core"; +import { installAnswerProvider } from "@executablemd/core/host"; +import type { ExecutionInstallation } from "@executablemd/core/host"; +import { forEach } from "@effectionx/stream-helpers"; +import { useHostFiles } from "@executablemd/runtime"; +import { InMemoryStream } from "@executablemd/durable-streams"; +import { testHarnessInstallation, useTesting } from "@executablemd/testing"; +import type { + ChildInvocation, + ChildSettlement, + ExecutionHostProvider, + TestResult, +} from "@executablemd/testing"; +import { installTestAgentComponents } from "../src/components.ts"; +import { installChildTestAgent, testAgentChildDeclaration } from "../src/child-configuration.ts"; +import { useCommand } from "./command.ts"; +import { cliBase, runCli } from "@executablemd/test-support/launch"; +import type { Json } from "@executablemd/core"; +import { NativeSessionObserver } from "../src/controller.ts"; +import type { NativeSessionReport } from "../src/controller.ts"; + +const DOC = path.resolve("packages/test-agent/src/SessionConfiguration.test.md"); + +// The fixture relaunches xmd as a worker, so the run keeps this process's +// working directory and its whole environment. +const RUN = { inheritEnv: true, timeout: 120_000 }; + +/** + * The trusted `run` profile, as this package can state it. + * + * The shipped one is the CLI's, and the CLI depends on this package — so what + * stands in here is the same assembly the CLI performs for a declared child: + * this package's provider built from the frozen scenario data, the Agent + * defaults, ``, and core's matcher provider. The CLI case covers the real one. + */ +function nestedRunHost(worker: string[]): ExecutionHostProvider { + return { + *runChild(invocation: ChildInvocation): Operation { + const request = invocation.request; + const installations: ExecutionInstallation[] = []; + for (const configuration of request.configuration ?? []) { + switch (configuration.kind) { + case "test-agent": + yield* installAgentComponents( + (yield* installChildTestAgent(configuration, { workerCommand: worker })).components, + ); + installations.push({ components: agentIdentityComponents() }); + break; + case "answers": + yield* installAnswerProvider(configuration); + break; + } + } + yield* useHostFiles(); + const execution = yield* executeInstalled( + { + ...inlineSource(request.source ?? ""), + stream: new InMemoryStream(), + props: request.props, + }, + installations, + ); + const output = yield* forEach(function* (chunk: string) { + yield* invocation.chunk(chunk); + }, execution.output); + return { outcome: { kind: "settled", result: yield* execution }, output }; + }, + }; +} + +function* runConfiguration( + stream: InMemoryStream, + xmd: string[], + doc = DOC, +): Operation<{ + result: Result; + output: string; + results: readonly TestResult[]; + sessions: NativeSessionReport[]; +}> { + // The scope closes before the assertions run, so the provider and its + // workers finish teardown and the completion settles. + return yield* scoped(function* () { + const sessions: NativeSessionReport[] = []; + yield* NativeSessionObserver.set((report) => sessions.push(report)); + // Session configuration refusals are raw Errors, outside AssertThrows' raised-segment contract. + yield* registerComponents([ + { + name: "ExpectRefusal", + origin: "session-configuration.test.ts", + props: { + type: "object", + properties: { message: { type: "string" } }, + required: ["message"], + additionalProperties: false, + }, + *fn(props) { + let caught: unknown; + try { + yield* content(); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(Error); + expect(caught instanceof Error ? caught.message : "").toContain(props.message); + return ""; + }, + }, + ]); + const testing = yield* useTesting(); + yield* useCommand(xmd); + yield* installTestAgentComponents(); + yield* installAgentComponents(); + const execution = yield* executeInstalled({ path: doc, stream }, [ + { components: agentIdentityComponents() }, + testHarnessInstallation(nestedRunHost([...xmd, "test-agent"]), [testAgentChildDeclaration()]), + ]); + const subscription = yield* execution.output; + let next = yield* subscription.next(); + while (!next.done) { + next = yield* subscription.next(); + } + const result = yield* execution; + const results = yield* testing.results; + return { result, output: next.value, results, sessions }; + }); +} + +function configurationSucceeded(run: { + result: Result; + output: string; + results: readonly TestResult[]; +}): void { + if (!run.result.ok || run.results.some((entry) => entry.status !== "pass")) { + throw new Error( + JSON.stringify( + { + error: run.result.ok + ? undefined + : { message: run.result.error.message, stack: run.result.error.stack }, + output: run.output, + results: run.results, + }, + (_key, value) => + value instanceof Error ? { message: value.message, stack: value.stack } : value, + 2, + ), + ); + } +} + +describe( + "Session configuration through TestAgent", + { sanitizeOps: false, sanitizeResources: false }, + () => { + it("runs configured Sessions and nested declarations through the source CLI and ACPX worker", function* () { + const cli = yield* runCli(["test", DOC], RUN).join(); + if (cli.code !== 0) { + throw new Error( + `SessionConfiguration CLI exited ${cli.code}\nstdout:\n${cli.stdout}\nstderr:\n${cli.stderr}`, + ); + } + expect(cli.stdout).toContain("review deep"); + expect(cli.stdout).not.toContain("ERROR"); + }); + + it("refuses unknown, unsupported and unavailable Session selections before consuming the first stage", function* () { + const run = yield* runConfiguration( + new InMemoryStream(), + cliBase(), + path.resolve("packages/test-agent/src/SessionConfiguration.refusals.md"), + ); + expect(run.result.ok ? "ok" : run.result.error.message).toBe("ok"); + expect(run.results.map((entry) => entry.status)).toEqual(["pass", "pass"]); + expect(new Set(run.sessions.map((report) => report.nativeSessionId)).size).toBe(2); + }); + + it("retains named native identity, isolates Tests, and replays without a worker", function* () { + const stream = new InMemoryStream(); + const live = yield* runConfiguration(stream, cliBase()); + configurationSucceeded(live); + expect(live.results.map((entry) => entry.status)).toEqual([ + "pass", + "pass", + "pass", + "pass", + "pass", + ]); + const byScenario = new Map>(); + for (const report of live.sessions) { + const ids = byScenario.get(report.scenarioId) ?? new Set(); + ids.add(report.nativeSessionId); + byScenario.set(report.scenarioId, ids); + } + expect(byScenario.size).toBeGreaterThanOrEqual(4); + expect([...byScenario.values()].every((ids) => ids.size === 1)).toBe(true); + expect(new Set(live.sessions.map((report) => report.nativeSessionId)).size).toBe( + byScenario.size, + ); + expect(live.sessions.length).toBeGreaterThan(byScenario.size); + const events = JSON.stringify(yield* stream.readAll()); + expect(events).toContain('"configuration":{"model":"fixture-review","effort":"deep"}'); + const appended = stream.appendCount; + const replay = yield* runConfiguration(stream, [ + "/nonexistent/xmd-configuration-must-not-spawn", + ]); + configurationSucceeded(replay); + expect(replay.results.map((entry) => entry.status)).toEqual( + live.results.map((entry) => entry.status), + ); + expect(replay.sessions).toEqual([]); + expect(replay.output).toBe(live.output); + expect(stream.appendCount).toBe(appended); + }); + }, +); diff --git a/packages/test-agent/tests/worker-lifecycle.test.ts b/packages/test-agent/tests/worker-lifecycle.test.ts index 35d32f83c..f390ce233 100644 --- a/packages/test-agent/tests/worker-lifecycle.test.ts +++ b/packages/test-agent/tests/worker-lifecycle.test.ts @@ -17,6 +17,8 @@ import * as path from "node:path"; import * as os from "node:os"; import process from "node:process"; import { useTestAgentController } from "../src/controller.ts"; +import { configOptions, selectConfiguration } from "../src/configuration.ts"; +import type { ScenarioModels } from "@executablemd/testing"; import type { TestAgentControllerInternals } from "../src/controller.ts"; import { useLineServer } from "../src/net.ts"; import { cliCommand } from "@executablemd/test-support/launch"; @@ -163,7 +165,7 @@ interface GateHandle { release(): void; } -type GateKind = "journal" | "recorded"; +type GateKind = "journal" | "recorded" | "configuration"; interface GatedController { route: string; @@ -178,7 +180,11 @@ interface GatedController { * worker inside a controller round-trip and drive cancellation * deterministically, without timing sleeps. */ -function* useGatedController(doc: { path: string; source: string }): Operation { +function* useGatedController( + doc: { path: string; source: string }, + initialModels?: ScenarioModels, +): Operation { + let models = initialModels; const journal: DurableEvent[] = []; let gate: { kind: GateKind; held: WithResolvers; release: WithResolvers } | undefined; function* holdIf(kind: GateKind): Operation { @@ -208,12 +214,20 @@ function* useGatedController(doc: { path: string; source: string }): Operation { + it("waits for configuration acknowledgement before returning refreshed choices or accepting a turn", function* () { + const controller = yield* useGatedController( + { + path: "config.md", + source: '\n\nacknowledged\n', + }, + MODELS, + ); + const worker = yield* useWorker(controller.route); + yield* worker.request("initialize", { protocolVersion: 1, clientCapabilities: {} }); + const created = yield* worker.request("session/new", { cwd: "/", mcpServers: [] }); + expect(created.result?.configOptions).toEqual(configOptions(MODELS)); + const gate = controller.armGate("configuration"); + let returned = false; + const pending = yield* spawn(function* () { + const result = yield* worker.request("session/set_config_option", { + sessionId: "s", + configId: "model", + value: "B", + }); + returned = true; + return result; + }); + yield* gate.held; + const refused = yield* worker.request("session/prompt", { + sessionId: "s", + prompt: [{ type: "text", text: "go" }], + }); + expect(refused.error?.message).toContain("still being acknowledged"); + expect(returned).toBe(false); + gate.release(); + expect((yield* pending).result?.configOptions).toEqual( + configOptions(selectConfiguration(MODELS, "model", "B")), + ); + expect( + (yield* worker.request("session/prompt", { + sessionId: "s", + prompt: [{ type: "text", text: "go" }], + })).result, + ).toMatchObject({ stopReason: "end_turn" }); + }); + + it("reconnects with acknowledged settings and rehydrates constrained observations, refusing a missing snapshot", function* () { + const source = + '\n\n{r.subject}\n\n\n\nfinished\n'; + const dir = path.join(os.tmpdir(), `xmd-config-${randomUUID()}`); + yield* ensureDir(dir); + yield* ensure(() => rm(dir, { recursive: true, force: true })); + const controller = yield* useTestAgentController(); + const scenario = yield* controller.useScenario({ + document: { path: "config.md", source }, + rootDir: dir, + models: MODELS, + }); + let nativeId: unknown; + let journalBefore = 0; + yield* scoped(function* () { + const worker = yield* useWorker(scenario.route); + yield* worker.request("initialize", { protocolVersion: 1, clientCapabilities: {} }); + const created = yield* worker.request("session/new", { cwd: "/", mcpServers: [] }); + const meta = created.result?._meta; + nativeId = isRecord(meta) ? meta.agentSessionId : undefined; + expect(typeof nativeId).toBe("string"); + const record = controller.getScenarioRecord(scenario.id)!; + journalBefore = record.journal.length; + const invalid = yield* worker.request("session/set_config_option", { + sessionId: nativeId, + configId: "model", + value: "unknown", + }); + expect(invalid.error).toBeDefined(); + expect(record.journal.length).toBe(journalBefore); + expect(record.models).toEqual(MODELS); + for (const [configId, value] of [ + ["model", "B"], + ["effort", "deep"], + ]) { + expect( + (yield* worker.request("session/set_config_option", { + sessionId: nativeId, + configId, + value, + })).error, + ).toBe(undefined); + } + const beforeMismatch = record.journal.length; + const mismatch = yield* worker.request("session/prompt", { + sessionId: nativeId, + prompt: [{ type: "text", text: "wrong" }], + }); + expect(mismatch.error).toBeDefined(); + expect(record.journal.length).toBe(beforeMismatch); + const matched = yield* worker.request("session/prompt", { + sessionId: nativeId, + prompt: [{ type: "text", text: "Review core" }], + }); + expect(matched.result).toMatchObject({ stopReason: "end_turn" }); + expect(chunkText(worker.notifications)).toContain("core"); + yield* worker.request("session/set_config_option", { + sessionId: nativeId, + configId: "model", + value: "A", + }); + const stages = record.journal.filter( + (event) => event.type === "yield" && event.description.type === "when_prompt", + ); + expect(JSON.stringify(stages)).toContain('"configuration":{"model":"B","effort":"deep"}'); + }); + yield* scoped(function* () { + const worker = yield* useWorker(scenario.route); + yield* worker.request("initialize", { protocolVersion: 1, clientCapabilities: {} }); + const loaded = yield* worker.request("session/load", { + sessionId: nativeId, + cwd: "/", + mcpServers: [], + }); + expect(loaded.error).toBe(undefined); + expect(loaded.result?.configOptions).toEqual(configOptions(MODELS)); + const renewed = yield* worker.request("session/new", { cwd: "/", mcpServers: [] }); + expect(renewed.result?._meta).toMatchObject({ agentSessionId: nativeId }); + }); + const record = controller.getScenarioRecord(scenario.id)!; + let removed = false; + for (const event of record.journal) { + if ( + event.type === "yield" && + event.description.type === "when_prompt" && + event.result.status === "ok" + ) { + const value = event.result.value; + if (isRecord(value) && "configuration" in value) { + delete value.configuration; + removed = true; + break; + } + } + } + expect(removed).toBe(true); + const beforeCorruptLoad = record.journal.length; + yield* scoped(function* () { + const worker = yield* useWorker(scenario.route); + yield* worker.request("initialize", { protocolVersion: 1, clientCapabilities: {} }); + const loaded = yield* worker.request("session/load", { + sessionId: nativeId, + cwd: "/", + mcpServers: [], + }); + expect(loaded.error?.message).toContain("Divergence: workflow root terminated"); + expect(loaded.error?.message).toContain("unreached entry"); + expect(record.journal.length).toBe(beforeCorruptLoad); + }); + }); it("TW1: initialize, session/new, matched turns, mismatch, restart + session/load", function* () { const dir = path.join(os.tmpdir(), `xmd-tw-${randomUUID()}`); yield* ensureDir(dir); @@ -399,11 +574,17 @@ describe("Tier TW — worker lifecycle", { sanitizeOps: false, sanitizeResources const scenario = yield* controller.useScenario({ document: { path: "slow.md", source: behavior }, rootDir: dir, + models: MODELS, }); const worker = yield* useWorker(scenario.route); yield* worker.request("initialize", { protocolVersion: 1, clientCapabilities: {} }); const created = yield* worker.request("session/new", { cwd: "/", mcpServers: [] }); const sessionId = created.result?.sessionId; + yield* worker.request("session/set_config_option", { + sessionId, + configId: "effort", + value: "high", + }); const baseline = controller.getScenarioRecord(scenario.id)!.journal.length; const pending = yield* spawn(() => @@ -416,6 +597,9 @@ describe("Tier TW — worker lifecycle", { sanitizeOps: false, sanitizeResources worker.notify("session/cancel", { sessionId }); const cancelled = yield* pending; expect(cancelled.result).toMatchObject({ stopReason: "cancelled" }); + expect(controller.getScenarioRecord(scenario.id)?.models?.options[0].efforts?.selected).toBe( + "high", + ); // Nothing committed: the controller journal is exactly where it // was before the cancelled turn. expect(controller.getScenarioRecord(scenario.id)!.journal.length).toBe(baseline); diff --git a/packages/testing/mod.ts b/packages/testing/mod.ts index 0502b0a66..21307355c 100644 --- a/packages/testing/mod.ts +++ b/packages/testing/mod.ts @@ -86,6 +86,9 @@ export type { ChildDeclaration, ChildDeclarationChild, ChildScenario, + ScenarioModels, + ScenarioModel, + ScenarioEfforts, OpenChildDeclaration, TestAgentChildConfiguration, } from "./src/child-configuration.ts"; diff --git a/packages/testing/src/child-configuration.ts b/packages/testing/src/child-configuration.ts index 7aae374b8..5ab797249 100644 --- a/packages/testing/src/child-configuration.ts +++ b/packages/testing/src/child-configuration.ts @@ -57,8 +57,22 @@ import type { AnswerMatcher } from "@executablemd/core/host"; /** Everything one ``'s declarations configured, in declared order. */ export type ChildConfiguration = TestAgentChildConfiguration | AnswersChildConfiguration; +export interface ScenarioEfforts { + readonly selected: string; + readonly options: readonly string[]; +} +export interface ScenarioModel { + readonly id: string; + readonly efforts?: ScenarioEfforts | null; +} +export interface ScenarioModels { + readonly selected: string; + readonly options: readonly ScenarioModel[]; +} + /** One `` mapping, normalized where it was written. */ export interface ChildScenario { + readonly models?: ScenarioModels | null; /** The agent this mapping answers for, with the declaration's default applied. */ readonly agent: string; /** The logical session, or the empty string for the unnamed one. */ @@ -162,6 +176,34 @@ function detachScenario(scenario: ChildScenario): ChildScenario { session: scenario.session, rootDir: scenario.rootDir, document: frozen({ path: scenario.document.path, source: scenario.document.source }), + ...(scenario.models === undefined + ? {} + : { + models: + scenario.models === null + ? null + : frozen({ + selected: scenario.models.selected, + options: frozen( + scenario.models.options.map((model) => + frozen({ + id: model.id, + ...(model.efforts === undefined + ? {} + : { + efforts: + model.efforts === null + ? null + : frozen({ + selected: model.efforts.selected, + options: frozen([...model.efforts.options]), + }), + }), + }), + ), + ), + }), + }), }); } diff --git a/packages/testing/src/execution-harness.ts b/packages/testing/src/execution-harness.ts index 6ff0d614e..43dcfe8c4 100644 --- a/packages/testing/src/execution-harness.ts +++ b/packages/testing/src/execution-harness.ts @@ -859,6 +859,9 @@ function* installScan(state: HarnessState, declared: DeclarationState): Operatio // ends the projection instead of printing into a discarded buffer — and // so an enclosing `` never sees a failure that did not happen. if (state.phase === "scan") { + if (declared.inside !== undefined) { + declared.collect.refuse(segment.message); + } throw new ScanBoundary(segment.source ?? ""); } return yield* next(segment); diff --git a/packages/testing/tests/execution-harness.test.ts b/packages/testing/tests/execution-harness.test.ts index 1d6fd1884..fbffd884f 100644 --- a/packages/testing/tests/execution-harness.test.ts +++ b/packages/testing/tests/execution-harness.test.ts @@ -30,6 +30,7 @@ import type { ChildDeclaration, ChildDeclarationChild, } from "../src/child-configuration.ts"; +import { detachChildConfiguration } from "../src/child-configuration.ts"; import { useTesting } from "../src/use-testing.ts"; import { testHarnessInstallation } from "../src/execution-harness.ts"; import { ExecutionHost } from "../src/execution-host.ts"; @@ -995,6 +996,65 @@ describe("child configuration declarations", () => { return first; } + it("refuses a recognized declaration child's schema error before creating any child", function* () { + const run = yield* runDeclaring({ + "README.md": doc( + '', + '', + "", + "", + "", + "", + "", + ), + }); + expect(single(run).status).toBe("fail"); + expect(single(run).error?.message).toContain("must be string"); + expect(run.log.requests).toEqual([]); + }); + + it("deeply detaches and freezes scenario model and effort catalogs", function* () { + const effortOptions = ["low", "high"]; + const efforts = { selected: "low", options: effortOptions }; + const model = { id: "A", efforts }; + const options = [model]; + const models = { selected: "A", options }; + const detached = detachChildConfiguration([ + { + kind: "test-agent", + defaultAgent: "reviewer", + scenarios: [ + { + agent: "reviewer", + session: "review", + rootDir: "/fixture", + document: { path: "a.md", source: "hi" }, + models, + }, + ], + }, + ]); + models.selected = "changed"; + model.id = "changed"; + efforts.selected = "changed"; + effortOptions.push("changed"); + options.push({ id: "changed", efforts }); + const entry = detached[0]; + if (entry?.kind !== "test-agent") { + throw new Error("missing test-agent configuration"); + } + const captured = entry.scenarios[0].models; + expect(captured).toEqual({ + selected: "A", + options: [{ id: "A", efforts: { selected: "low", options: ["low", "high"] } }], + }); + expect(Object.isFrozen(captured)).toBe(true); + expect(Object.isFrozen(captured?.options)).toBe(true); + expect(Object.isFrozen(captured?.options[0])).toBe(true); + expect(Object.isFrozen(captured?.options[0].efforts)).toBe(true); + expect(Object.isFrozen(captured?.options[0].efforts?.options)).toBe(true); + }); + it("carries both declarations to the host as ordered, detached data", function* () { const run = yield* runDeclaring({ "README.md": doc( diff --git a/scripts/tests/ci-workflow.test.ts b/scripts/tests/ci-workflow.test.ts index d3bec6185..981f9fed2 100644 --- a/scripts/tests/ci-workflow.test.ts +++ b/scripts/tests/ci-workflow.test.ts @@ -26,6 +26,7 @@ interface Step { name?: string; run?: string; uses?: string; + with?: Record; } interface Strategy { @@ -80,6 +81,7 @@ function steps(value: unknown, label: string): Step[] { name: "name" in step ? string(step.name, `${label}[${index}].name`) : undefined, run: "run" in step ? string(step.run, `${label}[${index}].run`) : undefined, uses: "uses" in step ? string(step.uses, `${label}[${index}].uses`) : undefined, + with: "with" in step ? stringRecord(step.with, `${label}[${index}].with`) : undefined, }; }); } @@ -410,6 +412,23 @@ describe("the sharded runtime jobs", () => { } }); + it("provides the real Deno lock host before Bun tests", function* () { + const parsed = yield* workflow(); + const bun = matrixJob(parsed, "test-bun"); + const node = matrixJob(parsed, "test-node"); + const deno = bun.steps.findIndex((step) => step.uses?.startsWith("denoland/setup-deno@")); + const install = bun.steps.findIndex((step) => step.run === "bun install"); + const nodeDeno = node.steps.find((step) => step.uses?.startsWith("denoland/setup-deno@")); + expect(deno).toBeGreaterThanOrEqual(0); + expect(deno).toBeLessThan(install); + expect(bun.steps[deno]?.uses).toBe( + "denoland/setup-deno@e95548e56dfa95d4e1a28d6f422fafe75c4c26fb", + ); + expect(bun.steps[deno]?.uses).toBe(nodeDeno?.uses); + expect(bun.steps[deno]?.with?.["deno-version"]).toBe("v2.9.5"); + expect(bun.steps[deno]?.with).toEqual(nodeDeno?.with); + }); + it("keeps the Bun entrypoint smoke beside the Bun shards", function* () { const bun = matrixJob(yield* workflow(), "test-bun"); const commands = bun.steps.map((step) => step.run ?? "").join("\n"); diff --git a/specs/test-agent-spec.md b/specs/test-agent-spec.md index 4102f4874..d3fe9ffbf 100644 --- a/specs/test-agent-spec.md +++ b/specs/test-agent-spec.md @@ -95,6 +95,45 @@ The test runtime does not inject agent, session, cwd, or harness metadata into the behavior document. The document sees its own frontmatter and bindings, including values captured by prompt matchers. +### Model and effort choices + +Declare fictional configuration choices on the scenario that supplies the +session. The ordinary Session selects from these choices before its prompt: + +```md + + + + + + Review the supplied change. + + + + +``` + +Omitted or null `models` makes configuration choices unavailable and preserves +the response shape of an unconfigured worker. A present catalog contains a +nonempty ordered `options` array of unique, nonempty exact `id` strings and a +`selected` member. Each model's optional or null `efforts` makes effort choices +unavailable for that model; a present effort catalog has unique nonempty string +options and a selected member. IDs are also their displayed labels. Malformed +catalogs, empty choices and invalid initial membership refuse before scenario +provisioning in both declaration forms. + +Each scenario remembers its selected model and an effort selection for every +model offering efforts. Switching models exposes that model's choices and +remembered effort. Selecting the same model preserves its effort; a rejected +write changes nothing. These settings share the scenario's existing isolation +and teardown lifetime. A Session with no configuration props performs no +configuration read or write. Unknown or unavailable selections retain the +ordinary Session diagnostics and model-first restoration behavior. + ### Configuring one nested run Inside the declaration prefix of one canonical test's ``, @@ -217,6 +256,20 @@ same either way; only which layer reports an absent binding differs. Repeated uses of one capture name must match the same text. Adjacent capture holes without literal text between them are rejected as ambiguous. +`model` and `effort` on `` constrain the acknowledged configuration +at prompt entry, together with the complete text match: + +```md + +The requested configuration was consumed. +``` + +Omitting a constraint skips it; null asserts that the corresponding choices +are unavailable. Supplied strings are nonempty exact IDs. These props observe +provider state and never configure a Session. A configuration mismatch fails +the turn and owning test while leaving the matcher active and captures intact. + ## Scenarios A scenario declaration is a blueprint, not a singleton runtime. Each resolved @@ -317,6 +370,7 @@ interface TestAgentController { useScenario(options: { document: { path: string; source: string }; rootDir: string; + models?: ScenarioModels | null; }): Operation; } @@ -351,9 +405,19 @@ stdin and stdout. It supports the successful ACP lifecycle used by ACPX: - initialize; - create a session; - load or resume a session; +- set a configuration option and return the complete refreshed selectors; - prompt; - cancel. +Session creation and loading advertise ordinary ACP `configOptions` when a +catalog exists: `model` has category `model`, and the selected model's `effort` +has category `thought_level`. The worker waits for the controller's +acknowledgement of `session/set_config_option` before returning success and the +complete refreshed list. Its existing ordered response queue also carries these +acknowledgements. Reconnection reads the scenario's acknowledged settings rather +than resetting them. Cancellation restores behavior progress while preserving +successful configuration changes. + An agent availability check starts a disposable worker, completes ACP initialization, and closes it. Starting a session may start a separate worker. Successful validation therefore proves that the configured test agent is @@ -401,6 +465,11 @@ test scope teardown Behavior journals are private to the test-agent controller and separate from the document-under-test journal. Each isolated scenario has one journal containing the matched prompt, captures, and completed stage transitions. +Constrained stages also retain their detached actual configuration snapshot and +distinguish omitted constraints from null in their durable identity. A missing +required snapshot refuses replay. Rehydration uses the retained observation even +if current selections have changed. Unconstrained stages retain their previous +identity, record shape and captures. Restart recovery covers crashes between completed turns; a crash inside a turn's final delivery window is outside the supported guarantee. When a diff --git a/specs/testing-spec.md b/specs/testing-spec.md index cb9f53b68..4bc8ed2d0 100644 --- a/specs/testing-spec.md +++ b/specs/testing-spec.md @@ -391,6 +391,9 @@ without changing the document it executes: @@ -410,6 +413,10 @@ In this exact position, `` and `` are declarations rather than wrappers around assertion content. Each is a direct child of `` in its declaration prefix and may appear at most once. The TestAgent declaration contains one or more direct `` children and no other content. +Scenario `models` follows the same nullable catalog validation as the wrapper +form. Every catalog, model, effort catalog and options array is detached and +frozen before host middleware receives the request. Only data crosses the +boundary; configuration state is created inside the child. The Answers declaration contains one or more direct `` matchers and no other content. Their props, scenario documents, matcher templates, declaration order and validation keep the contracts in `test-agent-spec.md` and executable