Support model and effort configuration in TestAgent scenarios

.github/workflows/ci.yml

@@ -662,6 +662,10 @@ jobs:
         with:
           bun-version: 1.4.0
 
+      - uses: denoland/setup-deno@e95548e56dfa95d4e1a28d6f422fafe75c4c26fb # v2.0.3
+        with:
+          deno-version: v2.9.5
+
       - run: bun install
 
       - name: Test

packages/test-agent/README.md

@@ -97,6 +97,42 @@ the unnamed session, not every named session. Each test and each working
 directory receive independent scenario state, even when they use the same
 mapping. A test may finish with later scenario stages unused.
 
+## Exercise model and effort selection
+
+Declare fictional choices on `TestAgent.Scenario` and keep the workflow's
+ordinary Session configuration:
+
+```md
+<TestAgent agent="reviewer">
+  <TestAgent.Scenario session="review" src="./agents/review.md"
+    models={{selected:"small",options:[
+      {id:"small",efforts:{selected:"low",options:["low","high"]}},
+      {id:"review",efforts:{selected:"brief",options:["brief","deep"]}}
+    ]}} />
+  <Test name="uses review configuration">
+    <Agent name="reviewer"><Session name="review" model="review" effort="deep">
+      <Prompt>Review this change.</Prompt>
+    </Session></Agent>
+  </Test>
+</TestAgent>
+```
+
+In `agents/review.md`, observe what the prompt actually consumed:
+
+```md
+<WhenPrompt template="Review this change." model="review" effort="deep" />
+Reviewed with the requested configuration.
+```
+
+The IDs are exact strings supplied by the test. Each nonempty catalog has a
+selected member and unique options. A model remembers its own effort selection
+when the session switches away and back. Omitted or null catalogs represent
+unavailable choices; ordinary unconfigured Sessions still run. A missing
+WhenPrompt constraint skips that check, while null asserts unavailable choices.
+A mismatch leaves the stage active and fails the test. The same catalog syntax
+works in nested Execution declarations; separate Tests and children start from
+their own defaults. No real provider or credentials are needed.
+
 ## Use a failure to fix the workflow
 
 If the document sends this instead:

packages/test-agent/mod.ts

@@ -58,3 +58,5 @@ export { runTestAgentWorker } from "./src/worker/run.ts";
 export { installChildTestAgent, testAgentChildDeclaration } from "./src/child-configuration.ts";
 export type { PlanProviderAssembly, PlanProviderPolicy } from "./src/child-configuration.ts";
 export type { ChildTestAgentInstallation } from "./src/child-configuration.ts";
+
+export type { ScenarioModels, ScenarioModel, ScenarioEfforts } from "@executablemd/testing";

packages/test-agent/src/SessionConfiguration.defaults.md

@@ -0,0 +1,5 @@
+<WhenPrompt template="defaults" model="fixture-small" effort="low" />
+declared defaults
+
+<WhenPrompt template="changed" model="fixture-small" effort="high" />
+changed defaults

packages/test-agent/src/SessionConfiguration.refusal-behavior.md

@@ -0,0 +1,5 @@
+<WhenPrompt template="restored" model="A" effort="low" />
+restored
+
+<WhenPrompt template="still restored" model="A" effort="low" />
+still restored

packages/test-agent/src/SessionConfiguration.refusals.md

@@ -0,0 +1,27 @@
+<TestAgent agent="reviewer">
+<TestAgent.Scenario session="review" src="./SessionConfiguration.refusal-behavior.md" models={{selected:"A",options:[{id:"A",efforts:{selected:"low",options:["low","high"]}},{id:"B",efforts:{selected:"brief",options:["brief","deep"]}},{id:"none"}]}} />
+<TestAgent.Scenario session="unavailable" src="./SessionConfiguration.unavailable.md" />
+<Test name="invalid configuration starts no prompt and restores the original session" timeout="60s">
+<Agent name="reviewer">
+<ExpectRefusal message="Unknown model">
+<Session name="review" model="unknown"><Prompt>not started</Prompt></Session>
+</ExpectRefusal>
+<ExpectRefusal message="Invalid effort level">
+<Session name="review" model="B" effort="high"><Prompt>not started</Prompt></Session>
+</ExpectRefusal>
+<Session name="review"><Prompt>restored</Prompt></Session>
+<ExpectRefusal message="Effort choices are unavailable">
+<Session name="review" model="none" effort="high"><Prompt>not started</Prompt></Session>
+</ExpectRefusal>
+<Session name="review"><Prompt>still restored</Prompt></Session>
+</Agent>
+</Test>
+<Test name="unavailable configuration starts no prompt">
+<Agent name="reviewer">
+<ExpectRefusal message="Model choices are unavailable">
+<Session name="unavailable" model="A"><Prompt>not started</Prompt></Session>
+</ExpectRefusal>
+<Session name="unavailable"><Prompt>unavailable</Prompt></Session>
+</Agent>
+</Test>
+</TestAgent>

packages/test-agent/src/SessionConfiguration.review.md

@@ -0,0 +1,14 @@
+<WhenPrompt template="restored" model="fixture-small" effort="low" />
+small low
+<WhenPrompt template="configured" model="fixture-review" effort="deep" />
+review deep
+<WhenPrompt template="remember high" model="fixture-small" effort="high" />
+small high
+<WhenPrompt template="other model" model="fixture-review" effort="brief" />
+review brief
+<WhenPrompt template="stored effort" model="fixture-small" effort="high" />
+stored high
+<WhenPrompt template="same model" model="fixture-small" effort="high" />
+same high
+<WhenPrompt template="restored high" model="fixture-small" effort="high" />
+restored high

packages/test-agent/src/SessionConfiguration.test.md

@@ -0,0 +1,48 @@
+# Scenario model and effort selection
+
+<Let as="catalog" value={{selected:"fixture-small",options:[{id:"fixture-small",efforts:{selected:"low",options:["low","high"]}},{id:"fixture-review",efforts:{selected:"brief",options:["brief","deep"]}},{id:"no-effort"}]}} />
+<TestAgent agent="reviewer">
+<TestAgent.Scenario session="review" src="./SessionConfiguration.review.md" models={catalog} />
+<TestAgent.Scenario session="defaults" src="./SessionConfiguration.defaults.md" models={catalog} />
+<TestAgent.Scenario session="unavailable" src="./SessionConfiguration.unavailable.md" models={null} />
+
+<Test name="configuration and dependent effort memory share one session" timeout="60s">
+<Agent name="reviewer">
+<Session name="review"><Prompt as="restored">restored</Prompt></Session>
+<AssertStringIncludes actual={restored} expected="small low" />
+<Session name="review" model="fixture-review" effort="deep"><Prompt as="configured">configured</Prompt></Session>
+<AssertStringIncludes actual={configured} expected="review deep" />
+<Session name="review" model="fixture-small" effort="high"><Prompt>remember high</Prompt></Session>
+<Session name="review" model="fixture-review" effort="brief"><Prompt>other model</Prompt></Session>
+<Session name="review" model="fixture-small"><Prompt>stored effort</Prompt></Session>
+<Session name="review" model="fixture-small"><Prompt>same model</Prompt></Session>
+<Session name="review"><Prompt>restored high</Prompt></Session>
+</Agent>
+</Test>
+
+<Test name="catalog defaults are used without Session settings">
+<Agent name="reviewer">
+<Session name="defaults"><Prompt>defaults</Prompt></Session>
+<Session name="defaults" model="fixture-small" effort="high"><Prompt>changed</Prompt></Session>
+</Agent>
+</Test>
+<Test name="an independent Test starts at declared defaults">
+<Agent name="reviewer"><Session name="defaults"><Prompt>defaults</Prompt></Session></Agent>
+</Test>
+<Test name="unavailable choices allow an unconfigured Session">
+<Agent name="reviewer">
+<Session name="unavailable"><Prompt>unavailable</Prompt></Session>
+</Agent>
+</Test>
+</TestAgent>
+
+<Test name="sibling nested Executions have independent catalogs">
+<Execution host="run" source={'<Agent name="reviewer"><Session name="defaults"><Prompt>defaults</Prompt></Session><Session name="defaults" model="fixture-small" effort="high"><Prompt>changed</Prompt></Session></Agent>'} as="first">
+<TestAgent agent="reviewer"><TestAgent.Scenario session="defaults" src="./SessionConfiguration.defaults.md" models={catalog} /></TestAgent>
+<AssertEquals actual={first.result.ok} expected={true} />
+</Execution>
+<Execution host="run" source={'<Agent name="reviewer"><Session name="defaults"><Prompt>defaults</Prompt></Session></Agent>'} as="second">
+<TestAgent agent="reviewer"><TestAgent.Scenario session="defaults" src="./SessionConfiguration.defaults.md" models={catalog} /></TestAgent>
+<AssertEquals actual={second.result.ok} expected={true} />
+</Execution>
+</Test>

packages/test-agent/src/SessionConfiguration.unavailable.md

@@ -0,0 +1,2 @@
+<WhenPrompt template="unavailable" model={null} effort={null} />
+choices unavailable

packages/test-agent/src/child-configuration.ts

@@ -56,6 +56,8 @@ import {
 import type { ScenarioDeclaration } from "./components.ts";
 import { TEST_AGENT_PROVIDER } from "./provider.ts";
 
+import { normalizeModels } from "./configuration.ts";
+
 const TEST_AGENT = "TestAgent";
 const SCENARIO = "TestAgent.Scenario";
 
@@ -107,6 +109,13 @@ function open(collect: {
       refuse(`<${SCENARIO}> requires a "src" prop.`);
       return "";
     }
+    let models;
+    try {
+      models = normalizeModels(props.models);
+    } catch (error) {
+      refuse(`<${SCENARIO}> ${error instanceof Error ? error.message : String(error)}`);
+      return "";
+    }
     const read = yield* readScenarioSource(src);
     if (!read.ok) {
       refuse(`<${SCENARIO}> cannot read the behavior document "${src}": ${read.error.message}`);
@@ -128,6 +137,7 @@ function open(collect: {
       session,
       rootDir: read.value.rootDir,
       document: read.value.document,
+      ...(models === undefined ? {} : { models }),
     });
     return "";
   };
@@ -266,6 +276,7 @@ export function* installChildTestAgent(
       rootDir: scenario.rootDir,
       document: { path: scenario.document.path, source: scenario.document.source },
       duplicate: false,
+      models: normalizeModels(scenario.models),
     });
   }
   const partition = yield* provisionPartition({

packages/test-agent/src/components.ts

@@ -38,6 +38,8 @@ import type { ScenarioHandle, TestAgentControllerInternals } from "./controller.
 import { createControlledExecutableObserver } from "./executable-observer.ts";
 import { createDeterministicSessionCoordinator } from "./session-coordinator.ts";
 import type { AcpxProviderDependencies } from "@executablemd/acp";
+import { normalizeModels, MODELS_PROPS } from "./configuration.ts";
+import type { ScenarioModels } from "@executablemd/testing";
 import { duplicateScenario, missingScenario } from "./child-configuration.ts";
 import { TEST_AGENT_CLIENT_NATIVE, TEST_AGENT_PROVIDER, useTestAgentProvider } from "./provider.ts";
 import type { SessionRouting } from "./provider.ts";
@@ -47,6 +49,7 @@ export const DEFAULT_TEST_AGENT = "test";
 
 /** One `<TestAgent.Scenario>` mapping, before any worker exists. */
 export interface ScenarioDeclaration {
+  models?: ScenarioModels;
   agent: string;
   sessionName: string;
   rootDir: string;
@@ -196,6 +199,7 @@ export function* provisionPartition(options: {
       try {
         scenario = yield* controller.useScenario({
           document: declared.document,
+          models: declared.models,
           rootDir: declared.rootDir,
         });
       } catch (error) {
@@ -414,6 +418,7 @@ export function* Scenario(props: Record<string, Json>): Operation<unknown> {
     return reported.message;
   }
 
+  const models = normalizeModels(props.models);
   const read = yield* readScenarioSource(src);
   if (!read.ok) {
     // The read's own failure, unwrapped: a wrapper's scenario is read while a
@@ -436,6 +441,7 @@ export function* Scenario(props: Record<string, Json>): Operation<unknown> {
     rootDir: read.value.rootDir,
     document: read.value.document,
     duplicate: false,
+    ...(models === undefined ? {} : { models }),
   });
   return "";
 }
@@ -497,6 +503,7 @@ export const SCENARIO_PROPS: PropsSchema = {
   type: "object",
   properties: {
     src: { type: "string" },
+    models: MODELS_PROPS,
     agent: { type: "string" },
     session: { type: "string" },
   },