diff --git a/packages/evals/core/contracts/tool.ts b/packages/evals/core/contracts/tool.ts index f9d52bf31..565c499f3 100644 --- a/packages/evals/core/contracts/tool.ts +++ b/packages/evals/core/contracts/tool.ts @@ -1,3 +1,4 @@ +import type { ProbeEvidence } from "stagehand-v3"; import type { EvalLogger } from "../../logger.js"; import type { ActionTarget, FocusedTarget, TargetKind, WaitSpec } from "./targets.js"; import type { PageRepresentation, RepresentationOpts } from "./representation.js"; @@ -5,6 +6,7 @@ import type { Artifact, BrowserOwnership, ConnectionMode, EnvironmentName } from export type ToolSurface = | "understudy_code" + | "stagehand_code" | "playwright_code" | "cdp_code" | "playwright_mcp" @@ -134,6 +136,19 @@ export interface ToolStartInput { export interface ToolStartResult { session: CoreSession; + /** + * Optional agent-facing binding for this running surface. `via` describes + * delivery to the agent and is independent of `CoreTool.surface`; for + * example, a code surface may be delivered through MCP or a CLI wrapper. + */ + agentMount?: AgentMount; + /** + * Best-effort evidence captured from the current surface state. Harnesses may + * call this after individual actions and once more at the end of a run. + * Implementations must swallow per-field failures and must not throw. + */ + captureEvidence?: () => Promise; + /** Releases the runtime; `captureEvidence` is invalid after this resolves. */ cleanup: () => Promise; metadata: { environment: EnvironmentName; @@ -146,9 +161,59 @@ export interface ToolStartResult { export interface CoreTool { id: ToolSurface; surface: "code" | "mcp" | "cli"; - family: "understudy" | "playwright" | "cdp" | "stagehand_cli" | "chrome_devtools"; + family: "understudy" | "stagehand" | "playwright" | "cdp" | "stagehand_cli" | "chrome_devtools"; supportedStartupProfiles: StartupProfile[]; supportedCapabilities: CoreCapability[]; supportedTargetKinds: TargetKind[]; start(input: ToolStartInput): Promise; } + +/** + * The MCP server / tool name used when an agent harness wraps handles in a + * code-execution tool. + */ +export const AGENT_RUN_TOOL_SERVER = "stagehand_browser"; +export const AGENT_RUN_TOOL_NAME = `mcp__${AGENT_RUN_TOOL_SERVER}__run`; +export const AGENT_RUN_TOOL_RESERVED_HANDLES = ["startUrl", "task", "console"] as const; + +/** + * Surface-specific copy for the harness's code-execution tool. The harness + * owns mechanics and task bindings; the surface owns what the agent sees. + */ +export interface AgentRunToolSpec { + /** MCP tool description shown to the model. */ + description: string; + /** Description of the tool's `code` parameter. */ + codeParamDescription: string; + /** Message from the harness-owned tool allowlist when access is denied. */ + denyMessage: string; +} + +/** + * How an agent harness reaches an already-running surface. This is independent + * of `CoreTool.surface`; harnesses switch on `via` and need no surface-specific + * mounting logic. + */ +export type AgentMount = { promptInstructions: string } & ( + | { + via: "handles"; + /** + * Named values placed in snippet scope. Names, not order, bind values. + * `AGENT_RUN_TOOL_RESERVED_HANDLES` are injected by the harness and may + * not appear here. + */ + handles: Record; + runTool: AgentRunToolSpec; + } + | { via: "mcp"; mcpServers: Record } + | { + via: "cli"; + command: { + bin: string; + args?: string[]; + cwd?: string; + /** Extra variables merged over the harness environment. */ + env?: Record; + }; + } +); diff --git a/packages/evals/core/tools/cdp_code.ts b/packages/evals/core/tools/cdp_code.ts index c7d770965..e505e1877 100644 --- a/packages/evals/core/tools/cdp_code.ts +++ b/packages/evals/core/tools/cdp_code.ts @@ -1,13 +1,17 @@ -import type { - CoreCapability, - CoreLocatorHandle, - CorePageHandle, - CoreSession, - CoreTool, - StartupProfile, - ToolStartInput, - ToolStartResult, +import type { EvalLogger } from "../../logger.js"; +import type { ProbeEvidence } from "stagehand-v3"; +import { + AGENT_RUN_TOOL_NAME, + type CoreCapability, + type CoreLocatorHandle, + type CorePageHandle, + type CoreSession, + type CoreTool, + type StartupProfile, + type ToolStartInput, + type ToolStartResult, } from "../contracts/tool.js"; +import type { PageRepresentation } from "../contracts/representation.js"; import type { Artifact, ConnectionMode } from "../contracts/results.js"; import type { ActionTarget, TargetKind, WaitSpec } from "../contracts/targets.js"; import { loadWsModule } from "../runtime/coreDeps.js"; @@ -28,6 +32,7 @@ const SUPPORTED_CAPABILITIES: CoreCapability[] = [ "type", "press", "tabs", + "representation", ]; export type CdpEventMessage = { @@ -947,6 +952,27 @@ class CdpPageHandle implements CorePageHandle { this.state.sessionId, ); } + + async represent(): Promise { + await this.connection.send("Accessibility.enable", {}, this.state.sessionId); + const snapshot = await this.connection.send<{ nodes?: unknown[] }>( + "Accessibility.getFullAXTree", + {}, + this.state.sessionId, + ); + const nodes = snapshot.nodes ?? []; + const content = JSON.stringify(nodes, null, 2); + return { + kind: "accessibility_tree", + content, + metadata: { + bytes: Buffer.byteLength(content, "utf8"), + tokenEstimate: Math.ceil(content.length / 4), + nodeCount: nodes.length, + }, + raw: snapshot, + }; + } } class CdpSession implements CoreSession { @@ -970,10 +996,12 @@ class CdpSession implements CoreSession { } async listPages(): Promise { + await this.syncPages(); return [...this.pages.values()].map((state) => new CdpPageHandle(this.connection, state)); } async activePage(): Promise { + await this.syncPages(); if (this.activePageId) { const state = this.pages.get(this.activePageId); if (state) return new CdpPageHandle(this.connection, state); @@ -1039,23 +1067,39 @@ class CdpSession implements CoreSession { }; } + async createAgentRuntime(logger: EvalLogger): Promise { + await this.syncPages(); + const state = this.activePageId ? this.pages.get(this.activePageId) : undefined; + if (!state) throw new Error("No active page available"); + return buildCdpRuntime(this.connection, state, logger); + } + private async bootstrap(): Promise { - const targetInfos = await this.listPageTargets(); - if (targetInfos.length === 0) { + await this.syncPages(); + if (this.pages.size === 0) { const created = (await this.connection.send("Target.createTarget", { url: "about:blank", })) as { targetId: string }; await this.attachPage(created.targetId); - } else { - for (const targetInfo of targetInfos) { - await this.attachPage(targetInfo.targetId, targetInfo.url); - } } const firstPage = this.pages.keys().next().value as string | undefined; this.activePageId = firstPage ?? null; } + private async syncPages(): Promise { + const targetInfos = await this.listPageTargets(); + const currentIds = new Set(targetInfos.map((target) => target.targetId)); + for (const targetInfo of targetInfos) { + if (this.pages.has(targetInfo.targetId)) continue; + await this.attachPage(targetInfo.targetId, targetInfo.url); + if (this.activePageId !== null) this.activePageId = targetInfo.targetId; + } + for (const targetId of this.pages.keys()) { + if (!currentIds.has(targetId)) this.pages.delete(targetId); + } + } + private async listPageTargets(): Promise> { const response = (await this.connection.send("Target.getTargets")) as { targetInfos: Array<{ @@ -1124,6 +1168,29 @@ function connectionModeFromProfile( return "launch"; } +async function captureCdpEvidence(session: CoreSession): Promise { + const page = await session.activePage().catch((): undefined => undefined); + if (!page) return {}; + + const evidence: ProbeEvidence = {}; + try { + evidence.screenshot = await page.screenshot(); + } catch { + // Best effort: preserve other evidence modalities. + } + try { + evidence.url = page.url(); + } catch { + // Best effort: preserve other evidence modalities. + } + try { + evidence.ariaTree = (await page.represent?.())?.content; + } catch { + // Best effort: preserve other evidence modalities. + } + return evidence; +} + export class CdpCodeTool implements CoreTool { readonly id = "cdp_code"; readonly surface = "code"; @@ -1147,12 +1214,27 @@ export class CdpCodeTool implements CoreTool { const session = await CdpSession.connect({ providedEndpoint: input.providedEndpoint, }); + const cdp = await session.createAgentRuntime(input.logger); return { session, - cleanup: async () => { - await session.close(); + agentMount: { + via: "handles", + handles: { cdp }, + promptInstructions: buildCdpCodePromptInstructions(), + runTool: { + description: [ + "Execute JavaScript against the initialized Chrome DevTools Protocol browser.", + "The snippet runs inside an async function with cdp, startUrl, task, and console in scope.", + "Use await directly. Return a JSON-serializable value when useful.", + ].join(" "), + codeParamDescription: + "JavaScript function body to execute. cdp/startUrl/task are already in scope.", + denyMessage: `Use Bash for inspection and ${AGENT_RUN_TOOL_NAME} for CDP browser automation.`, + }, }, + captureEvidence: () => captureCdpEvidence(session), + cleanup: () => session.close(), metadata: { environment: input.environment === "BROWSERBASE" ? "browserbase" : "local", browserOwnership: input.startupProfile.startsWith("runner_provided") ? "runner" : "tool", @@ -1165,3 +1247,189 @@ export class CdpCodeTool implements CoreTool { }; } } + +type ActiveCdpPage = { + targetId: string; + sessionId: string; +}; + +type CdpRuntime = { + readonly targetId: string; + readonly sessionId: string; + send(method: string, params?: Record): Promise; + browser(method: string, params?: Record): Promise; + on(method: string, listener: (event: CdpEventMessage) => unknown | Promise): () => void; + off(method: string, listener: (event: CdpEventMessage) => unknown | Promise): void; + once( + method: string, + listenerOrTimeout?: ((event: CdpEventMessage) => unknown | Promise) | number, + timeoutMs?: number, + ): Promise | (() => void); + waitForEvent(method: string, timeoutMs?: number): Promise; + wait(ms: number): Promise; +}; + +function buildCdpCodePromptInstructions(): string { + return [ + "Browser tool surface: cdp_code.", + `Use the ${AGENT_RUN_TOOL_NAME} tool for browser automation. It exposes an initialized cdp object, startUrl, and task object.`, + "Use cdp.send(method, params) for page-scoped CDP commands and cdp.browser(method, params) for browser-level commands.", + "Helpers available: cdp.on(method, listener), cdp.once(method), cdp.waitForEvent(method, timeoutMs), cdp.wait(ms), cdp.targetId, cdp.sessionId.", + 'The first browser action should usually be: const loaded = cdp.waitForEvent("Page.loadEventFired"); await cdp.send("Page.navigate", { url: startUrl }); await loaded.', + "Use Bash for inspection and lightweight scripting. Do not create a separate browser process.", + "Do not edit repository files.", + "Return useful JSON-serializable values from run snippets so you can inspect progress.", + ].join("\n"); +} + +function buildCdpRuntime( + connection: CdpConnection, + activePage: ActiveCdpPage, + logger: EvalLogger, +): CdpRuntime { + const listenerUnsubscribes = new Map< + (event: CdpEventMessage) => unknown | Promise, + () => void + >(); + return { + targetId: activePage.targetId, + sessionId: activePage.sessionId, + send: (method: string, params?: Record): Promise => + connection.send(method, params, activePage.sessionId), + browser: (method: string, params?: Record): Promise => + connection.send(method, params), + on: ( + method: string, + listener: (event: CdpEventMessage) => unknown | Promise, + ): (() => void) => { + const unsubscribe = onCdpEvent(connection, activePage.sessionId, method, listener, logger); + listenerUnsubscribes.set(listener, unsubscribe); + return () => { + listenerUnsubscribes.delete(listener); + unsubscribe(); + }; + }, + off: ( + _method: string, + listener: (event: CdpEventMessage) => unknown | Promise, + ): void => { + const unsubscribe = listenerUnsubscribes.get(listener); + listenerUnsubscribes.delete(listener); + unsubscribe?.(); + }, + once: ( + method: string, + listenerOrTimeout?: ((event: CdpEventMessage) => unknown | Promise) | number, + timeoutMs = 15_000, + ): Promise | (() => void) => { + if (typeof listenerOrTimeout === "function") { + const listener = listenerOrTimeout; + const unsubscribe = onCdpEvent( + connection, + activePage.sessionId, + method, + (event) => { + unsubscribe?.(); + listenerUnsubscribes.delete(listener); + return listener(event); + }, + logger, + ); + listenerUnsubscribes.set(listener, unsubscribe); + return () => { + listenerUnsubscribes.delete(listener); + unsubscribe?.(); + }; + } + return waitForCdpEvent( + connection, + activePage.sessionId, + method, + listenerOrTimeout ?? timeoutMs, + ); + }, + waitForEvent: (method: string, timeoutMs = 15_000): Promise => + waitForCdpEvent(connection, activePage.sessionId, method, timeoutMs), + wait: sleep, + }; +} + +function onCdpEvent( + connection: CdpConnection, + sessionId: string, + method: string, + listener: (event: CdpEventMessage) => unknown | Promise, + logger: EvalLogger, +): () => void { + return connection.onEvent((event) => { + if (event.method !== method || (event.sessionId && event.sessionId !== sessionId)) { + return; + } + try { + const result = listener(event); + if (isPromiseLike(result)) { + result.catch((error: unknown) => { + logger.warn({ + category: "claude_code", + message: `cdp event listener failed: ${error instanceof Error ? error.message : String(error)}`, + level: 1, + }); + }); + } + } catch (error) { + logger.warn({ + category: "claude_code", + message: `cdp event listener failed: ${error instanceof Error ? error.message : String(error)}`, + level: 1, + }); + } + }); +} + +export function waitForCdpEvent( + connection: CdpConnection, + sessionId: string, + method: string, + timeoutMs: number, +): Promise { + let timeout: NodeJS.Timeout | undefined; + let unsubscribe: (() => void) | undefined; + const promise = new Promise((resolve, reject) => { + const cleanup = () => { + if (timeout) clearTimeout(timeout); + unsubscribe?.(); + }; + unsubscribe = connection.onEvent((event) => { + if (event.method !== method || (event.sessionId && event.sessionId !== sessionId)) { + return; + } + cleanup(); + resolve(event); + }); + timeout = setTimeout(() => { + cleanup(); + reject(new Error(`Timed out waiting for CDP event "${method}"`)); + }, timeoutMs); + timeout.unref(); + }); + + // Claude-generated snippets often assign an event wait promise before a CDP + // action and may abandon it after another branch finishes. Keep the promise + // rejectable for awaited callers, but prevent abandoned waits from crashing + // the eval process as unhandled rejections. + promise.catch((): undefined => undefined); + return promise; +} + +function isPromiseLike(value: unknown): value is PromiseLike & { + catch: (handler: (error: unknown) => void) => unknown; +} { + return ( + value !== null && + typeof value === "object" && + "then" in value && + typeof (value as { then?: unknown }).then === "function" && + "catch" in value && + typeof (value as { catch?: unknown }).catch === "function" + ); +} diff --git a/packages/evals/core/tools/playwright_code.ts b/packages/evals/core/tools/playwright_code.ts index a3984e50c..23f33d68a 100644 --- a/packages/evals/core/tools/playwright_code.ts +++ b/packages/evals/core/tools/playwright_code.ts @@ -1,14 +1,16 @@ import { chromium, type Browser, type BrowserContext, type Locator, type Page } from "playwright"; +import type { ProbeEvidence } from "stagehand-v3"; import { resolveLocalChromeExecutablePath } from "../targets/localChrome.js"; -import type { - CoreCapability, - CoreLocatorHandle, - CorePageHandle, - CoreSession, - CoreTool, - StartupProfile, - ToolStartInput, - ToolStartResult, +import { + AGENT_RUN_TOOL_NAME, + type CoreCapability, + type CoreLocatorHandle, + type CorePageHandle, + type CoreSession, + type CoreTool, + type StartupProfile, + type ToolStartInput, + type ToolStartResult, } from "../contracts/tool.js"; import type { PageRepresentation } from "../contracts/representation.js"; import type { Artifact, ConnectionMode } from "../contracts/results.js"; @@ -406,6 +408,9 @@ class PlaywrightSession implements CoreSession { const handle = this.wrap(initialPage); this.activePageId = handle.id; } + this.context.on("page", (page) => { + this.activePageId = this.wrap(page).id; + }); } private nextPageId(): string { @@ -513,6 +518,29 @@ function connectionModeFromProfile( return "launch"; } +async function capturePlaywrightEvidence(session: CoreSession): Promise { + const page = await session.activePage().catch((): undefined => undefined); + if (!page) return {}; + + const evidence: ProbeEvidence = {}; + try { + evidence.screenshot = await page.screenshot(); + } catch { + // Best effort: preserve other evidence modalities. + } + try { + evidence.url = page.url(); + } catch { + // Best effort: preserve other evidence modalities. + } + try { + evidence.ariaTree = (await page.represent?.({ includeIframes: true }))?.content; + } catch { + // Best effort: preserve other evidence modalities. + } + return evidence; +} + export class PlaywrightCodeTool implements CoreTool { readonly id = "playwright_code"; readonly surface = "code"; @@ -536,7 +564,7 @@ export class PlaywrightCodeTool implements CoreTool { async start(input: ToolStartInput): Promise { let browser: Browser; let context: BrowserContext; - let initialPage: Page | undefined; + let initialPage: Page; if (input.startupProfile === "tool_launch_local") { const executablePath = resolveLocalChromeExecutablePath(); @@ -575,9 +603,23 @@ export class PlaywrightCodeTool implements CoreTool { return { session, - cleanup: async () => { - await session.close(); + agentMount: { + via: "handles", + handles: { page: initialPage, context, browser }, + promptInstructions: buildPlaywrightCodePromptInstructions(), + runTool: { + description: [ + "Execute JavaScript against the initialized Playwright browser.", + "The snippet runs inside an async function with page, context, browser, startUrl, task, and console in scope.", + "Use await directly. Return a JSON-serializable value when useful.", + ].join(" "), + codeParamDescription: + "JavaScript function body to execute. page/context/browser/startUrl/task are already in scope.", + denyMessage: `Use Bash for inspection and ${AGENT_RUN_TOOL_NAME} for browser automation.`, + }, }, + captureEvidence: () => capturePlaywrightEvidence(session), + cleanup: () => session.close(), metadata: { environment: input.environment === "BROWSERBASE" ? "browserbase" : "local", browserOwnership: input.startupProfile.startsWith("runner_provided") ? "runner" : "tool", @@ -590,3 +632,14 @@ export class PlaywrightCodeTool implements CoreTool { }; } } + +function buildPlaywrightCodePromptInstructions(): string { + return [ + "Browser tool surface: playwright_code.", + `Use the ${AGENT_RUN_TOOL_NAME} tool for browser automation. It exposes an initialized Playwright page, context, browser, startUrl, and task object.`, + "Use Bash for inspection and lightweight scripting. Do not create a separate browser process.", + "The first browser action should usually be: await page.goto(startUrl, { waitUntil: 'domcontentloaded' }).", + "Do not edit repository files.", + "Return useful JSON-serializable values from run snippets so you can inspect progress.", + ].join("\n"); +} diff --git a/packages/evals/core/tools/registry.ts b/packages/evals/core/tools/registry.ts index 65384f137..7a14bce99 100644 --- a/packages/evals/core/tools/registry.ts +++ b/packages/evals/core/tools/registry.ts @@ -4,11 +4,13 @@ import { CdpCodeTool } from "./cdp_code.js"; import { ChromeDevtoolsMcpTool } from "./chrome_devtools_mcp.js"; import { PlaywrightCodeTool } from "./playwright_code.js"; import { PlaywrightMcpTool } from "./playwright_mcp.js"; +import { StagehandCodeTool } from "./stagehand_code.js"; import { UnderstudyCodeTool } from "./understudy_code.js"; export function listCoreTools(): ToolSurface[] { return [ "understudy_code", + "stagehand_code", "playwright_code", "cdp_code", "playwright_mcp", @@ -21,6 +23,8 @@ export function getCoreTool(toolSurface: ToolSurface): CoreTool { switch (toolSurface) { case "understudy_code": return new UnderstudyCodeTool(); + case "stagehand_code": + return new StagehandCodeTool(); case "playwright_code": return new PlaywrightCodeTool(); case "cdp_code": diff --git a/packages/evals/core/tools/stagehand_code.ts b/packages/evals/core/tools/stagehand_code.ts new file mode 100644 index 000000000..0f800307c --- /dev/null +++ b/packages/evals/core/tools/stagehand_code.ts @@ -0,0 +1,520 @@ +import type { Locator, Page, Stagehand } from "@browserbasehq/stagehand"; +import type { ProbeEvidence } from "stagehand-v3"; +import { z } from "zod/v4"; +import { initStagehand, type StagehandInitResult } from "../../initStagehand.js"; +import type { PageRepresentation } from "../contracts/representation.js"; +import type { Artifact, ConnectionMode } from "../contracts/results.js"; +import type { ActionTarget, TargetKind, WaitSpec } from "../contracts/targets.js"; +import { + AGENT_RUN_TOOL_NAME, + type CoreCapability, + type CoreLocatorHandle, + type CorePageHandle, + type CoreSession, + type CoreTool, + type StartupProfile, + type ToolStartInput, + type ToolStartResult, +} from "../contracts/tool.js"; + +const SURFACE_MODEL = "openai/gpt-4.1-mini"; + +const SUPPORTED_CAPABILITIES: CoreCapability[] = [ + "session", + "navigation", + "evaluation", + "screenshot", + "viewport", + "wait", + "click", + "hover", + "scroll", + "type", + "press", + "tabs", + "representation", +]; + +class StagehandLocatorHandle implements CoreLocatorHandle { + constructor(private readonly locatorHandle: Locator) {} + + async count(): Promise { + return this.locatorHandle.count(); + } + + async click(): Promise { + await this.locatorHandle.click(); + } + + async hover(): Promise { + await this.locatorHandle.hover(); + } + + async fill(value: string): Promise { + await this.locatorHandle.fill(value); + } + + async type(text: string, opts?: { delay?: number }): Promise { + await this.locatorHandle.type(text, opts); + } + + async isVisible(): Promise { + return this.locatorHandle.isVisible(); + } + + async textContent(): Promise { + return this.locatorHandle.textContent(); + } + + async inputValue(): Promise { + return this.locatorHandle.inputValue(); + } +} + +class StagehandPageHandle implements CorePageHandle { + readonly id: string; + private lastUrl: string; + + constructor(private readonly page: Page) { + this.id = page.pageId; + this.lastUrl = page.ref.url ?? "about:blank"; + } + + private async refreshUrl(): Promise { + this.lastUrl = await this.page.url(); + } + + async goto( + url: string, + opts?: { + waitUntil?: "load" | "domcontentloaded" | "networkidle"; + timeoutMs?: number; + }, + ): Promise { + await this.page.goto(url, { + waitUntil: opts?.waitUntil, + timeout: opts?.timeoutMs, + }); + await this.refreshUrl(); + } + + async reload(opts?: { + waitUntil?: "load" | "domcontentloaded" | "networkidle"; + timeoutMs?: number; + }): Promise { + await this.page.reload({ + waitUntil: opts?.waitUntil, + timeout: opts?.timeoutMs, + }); + await this.refreshUrl(); + } + + async back(opts?: { + waitUntil?: "load" | "domcontentloaded" | "networkidle"; + timeoutMs?: number; + }): Promise { + const response = await this.page.goBack({ + waitUntil: opts?.waitUntil, + timeout: opts?.timeoutMs, + }); + await this.refreshUrl(); + return response !== null; + } + + async goBack(opts?: { + waitUntil?: "load" | "domcontentloaded" | "networkidle"; + timeoutMs?: number; + }): Promise { + return this.back(opts); + } + + async forward(opts?: { + waitUntil?: "load" | "domcontentloaded" | "networkidle"; + timeoutMs?: number; + }): Promise { + const response = await this.page.goForward({ + waitUntil: opts?.waitUntil, + timeout: opts?.timeoutMs, + }); + await this.refreshUrl(); + return response !== null; + } + + async goForward(opts?: { + waitUntil?: "load" | "domcontentloaded" | "networkidle"; + timeoutMs?: number; + }): Promise { + return this.forward(opts); + } + + url(): string { + return this.lastUrl; + } + + async title(): Promise { + return this.page.title(); + } + + async evaluate( + pageFunctionOrExpression: string | ((arg: Arg) => R | Promise), + arg?: Arg, + ): Promise { + return this.page.evaluate(pageFunctionOrExpression, arg); + } + + async screenshot(opts?: { + fullPage?: boolean; + type?: "png" | "jpeg"; + quality?: number; + }): Promise { + return this.page.screenshot(opts); + } + + async setViewport(size: { width: number; height: number }): Promise { + await this.page.setViewportSize(size.width, size.height); + } + + async setViewportSize(width: number, height: number): Promise { + await this.page.setViewportSize(width, height); + } + + async wait(spec: WaitSpec): Promise { + switch (spec.kind) { + case "selector": + await this.page.waitForSelector(spec.selector, { + timeout: spec.timeoutMs, + state: spec.state, + }); + return; + case "timeout": + await this.page.waitForTimeout(spec.timeoutMs); + return; + case "load_state": + await this.page.waitForLoadState(spec.state, spec.timeoutMs); + return; + default: { + const exhaustive: never = spec; + throw new Error(`Unsupported wait spec: ${JSON.stringify(exhaustive)}`); + } + } + } + + async waitForSelector( + selector: string, + opts?: { + timeout?: number; + state?: "attached" | "detached" | "visible" | "hidden"; + }, + ): Promise { + return this.page.waitForSelector(selector, opts); + } + + async waitForTimeout(ms: number): Promise { + await this.page.waitForTimeout(ms); + } + + locator(selector: string): CoreLocatorHandle { + return new StagehandLocatorHandle(this.page.locator(selector)); + } + + async click(targetOrX: string | ActionTarget | number, y?: number): Promise { + if (typeof targetOrX === "number") { + if (typeof y !== "number") throw new Error("click(x, y) requires both numeric coordinates"); + await this.page.click(targetOrX, y); + return; + } + + const target = + typeof targetOrX === "string" ? ({ kind: "selector", value: targetOrX } as const) : targetOrX; + switch (target.kind) { + case "selector": + await this.page.locator(target.value).click(); + return; + case "coords": + await this.page.click(target.x, target.y); + return; + default: + throw new Error(`stagehand_code does not support click target kind "${target.kind}" yet`); + } + } + + async hover(targetOrX: string | ActionTarget | number, y?: number): Promise { + if (typeof targetOrX === "number") { + if (typeof y !== "number") throw new Error("hover(x, y) requires both numeric coordinates"); + await this.page.hover(targetOrX, y); + return; + } + + const target = + typeof targetOrX === "string" ? ({ kind: "selector", value: targetOrX } as const) : targetOrX; + switch (target.kind) { + case "selector": + await this.page.locator(target.value).hover(); + return; + case "coords": + await this.page.hover(target.x, target.y); + return; + default: + throw new Error(`stagehand_code does not support hover target kind "${target.kind}" yet`); + } + } + + async scroll(x: number, y: number, deltaX: number, deltaY: number): Promise { + await this.page.scroll(x, y, deltaX, deltaY); + } + + async type( + targetOrText: string | ActionTarget | { kind: "focused" }, + text?: string, + ): Promise { + if (typeof targetOrText === "string" && text === undefined) { + await this.page.type(targetOrText); + return; + } + if (typeof text !== "string") throw new Error("type(target, text) requires text"); + + const target = + typeof targetOrText === "string" + ? ({ kind: "selector", value: targetOrText } as const) + : targetOrText; + switch (target.kind) { + case "focused": + await this.page.type(text); + return; + case "selector": + await this.page.locator(target.value).type(text); + return; + case "coords": + await this.page.click(target.x, target.y); + await this.page.type(text); + return; + default: + throw new Error(`stagehand_code does not support type target kind "${target.kind}" yet`); + } + } + + async press( + targetOrKey: string | ActionTarget | { kind: "focused" }, + key?: string, + ): Promise { + if (typeof targetOrKey === "string" && key === undefined) { + await this.page.keyPress(targetOrKey); + return; + } + if (typeof key !== "string") throw new Error("press(target, key) requires key"); + + const target = + typeof targetOrKey === "string" + ? ({ kind: "selector", value: targetOrKey } as const) + : targetOrKey; + switch (target.kind) { + case "focused": + await this.page.keyPress(key); + return; + case "selector": + await this.page.locator(target.value).click(); + await this.page.keyPress(key); + return; + case "coords": + await this.page.click(target.x, target.y); + await this.page.keyPress(key); + return; + default: + throw new Error(`stagehand_code does not support press target kind "${target.kind}" yet`); + } + } + + async represent(opts?: { includeIframes?: boolean }): Promise { + const snapshot = await this.page.snapshot({ includeIframes: opts?.includeIframes }); + const content = snapshot.formattedTree; + return { + kind: "snapshot_refs", + content, + metadata: { + bytes: Buffer.byteLength(content, "utf8"), + tokenEstimate: Math.ceil(content.length / 4), + refCount: Object.keys(snapshot.xpathMap).length, + }, + raw: snapshot, + }; + } +} + +class StagehandCodeSession implements CoreSession { + private readonly handles = new Map(); + private closed = false; + + constructor(private readonly sdk: StagehandInitResult) {} + + private wrap(page: Page): StagehandPageHandle { + const existing = this.handles.get(page.pageId); + if (existing) return existing; + const handle = new StagehandPageHandle(page); + this.handles.set(page.pageId, handle); + return handle; + } + + async listPages(): Promise { + return (await this.sdk.stagehand.browser.context.pages()).map((page) => this.wrap(page)); + } + + async activePage(): Promise { + const page = await this.sdk.stagehand.browser.context.activePage(); + if (page) return this.wrap(page); + const pages = await this.sdk.stagehand.browser.context.pages(); + if (pages.length === 0) throw new Error("No active page available"); + return this.wrap(pages[0]); + } + + async newPage(url?: string): Promise { + return this.wrap(await this.sdk.stagehand.browser.context.newPage(url)); + } + + async selectPage(pageId: string): Promise { + const page = (await this.sdk.stagehand.browser.context.pages()).find( + (candidate) => candidate.pageId === pageId, + ); + if (!page) throw new Error(`Unknown page id "${pageId}"`); + await this.sdk.stagehand.browser.context.setActivePage(page); + } + + async closePage(pageId: string): Promise { + const page = (await this.sdk.stagehand.browser.context.pages()).find( + (candidate) => candidate.pageId === pageId, + ); + if (!page) throw new Error(`Unknown page id "${pageId}"`); + await page.close(); + this.handles.delete(pageId); + } + + async close(): Promise { + if (this.closed) return; + this.closed = true; + try { + await this.sdk.stagehand.close(); + } finally { + await this.sdk.stagehand.browser.close(); + } + } + + async getArtifacts(): Promise { + return []; + } + + async getRawMetrics(): Promise> { + return { + ...(await this.sdk.stagehand.metrics()), + browserProvider: this.sdk.stagehand.browser.provider, + browserOrigin: this.sdk.stagehand.browser.origin, + }; + } +} + +async function captureStagehandEvidence(stagehand: Stagehand): Promise { + const page = await stagehand.browser.context.activePage().catch((): undefined => undefined); + if (!page) return {}; + + const evidence: ProbeEvidence = {}; + try { + evidence.screenshot = await page.screenshot(); + } catch { + // Best effort: preserve other evidence modalities. + } + try { + evidence.url = await page.url(); + } catch { + // Best effort: preserve other evidence modalities. + } + try { + evidence.ariaTree = (await page.snapshot({ includeIframes: true })).formattedTree; + } catch { + // Best effort: preserve other evidence modalities. + } + return evidence; +} + +function connectionModeFromProfile(startupProfile: StartupProfile): ConnectionMode { + return startupProfile === "tool_create_browserbase" ? "browserbase_native" : "launch"; +} + +export class StagehandCodeTool implements CoreTool { + readonly id = "stagehand_code"; + readonly surface = "code"; + readonly family = "stagehand"; + readonly supportedStartupProfiles: StartupProfile[] = [ + "tool_launch_local", + "tool_create_browserbase", + ]; + readonly supportedCapabilities: CoreCapability[] = [...SUPPORTED_CAPABILITIES]; + readonly supportedTargetKinds: TargetKind[] = ["selector", "coords", "focused"]; + + async start(input: ToolStartInput): Promise { + if (!this.supportedStartupProfiles.includes(input.startupProfile)) { + throw new Error( + `stagehand_code does not support startup profile "${input.startupProfile}" yet`, + ); + } + + const sdk = await initStagehand({ + logger: input.logger, + modelName: SURFACE_MODEL, + environment: input.startupProfile === "tool_create_browserbase" ? "BROWSERBASE" : "LOCAL", + }); + const session = new StagehandCodeSession(sdk); + + input.logger.log({ + category: "stagehand_code", + message: "Initialized stagehand_code Stagehand SDK runtime.", + level: 1, + auxiliary: { + startupProfile: { value: input.startupProfile, type: "string" }, + environment: { value: input.environment, type: "string" }, + }, + }); + + return { + session, + agentMount: { + via: "handles", + handles: { stagehand: sdk.stagehand, page: sdk.page, z }, + promptInstructions: buildStagehandCodePromptInstructions(), + runTool: { + description: [ + "Execute JavaScript against the initialized Stagehand SDK.", + "The snippet runs inside an async function with stagehand, page, startUrl, task, z (zod), and console in scope.", + "Use await directly. Return a JSON-serializable value when useful.", + ].join(" "), + codeParamDescription: + "JavaScript function body to execute. stagehand/page/startUrl/task/z are already in scope.", + denyMessage: `Use Bash for inspection and ${AGENT_RUN_TOOL_NAME} for browser automation.`, + }, + }, + captureEvidence: () => captureStagehandEvidence(sdk.stagehand), + cleanup: () => session.close(), + metadata: { + environment: input.environment === "BROWSERBASE" ? "browserbase" : "local", + browserOwnership: "tool", + connectionMode: connectionModeFromProfile(input.startupProfile), + startupProfile: input.startupProfile, + }, + }; + } +} + +export function buildStagehandCodePromptInstructions(): string { + return [ + "Browser tool surface: stagehand_code (Stagehand SDK).", + `Use the ${AGENT_RUN_TOOL_NAME} tool for browser automation. It exposes an initialized Stagehand client (stagehand), its initial page, startUrl, and task object.`, + "AI methods live on the client: await stagehand.act('instruction'), await stagehand.observe('instruction'), await stagehand.extract('instruction', zodSchema) — a zod `z` is in scope for extract schemas (use single-word keys).", + "The page implements exactly these methods:", + " page: goto(url, opts), reload(), goBack()/goForward(), url(), title(), evaluate(fn, arg), screenshot(opts), setViewportSize(w,h), waitForSelector(sel, opts), waitForTimeout(ms), click(x,y), hover(x,y), scroll(x,y,dx,dy), type(text), keyPress(key).", + " page.locator(selector): count(), click(), hover(), fill(value), type(text), isVisible(), textContent(), inputValue().", + "For behavior not listed above, use await stagehand.act('describe the action').", + "Page accessors are async RPCs — always await them.", + "The first browser action should usually be: await page.goto(startUrl, { waitUntil: 'domcontentloaded' }).", + "Prefer batching complete workflows into each run call: chain multiple act/observe/extract and page steps in one snippet and return the final result.", + "Use Bash for inspection and lightweight scripting. Do not create a separate browser process.", + "Do not edit repository files.", + "Return useful JSON-serializable values from run snippets so you can inspect progress.", + ].join("\n"); +} diff --git a/packages/evals/framework/agentToolRuntime.ts b/packages/evals/framework/agentToolRuntime.ts new file mode 100644 index 000000000..b5d41f4e0 --- /dev/null +++ b/packages/evals/framework/agentToolRuntime.ts @@ -0,0 +1,63 @@ +import type { StartupProfile, ToolStartResult, ToolSurface } from "../core/contracts/tool.js"; +import { prepareCoreBrowserTarget } from "../core/targets/index.js"; +import { getCoreTool } from "../core/tools/registry.js"; +import { EvalsError } from "../errors.js"; +import type { EvalLogger } from "../logger.js"; + +export interface AgentToolRuntimeInput { + toolSurface: ToolSurface; + startupProfile: StartupProfile; + environment: "LOCAL" | "BROWSERBASE"; + logger: EvalLogger; +} + +export interface StartedAgentToolRuntime { + running: ToolStartResult; + /** Closes the tool-owned runtime, then the runner-owned browser target. */ + cleanup: () => Promise; +} + +/** + * Starts a CoreTool for an external agent harness without interpreting its + * agent mount. Harness adapters own delivery; this function owns the shared + * tool/target lifecycle and makes cleanup idempotent. + */ +export async function startAgentToolRuntime( + input: AgentToolRuntimeInput, +): Promise { + const tool = getCoreTool(input.toolSurface); + if (!tool.supportedStartupProfiles.includes(input.startupProfile)) { + throw new EvalsError( + `Tool surface "${input.toolSurface}" does not support startup profile "${input.startupProfile}".`, + ); + } + + const target = await prepareCoreBrowserTarget(input); + let running: ToolStartResult; + try { + running = await tool.start({ + logger: input.logger, + environment: input.environment, + startupProfile: input.startupProfile, + providedEndpoint: target.providedEndpoint, + }); + } catch (error) { + await target.cleanup().catch((): undefined => undefined); + throw error; + } + + let cleanupPromise: Promise | undefined; + return { + running, + cleanup: async () => { + cleanupPromise ??= (async () => { + try { + await running.cleanup(); + } finally { + await target.cleanup(); + } + })(); + await cleanupPromise; + }, + }; +} diff --git a/packages/evals/framework/claudeCodeRunner.ts b/packages/evals/framework/claudeCodeRunner.ts index 471730d68..304455f36 100644 --- a/packages/evals/framework/claudeCodeRunner.ts +++ b/packages/evals/framework/claudeCodeRunner.ts @@ -266,6 +266,13 @@ export async function runClaudeCodeAgent({ return baseResult; } + // Artifact-grounded grading: capture the terminal page state through the + // tool surface (harness-observed, independent of the agent's self-report) + // before cleanup, and drain the per-step probe observations collected by + // the run tool. + const finalObservation = await toolAdapter?.captureEvidence?.().catch((): undefined => undefined); + const stepObservations = toolAdapter?.drainStepObservations?.(); + // Build a Trajectory from the SDK message stream and grade it with the // rubric verifier; any failure in that path folds into `verifierError`. return gradeExternalTrajectory({ @@ -273,6 +280,8 @@ export async function runClaudeCodeAgent({ claudeCodeAdapter.fromHarnessResult( { messages, + ...(finalObservation && { finalObservation }), + ...(stepObservations?.length && { stepObservations }), finalAnswer: parsed.finalAnswer ?? resultText, status: status === "completed" ? "complete" : "error", usage: { diff --git a/packages/evals/framework/claudeCodeToolAdapter.ts b/packages/evals/framework/claudeCodeToolAdapter.ts index 886ab7ec6..f1f346269 100644 --- a/packages/evals/framework/claudeCodeToolAdapter.ts +++ b/packages/evals/framework/claudeCodeToolAdapter.ts @@ -3,7 +3,6 @@ import fsp from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import matter from "gray-matter"; -import type { Browser, BrowserContext, Page } from "playwright"; import { z } from "zod/v4"; import { EvalsError } from "../errors.js"; import type { EvalLogger } from "../logger.js"; @@ -13,10 +12,19 @@ import { BROWSE_CLI_PACKAGE_JSON, BROWSE_SKILL_SOURCE, } from "../browseCliPaths.js"; -import type { StartupProfile, ToolSurface } from "../core/contracts/tool.js"; -import { prepareCoreBrowserTarget } from "../core/targets/index.js"; -import { CdpConnection, type CdpEventMessage } from "../core/tools/cdp_code.js"; +import { + AGENT_RUN_TOOL_NAME, + AGENT_RUN_TOOL_SERVER, + type AgentRunToolSpec, + type StartupProfile, + type ToolSurface, +} from "../core/contracts/tool.js"; +import type { ProbeEvidence } from "stagehand-v3"; +import { startAgentToolRuntime } from "./agentToolRuntime.js"; import type { ExternalHarnessTaskPlan } from "./externalHarnessPlan.js"; +import { ObservationRecorder, type StepObservation } from "./observationRecorder.js"; + +export { waitForCdpEvent } from "../core/tools/cdp_code.js"; export interface ClaudeCodeToolAdapterInput { toolSurface?: ToolSurface; @@ -39,6 +47,9 @@ export interface PreparedClaudeCodeToolAdapter { toolName: string, input: Record, ) => Promise>; + /** Best-effort evidence from the currently running tool surface. */ + captureEvidence?: () => Promise; + drainStepObservations?: () => StepObservation[]; cleanup: () => Promise; } @@ -93,8 +104,8 @@ the guidance below: requested by the harness prompt. `; const ALLOW_UNSANDBOXED_LOCAL_ENV = "EVAL_CLAUDE_CODE_ALLOW_UNSANDBOXED_LOCAL"; -const RUN_TOOL_SERVER = "stagehand_browser"; -const RUN_TOOL_NAME = `mcp__${RUN_TOOL_SERVER}__run`; +const RUN_TOOL_SERVER = AGENT_RUN_TOOL_SERVER; +const RUN_TOOL_NAME = AGENT_RUN_TOOL_NAME; type ClaudeToolResult = { content: Array<{ type: "text"; text: string }>; @@ -116,28 +127,6 @@ type SdkMcpServerFactory = (options: { alwaysLoad?: boolean; }) => unknown; -type ActiveCdpPage = { - targetId: string; - sessionId: string; - url: string; -}; - -type CdpRuntime = { - readonly targetId: string; - readonly sessionId: string; - send(method: string, params?: Record): Promise; - browser(method: string, params?: Record): Promise; - on(method: string, listener: (event: CdpEventMessage) => unknown | Promise): () => void; - off(method: string, listener: (event: CdpEventMessage) => unknown | Promise): void; - once( - method: string, - listenerOrTimeout?: ((event: CdpEventMessage) => unknown | Promise) | number, - timeoutMs?: number, - ): Promise | (() => void); - waitForEvent(method: string, timeoutMs?: number): Promise; - wait(ms: number): Promise; -}; - export interface BrowseCliToolMetadata { toolCommand: "browse"; browseCliEntrypoint: string; @@ -178,31 +167,33 @@ export async function prepareClaudeCodeToolAdapter( startupProfile, }); case "playwright_code": - return preparePlaywrightCodeAdapter({ - ...input, - toolSurface, - startupProfile, - }); case "cdp_code": - return prepareCdpCodeAdapter({ + case "stagehand_code": { + return prepareMountedCoreToolAdapter({ ...input, toolSurface, startupProfile, }); + } default: throw new EvalsError( - `Claude Code harness supports --tool browse_cli, playwright_code, or cdp_code for execution right now; received "${toolSurface}".`, + `Claude Code harness supports --tool browse_cli, playwright_code, cdp_code, or stagehand_code for execution right now; received "${toolSurface}".`, ); } } export function resolveClaudeCodeToolSurface(requested?: ToolSurface): ToolSurface { if (!requested) return "browse_cli"; - if (requested === "browse_cli" || requested === "playwright_code" || requested === "cdp_code") { + if ( + requested === "browse_cli" || + requested === "playwright_code" || + requested === "cdp_code" || + requested === "stagehand_code" + ) { return requested; } throw new EvalsError( - `Claude Code harness supports --tool browse_cli, playwright_code, or cdp_code for execution right now; received "${requested}".`, + `Claude Code harness supports --tool browse_cli, playwright_code, cdp_code, or stagehand_code for execution right now; received "${requested}".`, ); } @@ -213,7 +204,9 @@ export function resolveClaudeCodeStartupProfile( ): StartupProfile { if (requested) return requested; - if (toolSurface === "browse_cli") { + // browse_cli and stagehand_code own their browser (the Stagehand SDK launches or + // creates it via the extension stack), so no runner-provided CDP endpoint. + if (toolSurface === "browse_cli" || toolSurface === "stagehand_code") { return environment === "BROWSERBASE" ? "tool_create_browserbase" : "tool_launch_local"; } if (toolSurface === "playwright_code" || toolSurface === "cdp_code") { @@ -283,7 +276,7 @@ export async function prepareBrowseCliHarnessAdapter( const missingArtifact = BROWSE_CLI_BUILD_ARTIFACTS.find((artifact) => !fs.existsSync(artifact)); if (missingArtifact) { throw new EvalsError( - `browse_cli dependency is incomplete; missing ${missingArtifact}. Reinstall workspace dependencies.`, + `browse_cli requires built CLI artifacts; missing ${missingArtifact}. Run pnpm --dir packages/cli build first.`, ); } @@ -349,190 +342,70 @@ export async function prepareBrowseCliHarnessAdapter( }; } -async function preparePlaywrightCodeAdapter( +/** + * Starts a CoreTool once and mounts the returned agent binding. The harness + * switches only on the binding modality, never on the tool surface identity. + */ +async function prepareMountedCoreToolAdapter( input: ClaudeCodeToolAdapterInput & { - toolSurface: "playwright_code"; + toolSurface: ToolSurface; startupProfile: StartupProfile; }, ): Promise { - if ( - input.startupProfile !== "runner_provided_local_cdp" && - input.startupProfile !== "runner_provided_browserbase_cdp" - ) { - throw new EvalsError( - `playwright_code startup profile "${input.startupProfile}" is not valid for Claude Code. Use runner_provided_local_cdp or runner_provided_browserbase_cdp.`, - ); - } - - const cwd = await fsp.mkdtemp(path.join(os.tmpdir(), "stagehand-evals-claude-playwright-")); - const env = { ...process.env } as Record; - let browser: Browser | undefined; - let targetCleanup: () => Promise = async () => {}; - + const runtime = await startAgentToolRuntime(input); try { - const target = await prepareCoreBrowserTarget({ - environment: input.environment, - toolSurface: "playwright_code", - startupProfile: input.startupProfile, - }); - targetCleanup = target.cleanup; - if (!target.providedEndpoint?.url) { - throw new EvalsError( - `playwright_code requires a runner-provided CDP endpoint for startup profile "${input.startupProfile}".`, - ); - } - - const { chromium } = await import("playwright"); - browser = await chromium.connectOverCDP(target.providedEndpoint.url, { - headers: target.providedEndpoint.headers, - }); - const context = browser.contexts()[0] ?? (await browser.newContext()); - const page = context.pages()[0] ?? (await context.newPage()); - const mcpServers = await buildPlaywrightRunMcpServers({ - browser, - context, - page, - plan: input.plan, - logger: input.logger, - }); - - input.logger.log({ - category: "claude_code", - message: `Initialized playwright_code browser runtime for Claude Code run tool.`, - level: 1, - auxiliary: { - startupProfile: { - value: input.startupProfile, - type: "string", - }, - environment: { - value: input.environment, - type: "string", - }, - ...(target.metadata && { - targetMetadata: { - value: JSON.stringify(target.metadata), - type: "object", - }, - }), - }, - }); - - return { - toolSurface: "playwright_code", - startupProfile: input.startupProfile, - cwd, - env, - allowedTools: ["Bash", RUN_TOOL_NAME], - settingSources: [], - mcpServers, - canUseTool: async (toolName, commandInput) => { - if (toolName === RUN_TOOL_NAME || toolName === "Bash") { - return { behavior: "allow", updatedInput: commandInput }; - } - return { - behavior: "deny", - message: `Use Bash for inspection and ${RUN_TOOL_NAME} for browser automation.`, - }; - }, - promptInstructions: buildPlaywrightCodePromptInstructions(input.plan), - cleanup: async () => { - try { - await browser?.close(); - } catch { - // best-effort only - } finally { - await targetCleanup(); - await fsp.rm(cwd, { recursive: true, force: true }); - } - }, - }; + return await prepareAgentMountAdapter(runtime.running, runtime.cleanup, input); } catch (error) { - try { - await browser?.close(); - } catch { - // best-effort only - } - await targetCleanup(); - await fsp.rm(cwd, { recursive: true, force: true }); + await runtime.cleanup().catch((): undefined => undefined); throw error; } } -async function prepareCdpCodeAdapter( +/** + * The generic mount point for handle bindings: wraps the binding's handles in + * the harness's MCP "run" tool, whose executor runs + * snippet code in an AsyncFunction scope over the handle names plus + * startUrl, task, and console. Surface specifics (handles, prompt + * instructions, run-tool copy, snippet task/console bindings, cleanup) all + * come from the mount — this function owns only harness mechanics. + */ +async function prepareAgentMountAdapter( + running: Awaited>["running"], + cleanupRuntime: () => Promise, input: ClaudeCodeToolAdapterInput & { - toolSurface: "cdp_code"; + toolSurface: ToolSurface; startupProfile: StartupProfile; }, ): Promise { - if ( - input.startupProfile !== "runner_provided_local_cdp" && - input.startupProfile !== "runner_provided_browserbase_cdp" - ) { - throw new EvalsError( - `cdp_code startup profile "${input.startupProfile}" is not valid for Claude Code. Use runner_provided_local_cdp or runner_provided_browserbase_cdp.`, - ); - } - - const cwd = await fsp.mkdtemp(path.join(os.tmpdir(), "stagehand-evals-claude-cdp-")); - const env = { ...process.env } as Record; - let connection: CdpConnection | undefined; - let targetCleanup: () => Promise = async () => {}; - + let cwd: string | undefined; try { - const target = await prepareCoreBrowserTarget({ - environment: input.environment, - toolSurface: "cdp_code", - startupProfile: input.startupProfile, - }); - targetCleanup = target.cleanup; - if (!target.providedEndpoint?.url) { + const mount = running.agentMount; + if (!mount) { + throw new EvalsError(`Tool surface "${input.toolSurface}" does not provide an agent mount.`); + } + if (mount.via !== "handles") { throw new EvalsError( - `cdp_code requires a runner-provided CDP endpoint for startup profile "${input.startupProfile}".`, + `Claude Code does not support agent mounts delivered via "${mount.via}" yet.`, ); } - connection = await CdpConnection.connect(target.providedEndpoint); - const activePage = await attachActiveCdpPage(connection); - const mcpServers = await buildCdpRunMcpServers({ - connection, - activePage, + cwd = await fsp.mkdtemp(path.join(os.tmpdir(), `stagehand-evals-claude-${input.toolSurface}-`)); + const cleanupCwd = cwd; + const env = { ...process.env } as Record; + const recorder = running.captureEvidence + ? new ObservationRecorder(running.captureEvidence) + : undefined; + const mcpServers = await buildCodeExposureRunMcpServers({ + handles: mount.handles, + runToolSpec: mount.runTool, plan: input.plan, logger: input.logger, + recordObservation: recorder ? () => recorder.record() : undefined, }); - - input.logger.log({ - category: "claude_code", - message: `Initialized cdp_code browser runtime for Claude Code run tool.`, - level: 1, - auxiliary: { - startupProfile: { - value: input.startupProfile, - type: "string", - }, - environment: { - value: input.environment, - type: "string", - }, - targetId: { - value: activePage.targetId, - type: "string", - }, - sessionId: { - value: activePage.sessionId, - type: "string", - }, - ...(target.metadata && { - targetMetadata: { - value: JSON.stringify(target.metadata), - type: "object", - }, - }), - }, - }); + let cleanupPromise: Promise | undefined; return { - toolSurface: "cdp_code", + toolSurface: input.toolSurface, startupProfile: input.startupProfile, cwd, env, @@ -545,39 +418,53 @@ async function prepareCdpCodeAdapter( } return { behavior: "deny", - message: `Use Bash for inspection and ${RUN_TOOL_NAME} for CDP browser automation.`, + message: mount.runTool.denyMessage, }; }, - promptInstructions: buildCdpCodePromptInstructions(input.plan), + promptInstructions: mount.promptInstructions, + ...(running.captureEvidence && { + captureEvidence: async (): Promise => { + try { + return await withTimeout( + running.captureEvidence!(), + readPositiveIntEnv("EVAL_CAPTURE_EVIDENCE_TIMEOUT_MS", 15_000), + ); + } catch { + return {}; + } + }, + }), + ...(recorder && { drainStepObservations: () => recorder.drain() }), cleanup: async () => { - try { - await connection?.close(); - } catch { - // best-effort only - } finally { - await targetCleanup(); - await fsp.rm(cwd, { recursive: true, force: true }); - } + cleanupPromise ??= (async () => { + try { + await withTimeout( + cleanupRuntime(), + readPositiveIntEnv("EVAL_AGENT_MOUNT_CLEANUP_TIMEOUT_MS", 30_000), + ); + } catch { + // Cleanup is best-effort, but temp-dir cleanup must run. + } finally { + await fsp.rm(cleanupCwd, { recursive: true, force: true }); + } + })(); + await cleanupPromise; }, }; } catch (error) { - try { - await connection?.close(); - } catch { - // best-effort only + if (cwd) { + await fsp.rm(cwd, { recursive: true, force: true }); } - await targetCleanup(); - await fsp.rm(cwd, { recursive: true, force: true }); throw error; } } -async function buildPlaywrightRunMcpServers(input: { - browser: Browser; - context: BrowserContext; - page: Page; +async function buildCodeExposureRunMcpServers(input: { + handles: Record; + runToolSpec: AgentRunToolSpec; plan: ExternalHarnessTaskPlan; logger: EvalLogger; + recordObservation?: () => Promise; }): Promise> { const sdk = (await import("@anthropic-ai/claude-agent-sdk")) as unknown as { createSdkMcpServer: SdkMcpServerFactory; @@ -586,27 +473,20 @@ async function buildPlaywrightRunMcpServers(input: { const runTool = sdk.tool( "run", - [ - "Execute JavaScript against the initialized Playwright browser.", - "The snippet runs inside an async function with page, context, browser, startUrl, task, and console in scope.", - "Use await directly. Return a JSON-serializable value when useful.", - ].join(" "), + input.runToolSpec.description, { - code: z - .string() - .describe( - "JavaScript function body to execute. page/context/browser/startUrl/task are already in scope.", - ), + code: z.string().describe(input.runToolSpec.codeParamDescription), }, async ({ code }) => { - return executePlaywrightRunTool({ + const result = await executeCodeExposureRunTool({ code, - browser: input.browser, - context: input.context, - page: input.page, + handles: input.handles, + runToolSpec: input.runToolSpec, plan: input.plan, logger: input.logger, }); + await input.recordObservation?.(); + return result; }, { alwaysLoad: true }, ); @@ -621,17 +501,16 @@ async function buildPlaywrightRunMcpServers(input: { }; } -async function executePlaywrightRunTool(input: { +async function executeCodeExposureRunTool(input: { code: string; - browser: Browser; - context: BrowserContext; - page: Page; + handles: Record; + runToolSpec: AgentRunToolSpec; plan: ExternalHarnessTaskPlan; logger: EvalLogger; }): Promise { try { const result = await withTimeout( - executePlaywrightSnippet(input), + executeCodeExposureSnippet(input), readPositiveIntEnv("EVAL_CLAUDE_CODE_RUN_TOOL_TIMEOUT_MS", 60_000), ); const text = stringifyToolResult(result); @@ -657,30 +536,28 @@ async function executePlaywrightRunTool(input: { } } -async function executePlaywrightSnippet(input: { +async function executeCodeExposureSnippet(input: { code: string; - browser: Browser; - context: BrowserContext; - page: Page; + handles: Record; + runToolSpec: AgentRunToolSpec; plan: ExternalHarnessTaskPlan; logger: EvalLogger; }): Promise { const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor as new ( ...args: string[] ) => (...values: unknown[]) => Promise; + // Snippet scope = the exposure's handle names plus startUrl/task/console. + // Object.keys/Object.values over the same object are guaranteed to align, + // so names — not positions — bind the values. const fn = new AsyncFunction( - "page", - "context", - "browser", + ...Object.keys(input.handles), "startUrl", "task", "console", input.code, ); return fn( - input.page, - input.context, - input.browser, + ...Object.values(input.handles), input.plan.startUrl, { dataset: input.plan.dataset, @@ -692,284 +569,6 @@ async function executePlaywrightSnippet(input: { ); } -async function buildCdpRunMcpServers(input: { - connection: CdpConnection; - activePage: ActiveCdpPage; - plan: ExternalHarnessTaskPlan; - logger: EvalLogger; -}): Promise> { - const sdk = (await import("@anthropic-ai/claude-agent-sdk")) as unknown as { - createSdkMcpServer: SdkMcpServerFactory; - tool: SdkToolFactory; - }; - - const runTool = sdk.tool( - "run", - [ - "Execute JavaScript against the initialized Chrome DevTools Protocol browser.", - "The snippet runs inside an async function with cdp, startUrl, task, and console in scope.", - "Use await directly. Return a JSON-serializable value when useful.", - ].join(" "), - { - code: z - .string() - .describe("JavaScript function body to execute. cdp/startUrl/task are already in scope."), - }, - async ({ code }) => { - return executeCdpRunTool({ - code, - connection: input.connection, - activePage: input.activePage, - plan: input.plan, - logger: input.logger, - }); - }, - { alwaysLoad: true }, - ); - - return { - [RUN_TOOL_SERVER]: sdk.createSdkMcpServer({ - name: RUN_TOOL_SERVER, - version: "1.0.0", - tools: [runTool], - alwaysLoad: true, - }), - }; -} - -async function executeCdpRunTool(input: { - code: string; - connection: CdpConnection; - activePage: ActiveCdpPage; - plan: ExternalHarnessTaskPlan; - logger: EvalLogger; -}): Promise { - try { - const result = await withTimeout( - executeCdpSnippet(input), - readPositiveIntEnv("EVAL_CLAUDE_CODE_RUN_TOOL_TIMEOUT_MS", 60_000), - ); - const text = stringifyToolResult(result); - input.logger.log({ - category: "claude_code", - message: `run tool completed: ${clip(text, 500)}`, - level: 1, - }); - return { - content: [{ type: "text", text }], - }; - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - input.logger.warn({ - category: "claude_code", - message: `run tool failed: ${message}`, - level: 1, - }); - return { - isError: true, - content: [{ type: "text", text: message }], - }; - } -} - -async function executeCdpSnippet(input: { - code: string; - connection: CdpConnection; - activePage: ActiveCdpPage; - plan: ExternalHarnessTaskPlan; - logger: EvalLogger; -}): Promise { - const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor as new ( - ...args: string[] - ) => (...values: unknown[]) => Promise; - const fn = new AsyncFunction("cdp", "startUrl", "task", "console", input.code); - return fn( - buildCdpRuntime(input.connection, input.activePage, input.logger), - input.plan.startUrl, - { - dataset: input.plan.dataset, - id: input.plan.taskId, - startUrl: input.plan.startUrl, - instruction: input.plan.instruction, - }, - buildRunToolConsole(input.logger), - ); -} - -function buildCdpRuntime( - connection: CdpConnection, - activePage: ActiveCdpPage, - logger: EvalLogger, -): CdpRuntime { - const listenerUnsubscribes = new Map< - (event: CdpEventMessage) => unknown | Promise, - () => void - >(); - return { - targetId: activePage.targetId, - sessionId: activePage.sessionId, - send: (method: string, params?: Record): Promise => - connection.send(method, params, activePage.sessionId), - browser: (method: string, params?: Record): Promise => - connection.send(method, params), - on: ( - method: string, - listener: (event: CdpEventMessage) => unknown | Promise, - ): (() => void) => { - const unsubscribe = onCdpEvent(connection, activePage.sessionId, method, listener, logger); - listenerUnsubscribes.set(listener, unsubscribe); - return () => { - listenerUnsubscribes.delete(listener); - unsubscribe(); - }; - }, - off: ( - _method: string, - listener: (event: CdpEventMessage) => unknown | Promise, - ): void => { - const unsubscribe = listenerUnsubscribes.get(listener); - listenerUnsubscribes.delete(listener); - unsubscribe?.(); - }, - once: ( - method: string, - listenerOrTimeout?: ((event: CdpEventMessage) => unknown | Promise) | number, - timeoutMs = 15_000, - ): Promise | (() => void) => { - if (typeof listenerOrTimeout === "function") { - const listener = listenerOrTimeout; - const unsubscribe = onCdpEvent( - connection, - activePage.sessionId, - method, - (event) => { - unsubscribe?.(); - listenerUnsubscribes.delete(listener); - return listener(event); - }, - logger, - ); - listenerUnsubscribes.set(listener, unsubscribe); - return () => { - listenerUnsubscribes.delete(listener); - unsubscribe?.(); - }; - } - return waitForCdpEvent( - connection, - activePage.sessionId, - method, - listenerOrTimeout ?? timeoutMs, - ); - }, - waitForEvent: (method: string, timeoutMs = 15_000): Promise => - waitForCdpEvent(connection, activePage.sessionId, method, timeoutMs), - wait: sleep, - }; -} - -function onCdpEvent( - connection: CdpConnection, - sessionId: string, - method: string, - listener: (event: CdpEventMessage) => unknown | Promise, - logger: EvalLogger, -): () => void { - return connection.onEvent((event) => { - if (event.method !== method || (event.sessionId && event.sessionId !== sessionId)) { - return; - } - try { - const result = listener(event); - if (isPromiseLike(result)) { - result.catch((error: unknown) => { - logger.warn({ - category: "claude_code", - message: `cdp event listener failed: ${error instanceof Error ? error.message : String(error)}`, - level: 1, - }); - }); - } - } catch (error) { - logger.warn({ - category: "claude_code", - message: `cdp event listener failed: ${error instanceof Error ? error.message : String(error)}`, - level: 1, - }); - } - }); -} - -async function attachActiveCdpPage(connection: CdpConnection): Promise { - const targets = await connection.send<{ - targetInfos: Array<{ - targetId: string; - type: string; - url?: string; - }>; - }>("Target.getTargets"); - - const existingPage = targets.targetInfos.find( - (target) => target.type === "page" && !target.url?.startsWith("devtools://"), - ); - const targetId = - existingPage?.targetId ?? - ( - await connection.send<{ targetId: string }>("Target.createTarget", { - url: "about:blank", - }) - ).targetId; - const attached = await connection.send<{ sessionId: string }>("Target.attachToTarget", { - targetId, - flatten: true, - }); - - await connection.send("Page.enable", {}, attached.sessionId); - await connection.send("Runtime.enable", {}, attached.sessionId); - await connection.send("DOM.enable", {}, attached.sessionId); - await connection.send("Page.setLifecycleEventsEnabled", { enabled: true }, attached.sessionId); - - return { - targetId, - sessionId: attached.sessionId, - url: existingPage?.url ?? "about:blank", - }; -} - -export function waitForCdpEvent( - connection: CdpConnection, - sessionId: string, - method: string, - timeoutMs: number, -): Promise { - let timeout: NodeJS.Timeout | undefined; - let unsubscribe: (() => void) | undefined; - const promise = new Promise((resolve, reject) => { - const cleanup = () => { - if (timeout) clearTimeout(timeout); - unsubscribe?.(); - }; - unsubscribe = connection.onEvent((event) => { - if (event.method !== method || (event.sessionId && event.sessionId !== sessionId)) { - return; - } - cleanup(); - resolve(event); - }); - timeout = setTimeout(() => { - cleanup(); - reject(new Error(`Timed out waiting for CDP event "${method}"`)); - }, timeoutMs); - }); - - // Claude-generated snippets often assign an event wait promise before a CDP - // action and may abandon it after another branch finishes. Keep the promise - // rejectable for awaited callers, but prevent abandoned waits from crashing - // the eval process as unhandled rejections. - promise.catch((): undefined => undefined); - return promise; -} - function buildRunToolConsole(logger: EvalLogger): Pick { const write = (level: "log" | "warn" | "error", values: unknown[]) => { logger.log({ @@ -985,32 +584,6 @@ function buildRunToolConsole(logger: EvalLogger): Pick 0 ? parsed : fallback; } -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - async function withTimeout(promise: Promise, timeoutMs: number): Promise { let timeout: NodeJS.Timeout | undefined; try { @@ -1132,19 +701,6 @@ function clip(value: string, maxLength: number): string { return value.length <= maxLength ? value : `${value.slice(0, maxLength - 1)}…`; } -function isPromiseLike(value: unknown): value is PromiseLike & { - catch: (handler: (error: unknown) => void) => unknown; -} { - return ( - value !== null && - typeof value === "object" && - "then" in value && - typeof (value as { then?: unknown }).then === "function" && - "catch" in value && - typeof (value as { catch?: unknown }).catch === "function" - ); -} - function createBrowseSessionName(): string { return `evals-claude-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; } diff --git a/packages/evals/framework/codexCodeBridge.ts b/packages/evals/framework/codexCodeBridge.ts new file mode 100644 index 000000000..c50fb671b --- /dev/null +++ b/packages/evals/framework/codexCodeBridge.ts @@ -0,0 +1,207 @@ +/** + * Code-execution bridge for the codex harness. + * + * codex-sdk has no in-process MCP server mounting (unlike claude-agent-sdk), + * and an external MCP process could not share this process's live surface + * handles (the Stagehand SDK client, playwright browser objects). So the + * codex mount for a handle binding is a loopback HTTP bridge: + * this process executes snippets against the in-memory handles; the codex + * workspace gets a tiny client script (`browser_run.mjs`) that posts a + * snippet file's contents to the bridge and prints the result. + * + * Scope semantics are identical to the claude_code run tool: the snippet + * runs inside an async function whose arguments are the mount's handle + * names plus startUrl, task, and console — names, not order, bind values. + */ +import http from "node:http"; +import type { AddressInfo } from "node:net"; +import type { AgentMount } from "../core/contracts/tool.js"; +import type { ExternalHarnessTaskPlan } from "./externalHarnessPlan.js"; +import type { EvalLogger } from "../logger.js"; + +const DEFAULT_RUN_TIMEOUT_MS = 60_000; + +function readPositiveIntEnv(name: string, fallback: number): number { + const parsed = Number.parseInt(process.env[name] ?? "", 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +function withTimeout(promise: Promise, timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error(`run timed out after ${timeoutMs}ms`)), + timeoutMs, + ); + promise.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (error) => { + clearTimeout(timer); + reject(error); + }, + ); + }); +} + +function stringifyResult(value: unknown): string { + if (typeof value === "undefined") return "undefined"; + if (typeof value === "string") return value; + try { + return JSON.stringify(value, null, 2) ?? String(value); + } catch { + return String(value); + } +} + +/** + * Snippet errors can embed connection URLs whose query strings carry + * credentials (Browserbase connect URLs include signing keys). Redact + * credential-bearing fragments before the message crosses the HTTP boundary + * or reaches the logs; the rest stays intact so the agent can self-correct. + */ +export function sanitizeErrorMessage(message: string): string { + return message + .replace(/([?&](?:signingKey|apiKey|api_key|token|key)=)[^&\s"']+/gi, "$1[redacted]") + .replace(/\b(sk-[A-Za-z0-9_-]{6})[A-Za-z0-9_-]+/g, "$1[redacted]"); +} + +export interface CodeBridge { + port: number; + close: () => Promise; +} + +export async function startCodeBridge(input: { + mount: Extract; + plan: ExternalHarnessTaskPlan; + logger: EvalLogger; + /** Awaited after every bridge run, success or failure (per-step probe). */ + onRunExecuted?: () => Promise; +}): Promise { + const { mount, plan, logger } = input; + const handles = mount.handles; + const bridgeConsole = { + log: (...args: unknown[]) => + logger.log({ category: "codex", level: 1, message: args.join(" ") }), + warn: (...args: unknown[]) => + logger.warn({ category: "codex", level: 1, message: args.join(" ") }), + error: (...args: unknown[]) => + logger.error({ category: "codex", level: 0, message: args.join(" ") }), + }; + + async function executeSnippet(code: string): Promise { + const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor as new ( + ...args: string[] + ) => (...values: unknown[]) => Promise; + const fn = new AsyncFunction(...Object.keys(handles), "startUrl", "task", "console", code); + return fn( + ...Object.values(handles), + plan.startUrl, + { + dataset: plan.dataset, + id: plan.taskId, + instruction: plan.instruction, + startUrl: plan.startUrl, + }, + bridgeConsole, + ); + } + + // Evidence collection is best-effort: a probe failure must never hang a + // bridge request or turn a successful run into an error response. + async function notifyRunExecuted(): Promise { + try { + await input.onRunExecuted?.(); + } catch { + // best-effort only + } + } + + const server = http.createServer((req, res) => { + if (req.method !== "POST" || req.url !== "/run") { + res.writeHead(404).end(); + return; + } + let body = ""; + req.on("data", (chunk) => (body += chunk)); + req.on("end", async () => { + let code: string; + try { + code = String((JSON.parse(body) as { code?: unknown }).code ?? ""); + } catch { + res.writeHead(400).end(JSON.stringify({ ok: false, error: "invalid JSON body" })); + return; + } + try { + const result = await withTimeout( + executeSnippet(code), + readPositiveIntEnv("EVAL_CODEX_RUN_TOOL_TIMEOUT_MS", DEFAULT_RUN_TIMEOUT_MS), + ); + const text = stringifyResult(result); + logger.log({ + category: "codex", + level: 1, + message: `bridge run completed: ${text.slice(0, 500)}`, + }); + await notifyRunExecuted(); + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: true, result: text })); + } catch (error) { + const message = sanitizeErrorMessage( + error instanceof Error ? error.message : String(error), + ); + logger.warn({ + category: "codex", + level: 1, + message: `bridge run failed: ${message}`, + }); + await notifyRunExecuted(); + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ ok: false, error: message })); + } + }); + }); + + await new Promise((resolve) => { + server.listen(0, "127.0.0.1", resolve); + }); + const port = (server.address() as AddressInfo).port; + + return { + port, + close: () => + new Promise((resolve) => { + server.close(() => resolve()); + server.closeAllConnections?.(); + }), + }; +} + +/** + * The workspace client codex invokes via shell. Kept dependency-free and + * tiny: read a snippet file (or stdin), post to the bridge, print the + * result text; non-zero exit on execution error so the agent notices. + */ +export function buildBridgeClientScript(port: number): string { + return `#!/usr/bin/env node +// browser_run.mjs — execute a browser-automation snippet via the eval bridge. +// Usage: node browser_run.mjs (or pipe the snippet on stdin) +import { readFileSync } from "node:fs"; + +const file = process.argv[2]; +const code = file ? readFileSync(file, "utf8") : readFileSync(0, "utf8"); +const res = await fetch("http://127.0.0.1:${port}/run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ code }), +}); +const payload = await res.json(); +if (payload.ok) { + console.log(payload.result); +} else { + console.error("run failed: " + payload.error); + process.exit(1); +} +`; +} diff --git a/packages/evals/framework/codexRunner.ts b/packages/evals/framework/codexRunner.ts index 46267e6f5..95f420ba5 100644 --- a/packages/evals/framework/codexRunner.ts +++ b/packages/evals/framework/codexRunner.ts @@ -212,6 +212,18 @@ export async function runCodexAgent({ return baseResult; } + // Artifact-grounded grading: capture the terminal page state through the + // tool surface (harness-observed, independent of the agent's self-report) + // before cleanup, mirroring the claude_code runner. + const finalObservation = + toolAdapter && "captureEvidence" in toolAdapter + ? await toolAdapter.captureEvidence?.().catch((): undefined => undefined) + : undefined; + const stepObservations = + toolAdapter && "drainStepObservations" in toolAdapter + ? toolAdapter.drainStepObservations?.() + : undefined; + // Build a Trajectory from the codex event stream and grade it with the // rubric verifier; any failure in that path folds into `verifierError`. return gradeExternalTrajectory({ @@ -219,6 +231,8 @@ export async function runCodexAgent({ codexAdapter.fromHarnessResult( { events, + ...(finalObservation && { finalObservation }), + ...(stepObservations?.length && { stepObservations }), finalAnswer: parsed.finalAnswer ?? finalResponse, status: status === "completed" ? "complete" : "error", usage: { diff --git a/packages/evals/framework/codexToolAdapter.ts b/packages/evals/framework/codexToolAdapter.ts index 5b5843d5f..ef3dd08e5 100644 --- a/packages/evals/framework/codexToolAdapter.ts +++ b/packages/evals/framework/codexToolAdapter.ts @@ -1,7 +1,18 @@ +import fsp from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; import { EvalsError } from "../errors.js"; import type { EvalLogger } from "../logger.js"; -import type { StartupProfile, ToolSurface } from "../core/contracts/tool.js"; +import { + AGENT_RUN_TOOL_NAME, + type StartupProfile, + type ToolSurface, +} from "../core/contracts/tool.js"; +import type { ProbeEvidence } from "stagehand-v3"; +import { startAgentToolRuntime } from "./agentToolRuntime.js"; import type { ExternalHarnessTaskPlan } from "./externalHarnessPlan.js"; +import { buildBridgeClientScript, startCodeBridge } from "./codexCodeBridge.js"; +import { ObservationRecorder, type StepObservation } from "./observationRecorder.js"; import { prepareBrowseCliHarnessAdapter, type PreparedBrowseCliHarnessAdapter, @@ -15,7 +26,22 @@ export interface CodexToolAdapterInput { logger: EvalLogger; } -export type PreparedCodexToolAdapter = PreparedBrowseCliHarnessAdapter; +/** Code-surface variant: same runner-facing fields as the browse_cli shape. */ +export interface PreparedCodexCodeAdapter { + toolSurface: ToolSurface; + startupProfile: StartupProfile; + cwd: string; + env: Record; + promptInstructions: string; + /** Best-effort evidence from the currently running tool surface. */ + captureEvidence?: () => Promise; + drainStepObservations?: () => StepObservation[]; + cleanup: () => Promise; +} + +export type PreparedCodexToolAdapter = PreparedBrowseCliHarnessAdapter | PreparedCodexCodeAdapter; + +const CODE_SURFACES = new Set(["stagehand_code", "playwright_code", "cdp_code"]); export async function prepareCodexToolAdapter( input: CodexToolAdapterInput, @@ -27,20 +53,122 @@ export async function prepareCodexToolAdapter( input.startupProfile, ); - return prepareBrowseCliHarnessAdapter({ + if (toolSurface === "browse_cli") { + return prepareBrowseCliHarnessAdapter({ + startupProfile, + environment: input.environment, + plan: input.plan, + logger: input.logger, + logCategory: "codex", + }); + } + + const runtime = await startAgentToolRuntime({ + toolSurface, startupProfile, environment: input.environment, - plan: input.plan, logger: input.logger, - logCategory: "codex", }); + + let cwd: string | undefined; + let bridge: Awaited> | undefined; + try { + const mount = runtime.running.agentMount; + if (!mount) { + throw new EvalsError(`Tool surface "${toolSurface}" does not provide an agent mount.`); + } + if (mount.via !== "handles") { + throw new EvalsError(`Codex does not support agent mounts delivered via "${mount.via}" yet.`); + } + const recorder = runtime.running.captureEvidence + ? new ObservationRecorder(runtime.running.captureEvidence) + : undefined; + bridge = await startCodeBridge({ + mount, + plan: input.plan, + logger: input.logger, + onRunExecuted: recorder ? () => recorder.record() : undefined, + }); + cwd = await fsp.mkdtemp( + path.join(os.tmpdir(), `stagehand-evals-codex-${toolSurface.replace(/_/g, "-")}-`), + ); + await fsp.writeFile(path.join(cwd, "browser_run.mjs"), buildBridgeClientScript(bridge.port)); + + input.logger.log({ + category: "codex", + message: `Initialized ${toolSurface} bridge runtime for Codex (port ${bridge.port}).`, + level: 1, + auxiliary: { + startupProfile: { value: startupProfile, type: "string" }, + environment: { value: input.environment, type: "string" }, + }, + }); + + const capturedBridge = bridge; + const capturedCwd = cwd; + return { + toolSurface, + startupProfile, + cwd, + env: { ...process.env } as Record, + promptInstructions: buildCodexCodePromptInstructions(mount, toolSurface), + ...(runtime.running.captureEvidence && { + captureEvidence: runtime.running.captureEvidence, + }), + ...(recorder && { drainStepObservations: () => recorder.drain() }), + cleanup: async () => { + try { + await capturedBridge.close(); + } catch { + // best-effort only + } + try { + await runtime.cleanup(); + } catch { + // best-effort only + } + await fsp.rm(capturedCwd, { recursive: true, force: true }); + }, + }; + } catch (error) { + await bridge?.close().catch((): undefined => undefined); + await runtime.cleanup().catch((): undefined => undefined); + if (cwd) await fsp.rm(cwd, { recursive: true, force: true }); + throw error; + } +} + +/** + * Codex has no MCP run tool — snippets go through the workspace bridge + * client. Reuse the surface's own API guidance, rewriting the claude-style + * run-tool reference to the codex invocation. + */ +function buildCodexCodePromptInstructions( + mount: { promptInstructions: string; handles: Record }, + toolSurface: ToolSurface, +): string { + const scopeNames = [...Object.keys(mount.handles), "startUrl", "task", "console"].join(", "); + const surfaceGuidance = mount.promptInstructions.replaceAll( + AGENT_RUN_TOOL_NAME, + "browser_run.mjs", + ); + return [ + `Browser automation for this task runs through a snippet bridge, not a browser you launch.`, + `Write a JavaScript snippet to a file (e.g. snippet.js), then execute it with: node browser_run.mjs snippet.js`, + `The snippet runs inside an async function with ${scopeNames} in scope. Use await directly; return a JSON-serializable value to inspect it.`, + `Never launch your own browser process; browser_run.mjs is the only browser access.`, + surfaceGuidance, + `Surface: ${toolSurface}.`, + ].join("\n"); } export function resolveCodexToolSurface(requested?: ToolSurface): ToolSurface { if (!requested) return "browse_cli"; - if (requested === "browse_cli") return requested; + if (requested === "browse_cli" || CODE_SURFACES.has(requested)) { + return requested; + } throw new EvalsError( - `Codex harness supports --tool browse_cli for execution right now; received "${requested}".`, + `Codex harness supports --tool browse_cli, playwright_code, cdp_code, or stagehand_code for execution right now; received "${requested}".`, ); } @@ -51,9 +179,16 @@ export function resolveCodexStartupProfile( ): StartupProfile { if (requested) return requested; - if (toolSurface === "browse_cli") { + // browse_cli and stagehand_code own their browser; playwright/cdp attach to a + // runner-provided CDP endpoint (same defaults as the claude_code harness). + if (toolSurface === "browse_cli" || toolSurface === "stagehand_code") { return environment === "BROWSERBASE" ? "tool_create_browserbase" : "tool_launch_local"; } + if (toolSurface === "playwright_code" || toolSurface === "cdp_code") { + return environment === "BROWSERBASE" + ? "runner_provided_browserbase_cdp" + : "runner_provided_local_cdp"; + } throw new EvalsError( `No Codex startup profile default for tool "${toolSurface}" in ${environment}.`, diff --git a/packages/evals/framework/harnesses/claudeCodeAdapter.ts b/packages/evals/framework/harnesses/claudeCodeAdapter.ts index bd8562581..3efe90afc 100644 --- a/packages/evals/framework/harnesses/claudeCodeAdapter.ts +++ b/packages/evals/framework/harnesses/claudeCodeAdapter.ts @@ -24,6 +24,8 @@ * can't ground. */ import type { ProbeEvidence, TaskSpec, Trajectory } from "stagehand-v3"; +import { AGENT_RUN_TOOL_NAME } from "../../core/contracts/tool.js"; +import type { StepObservation } from "../observationRecorder.js"; import { buildTrajectory, type NormalizedToolCall, @@ -40,6 +42,13 @@ export interface ClaudeCodeRunResult { status?: Trajectory["status"]; /** Optional usage to fold into Trajectory.usage. */ usage?: Partial; + /** + * Harness-observed terminal page state (captured through the tool surface + * after the agent finished) — anchors the verifier's final observation. + */ + finalObservation?: ProbeEvidence; + /** Per-step probe observations, indexed by run-tool execution order. */ + stepObservations?: StepObservation[]; } interface ToolUseBlock { @@ -141,10 +150,18 @@ export class ClaudeCodeTrajectoryAdapter implements TrajectoryAdapter [o.runIndex, o.evidence]), + ); + let runOrdinal = 0; const toolCalls: NormalizedToolCall[] = toolUses.map((use) => { const matched = toolResults.get(use.id); const ok = matched ? !matched.isError : true; const resultPayload = matched?.raw !== undefined ? matched.raw : (matched?.text ?? ""); + // The Nth run-tool call pairs with the Nth recorded observation; other + // tools (Bash etc.) never consume an index. + const observation = + use.name === AGENT_RUN_TOOL_NAME ? observationsByRunIndex.get(runOrdinal++) : undefined; return { name: use.name, args: use.input, @@ -153,6 +170,7 @@ export class ClaudeCodeTrajectoryAdapter implements TrajectoryAdapter 0 ? trailing : undefined); - // Anchor the closing frame with the most recent screenshot the agent - // captured. Claude Code doesn't run a post-task probe, so the last - // tool_result image is the best proxy for "terminal observation" — without - // it the verifier's final-screenshot anchor (evidence.ts:136-143) is empty. - let finalObservation: ProbeEvidence | undefined; - for (let i = toolUses.length - 1; i >= 0; i--) { + // Anchor the closing frame with the harness-observed terminal artifact + // when the runner captured one; otherwise fall back to the most recent + // screenshot the agent captured — without either, the verifier's + // final-screenshot anchor (evidence.ts:136-143) is empty. + let finalObservation: ProbeEvidence | undefined = result.finalObservation?.screenshot + ? result.finalObservation + : undefined; + for (let i = toolUses.length - 1; !finalObservation && i >= 0; i--) { const matched = toolResults.get(toolUses[i].id); const lastImage = matched?.images[matched.images.length - 1]; if (lastImage) { finalObservation = { screenshot: lastImage.bytes }; - break; } } diff --git a/packages/evals/framework/harnesses/codexAdapter.ts b/packages/evals/framework/harnesses/codexAdapter.ts index 714d74c4e..1f869ebdc 100644 --- a/packages/evals/framework/harnesses/codexAdapter.ts +++ b/packages/evals/framework/harnesses/codexAdapter.ts @@ -26,7 +26,8 @@ * query in args. * - todo_list items → not surfaced as tool calls (they aren't actions). */ -import type { TaskSpec, Trajectory } from "stagehand-v3"; +import type { ProbeEvidence, TaskSpec, Trajectory } from "stagehand-v3"; +import type { StepObservation } from "../observationRecorder.js"; import { buildTrajectory, type NormalizedToolCall, @@ -42,6 +43,13 @@ export interface CodexRunResult { status?: Trajectory["status"]; /** Optional usage to fold into Trajectory.usage. */ usage?: Partial; + /** + * Harness-observed terminal page state (captured through the tool surface + * after the agent finished) — anchors the verifier's final observation. + */ + finalObservation?: ProbeEvidence; + /** Per-step probe observations, indexed by bridge-run execution order. */ + stepObservations?: StepObservation[]; } export class CodexTrajectoryAdapter implements TrajectoryAdapter { @@ -76,6 +84,30 @@ export class CodexTrajectoryAdapter implements TrajectoryAdapter } } + // The Nth recorded observation pairs with the Nth bridge run — the + // command_execution items that invoke browser_run.mjs, in stream order. + // If a bridge run is not visible under that filter (the agent reached + // the bridge some other way), ordinals would shift and attach evidence + // to the wrong steps — misattribution is worse than a gap, so attach + // nothing and let the verifier take its evidence_insufficient path. + const observations = result.stepObservations ?? []; + if (observations.length > 0) { + const bridgeCalls = toolCalls.filter( + (call) => + typeof call.args.command === "string" && call.args.command.includes("browser_run.mjs"), + ); + // runIndex counts every bridge run (failed captures leave gaps), so + // max+1 is the number of runs the recorder actually saw. + const totalBridgeRuns = Math.max(...observations.map((o) => o.runIndex)) + 1; + if (bridgeCalls.length >= totalBridgeRuns) { + const observationsByRunIndex = new Map(observations.map((o) => [o.runIndex, o.evidence])); + bridgeCalls.forEach((call, ordinal) => { + const observation = observationsByRunIndex.get(ordinal); + if (observation) call.probeEvidence = observation; + }); + } + } + const finalAnswer = result.finalAnswer ?? latestAgentMessage; return buildTrajectory({ @@ -84,6 +116,9 @@ export class CodexTrajectoryAdapter implements TrajectoryAdapter finalAnswer, status: result.status ?? "complete", usage: result.usage, + ...(result.finalObservation?.screenshot && { + finalObservation: result.finalObservation, + }), }); } } diff --git a/packages/evals/framework/harnesses/trajectoryAdapter.ts b/packages/evals/framework/harnesses/trajectoryAdapter.ts index da6d6f2a7..2d43cf6d5 100644 --- a/packages/evals/framework/harnesses/trajectoryAdapter.ts +++ b/packages/evals/framework/harnesses/trajectoryAdapter.ts @@ -44,6 +44,11 @@ export interface NormalizedToolCall { * the verifier can ground visual criteria against them. */ images?: Array<{ bytes: Buffer; mediaType: string }>; + /** + * Harness-observed page state after this call (url/screenshot probe). + * Attached by adapters when the run collected per-step observations. + */ + probeEvidence?: ProbeEvidence; } /** @@ -107,10 +112,10 @@ export function toolCallToTrajectoryStep(call: NormalizedToolCall): TrajectorySt actionArgs: call.args, reasoning: call.reasoning ?? "", agentEvidence: actionToAgentEvidence(call), - // External harnesses don't natively produce screenshots/aria/scroll, so - // probeEvidence stays empty. The verifier handles this via the + // Runs that collected per-step observations carry them here; otherwise + // probeEvidence stays empty and the verifier degrades via the // evidence_insufficient path. - probeEvidence: {}, + probeEvidence: call.probeEvidence ?? {}, toolOutput: { ok: call.ok, result: call.result, diff --git a/packages/evals/framework/observationRecorder.ts b/packages/evals/framework/observationRecorder.ts new file mode 100644 index 000000000..6fd539a88 --- /dev/null +++ b/packages/evals/framework/observationRecorder.ts @@ -0,0 +1,73 @@ +import type { ProbeEvidence } from "stagehand-v3"; + +/** A probe observation captured after the Nth run-tool execution (0-based). */ +export interface StepObservation { + runIndex: number; + evidence: ProbeEvidence; +} + +const OBSERVATIONS_ENV = "EVAL_HARNESS_OBSERVATIONS"; +const OBSERVATION_TIMEOUT_ENV = "EVAL_OBSERVATION_TIMEOUT_MS"; +const DEFAULT_OBSERVATION_TIMEOUT_MS = 10_000; + +/** + * Whether this run collects per-step observations. Sampling is a run-level + * decision: batch tooling sets EVAL_HARNESS_OBSERVATIONS=none on runs it + * excludes from evidence collection; the default is to observe. + */ +export function harnessObservationsEnabled(): boolean { + return (process.env[OBSERVATIONS_ENV] ?? "all") !== "none"; +} + +/** + * Buffers per-step probe observations for an external-harness run. Each + * record() consumes one run index (matching the harness's Nth run-tool + * execution); a failed capture leaves a gap at its index rather than + * shifting later observations onto the wrong step. + */ +export class ObservationRecorder { + private readonly observations: StepObservation[] = []; + private runIndex = 0; + + constructor(private readonly capture: () => Promise) {} + + async record(): Promise { + const runIndex = this.runIndex++; + try { + const evidence = await withTimeout(this.capture(), observationTimeoutMs()); + if (evidence.screenshot || evidence.url || evidence.ariaTree) { + this.observations.push({ runIndex, evidence }); + } + } catch { + // best-effort only — a failed probe must never fail the run tool + } + } + + drain(): StepObservation[] { + return this.observations.splice(0); + } +} + +function observationTimeoutMs(): number { + const parsed = Number.parseInt(process.env[OBSERVATION_TIMEOUT_ENV] ?? "", 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_OBSERVATION_TIMEOUT_MS; +} + +function withTimeout(promise: Promise, timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error(`observation timed out after ${timeoutMs}ms`)), + timeoutMs, + ); + promise.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (error) => { + clearTimeout(timer); + reject(error); + }, + ); + }); +} diff --git a/packages/evals/framework/verifierGate.ts b/packages/evals/framework/verifierGate.ts new file mode 100644 index 000000000..f5ec94ef9 --- /dev/null +++ b/packages/evals/framework/verifierGate.ts @@ -0,0 +1,62 @@ +/** + * Per-arm verifiability accounting for bench batches (STG-2752). + * + * Each graded run reports `criterionCount` and `evidenceInsufficient` (the + * rubric criteria the verifier could not ground in evidence). This module + * aggregates those per arm — one (harness × tool surface × model) cell of + * the bench matrix — so unverifiable-criteria counts are reported per arm + * and, when EVAL_MAX_UNVERIFIABLE_CRITERIA is set, gate the batch. + */ +import type { EvalInput } from "../types/evals.js"; + +export interface ArmVerifiability { + arm: string; + /** Runs the verifier graded (criterionCount present). */ + gradedRuns: number; + unverifiableCriteria: number; + totalCriteria: number; +} + +const GATE_ENV = "EVAL_MAX_UNVERIFIABLE_CRITERIA"; + +export function summarizeArmVerifiability( + results: Array<{ input: EvalInput; output: Record }>, + harness: string, +): ArmVerifiability[] { + const arms = new Map(); + for (const { input, output } of results) { + if (typeof output.criterionCount !== "number") continue; + const toolSurface = + typeof input.params?.toolSurface === "string" ? input.params.toolSurface : undefined; + const key = [harness, toolSurface, input.modelName].filter(Boolean).join(" × "); + const arm = arms.get(key) ?? { + arm: key, + gradedRuns: 0, + unverifiableCriteria: 0, + totalCriteria: 0, + }; + arm.gradedRuns += 1; + arm.totalCriteria += output.criterionCount; + arm.unverifiableCriteria += Array.isArray(output.evidenceInsufficient) + ? output.evidenceInsufficient.length + : 0; + arms.set(key, arm); + } + return [...arms.values()]; +} + +/** + * Per-arm ceiling on unverifiable criteria. Unset or invalid = report only. + * Strict integer parsing: a malformed value ("1.5", "10foo") must not be + * coerced into an unintended gate. + */ +export function resolveUnverifiableCriteriaLimit(): number | undefined { + const raw = process.env[GATE_ENV]?.trim(); + if (!raw || !/^\d+$/.test(raw)) return undefined; + return Number(raw); +} + +/** Arms whose unverifiable-criteria count exceeds the limit. */ +export function armsOverLimit(arms: ArmVerifiability[], limit: number): ArmVerifiability[] { + return arms.filter((arm) => arm.unverifiableCriteria > limit); +} diff --git a/packages/evals/tests/core/tool-contract.test.ts b/packages/evals/tests/core/tool-contract.test.ts new file mode 100644 index 000000000..a65be6eb8 --- /dev/null +++ b/packages/evals/tests/core/tool-contract.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from "vitest"; +import type { + AgentMount, + CoreSession, + CoreTool, + ToolStartResult, +} from "../../core/contracts/tool.js"; + +describe("tool surface contract", () => { + it("keeps native surface and agent delivery independent", async () => { + const screenshot = Buffer.from("surface screenshot"); + const cliMount = { + via: "cli", + promptInstructions: "Use the wrapper command.", + command: { bin: "stagehand-browser", env: {} }, + } satisfies AgentMount; + + const runningCodeSurface: ToolStartResult = { + session: {} as CoreSession, + agentMount: cliMount, + captureEvidence: async () => ({ + screenshot, + url: "https://example.com", + ariaTree: "- document\n - heading: Example", + }), + cleanup: async () => {}, + metadata: { + environment: "local", + browserOwnership: "tool", + connectionMode: "launch", + }, + }; + + const codeSurface: CoreTool = { + id: "playwright_code", + surface: "code", + family: "playwright", + supportedStartupProfiles: [], + supportedCapabilities: [], + supportedTargetKinds: [], + start: async () => runningCodeSurface, + }; + + expect(codeSurface.surface).toBe("code"); + expect(runningCodeSurface.agentMount?.via).toBe("cli"); + await expect(runningCodeSurface.captureEvidence?.()).resolves.toEqual({ + screenshot, + url: "https://example.com", + ariaTree: "- document\n - heading: Example", + }); + }); + + it("describes injected handles without harness-owned task bindings", () => { + const mount = { + via: "handles", + promptInstructions: "Use the run tool.", + handles: { page: {} }, + runTool: { + description: "Run browser code.", + codeParamDescription: "JavaScript to execute.", + denyMessage: "Use the run tool.", + }, + } satisfies AgentMount; + + expect(mount.via).toBe("handles"); + }); + + it("requires delivery-specific fields at compile time", () => { + // @ts-expect-error CLI mounts require a command. + const invalidMount: AgentMount = { + via: "cli", + promptInstructions: "Use the CLI.", + }; + + expect(invalidMount.via).toBe("cli"); + }); +}); diff --git a/packages/evals/tests/core/tool-registry.test.ts b/packages/evals/tests/core/tool-registry.test.ts index 0d9c7dbf8..dd995617e 100644 --- a/packages/evals/tests/core/tool-registry.test.ts +++ b/packages/evals/tests/core/tool-registry.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { getCoreTool, listCoreTools } from "../../core/tools/registry.js"; +import { buildStagehandCodePromptInstructions } from "../../core/tools/stagehand_code.js"; describe("core tool registry", () => { it("lists extended tool surfaces", () => { @@ -12,5 +13,18 @@ describe("core tool registry", () => { expect(getCoreTool("playwright_mcp").id).toBe("playwright_mcp"); expect(getCoreTool("chrome_devtools_mcp").id).toBe("chrome_devtools_mcp"); expect(getCoreTool("browse_cli").id).toBe("browse_cli"); + expect(getCoreTool("stagehand_code").id).toBe("stagehand_code"); + }); + + it("shows Stagehand locator actions to coding agents", () => { + const prompt = buildStagehandCodePromptInstructions(); + + expect(prompt).toContain("goBack()/goForward()"); + expect(prompt).toContain("keyPress(key)"); + expect(prompt).toContain("click(x,y), hover(x,y)"); + expect(prompt).toContain( + "page.locator(selector): count(), click(), hover(), fill(value), type(text), isVisible(), textContent(), inputValue()", + ); + expect(prompt).toContain("Page accessors are async RPCs — always await them."); }); }); diff --git a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts index 754292f7b..0f1d2e85b 100644 --- a/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts +++ b/packages/evals/tests/framework/claudeCodeToolAdapter.test.ts @@ -51,18 +51,35 @@ describe("claude code tool adapter resolution", () => { it("rejects unsupported Claude Code tool surfaces for now", () => { expect(() => resolveClaudeCodeToolSurface("understudy_code")).toThrow( - /supports --tool browse_cli, playwright_code, or cdp_code/, + /supports --tool browse_cli, playwright_code, cdp_code, or stagehand_code/, ); }); - it("supports browse_cli as the first Codex tool surface", () => { + it("accepts stagehand_code with SDK-owned startup profiles", () => { + expect(resolveClaudeCodeToolSurface("stagehand_code")).toBe("stagehand_code"); + expect(resolveClaudeCodeStartupProfile("stagehand_code", "BROWSERBASE")).toBe( + "tool_create_browserbase", + ); + expect(resolveClaudeCodeStartupProfile("stagehand_code", "LOCAL")).toBe("tool_launch_local"); + }); + + it("supports browse_cli and the code surfaces on Codex", () => { expect(resolveCodexToolSurface()).toBe("browse_cli"); expect(resolveCodexToolSurface("browse_cli")).toBe("browse_cli"); + expect(resolveCodexToolSurface("stagehand_code")).toBe("stagehand_code"); + expect(resolveCodexToolSurface("playwright_code")).toBe("playwright_code"); + expect(resolveCodexToolSurface("cdp_code")).toBe("cdp_code"); + expect(resolveCodexStartupProfile("stagehand_code", "BROWSERBASE")).toBe( + "tool_create_browserbase", + ); + expect(resolveCodexStartupProfile("playwright_code", "BROWSERBASE")).toBe( + "runner_provided_browserbase_cdp", + ); + expect(() => resolveCodexToolSurface("playwright_mcp")).toThrow( + /browse_cli, playwright_code, cdp_code, or stagehand_code/, + ); expect(resolveCodexStartupProfile("browse_cli", "LOCAL")).toBe("tool_launch_local"); expect(resolveCodexStartupProfile("browse_cli", "BROWSERBASE")).toBe("tool_create_browserbase"); - expect(() => resolveCodexToolSurface("playwright_code")).toThrow( - /Codex harness supports --tool browse_cli/, - ); }); it("allows only direct browse commands through Bash", () => { diff --git a/packages/evals/tests/framework/codexCodeBridge.test.ts b/packages/evals/tests/framework/codexCodeBridge.test.ts new file mode 100644 index 000000000..671061222 --- /dev/null +++ b/packages/evals/tests/framework/codexCodeBridge.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, it, afterEach } from "vitest"; +import { + buildBridgeClientScript, + startCodeBridge, + type CodeBridge, +} from "../../framework/codexCodeBridge.js"; +import { EvalLogger } from "../../logger.js"; +import type { AgentMount } from "../../core/contracts/tool.js"; +import type { ExternalHarnessTaskPlan } from "../../framework/externalHarnessPlan.js"; + +const plan: ExternalHarnessTaskPlan = { + dataset: "webvoyager", + taskId: "test-1", + startUrl: "https://example.com", + instruction: "do the thing", +}; + +function mountWith(handles: Record): Extract { + return { + via: "handles", + handles, + promptInstructions: "test", + runTool: { + description: "test run tool", + codeParamDescription: "code", + denyMessage: "deny", + }, + }; +} + +async function post(bridge: CodeBridge, code: string) { + const res = await fetch(`http://127.0.0.1:${bridge.port}/run`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ code }), + }); + return (await res.json()) as { ok: boolean; result?: string; error?: string }; +} + +let bridge: CodeBridge | undefined; +afterEach(async () => { + await bridge?.close(); + bridge = undefined; +}); + +describe("codex code bridge", () => { + it("executes snippets with handles, startUrl, and task in scope", async () => { + const page = { url: async () => "https://example.com/live" }; + bridge = await startCodeBridge({ + mount: mountWith({ page, marker: 42 }), + plan, + logger: new EvalLogger(), + }); + const out = await post( + bridge, + "return { url: await page.url(), marker, startUrl, instruction: task.instruction };", + ); + expect(out.ok).toBe(true); + expect(JSON.parse(out.result!)).toEqual({ + url: "https://example.com/live", + marker: 42, + startUrl: "https://example.com", + instruction: "do the thing", + }); + }); + + it("reports snippet errors without killing the bridge", async () => { + bridge = await startCodeBridge({ + mount: mountWith({}), + plan, + logger: new EvalLogger(), + }); + const bad = await post(bridge, "throw new Error('boom');"); + expect(bad).toEqual({ ok: false, error: "boom" }); + const good = await post(bridge, "return 'still alive';"); + expect(good).toEqual({ ok: true, result: "still alive" }); + }); + + it("times out runaway snippets", async () => { + process.env.EVAL_CODEX_RUN_TOOL_TIMEOUT_MS = "150"; + try { + bridge = await startCodeBridge({ + mount: mountWith({}), + plan, + logger: new EvalLogger(), + }); + const out = await post(bridge, "await new Promise(() => {});"); + expect(out.ok).toBe(false); + expect(out.error).toMatch(/timed out after 150ms/); + } finally { + delete process.env.EVAL_CODEX_RUN_TOOL_TIMEOUT_MS; + } + }); + + it("client script embeds the bridge port and pipes code", () => { + const script = buildBridgeClientScript(45678); + expect(script).toContain("http://127.0.0.1:45678/run"); + expect(script).toContain("process.argv[2]"); + }); + + it("redacts credentials from snippet error messages", async () => { + bridge = await startCodeBridge({ + mount: mountWith({}), + plan, + logger: new EvalLogger(false), + }); + const payload = await post( + bridge, + `throw new Error("connect failed: wss://connect.example.com/session?signingKey=sk-secret-value&x=1 (key sk-abcdef1234567890)")`, + ); + expect(payload.ok).toBe(false); + expect(payload.error).not.toContain("sk-secret-value"); + expect(payload.error).not.toContain("sk-abcdef1234567890"); + expect(payload.error).toContain("signingKey=[redacted]"); + expect(payload.error).toContain("connect failed"); + }); +}); diff --git a/packages/evals/tests/framework/harnessObservations.test.ts b/packages/evals/tests/framework/harnessObservations.test.ts new file mode 100644 index 000000000..20cc8b4a9 --- /dev/null +++ b/packages/evals/tests/framework/harnessObservations.test.ts @@ -0,0 +1,251 @@ +import { afterEach, describe, expect, it } from "vitest"; +import type { TaskSpec } from "stagehand-v3"; +import { AGENT_RUN_TOOL_NAME } from "../../core/contracts/tool.js"; +import { claudeCodeAdapter } from "../../framework/harnesses/claudeCodeAdapter.js"; +import { codexAdapter } from "../../framework/harnesses/codexAdapter.js"; +import { + harnessObservationsEnabled, + ObservationRecorder, +} from "../../framework/observationRecorder.js"; +import { + armsOverLimit, + resolveUnverifiableCriteriaLimit, + summarizeArmVerifiability, +} from "../../framework/verifierGate.js"; + +const TASK_SPEC: TaskSpec = { id: "t", instruction: "do the thing" }; + +describe("observation recorder", () => { + afterEach(() => { + delete process.env.EVAL_HARNESS_OBSERVATIONS; + delete process.env.EVAL_MAX_UNVERIFIABLE_CRITERIA; + }); + + it("observes by default and can be disabled per run", () => { + expect(harnessObservationsEnabled()).toBe(true); + process.env.EVAL_HARNESS_OBSERVATIONS = "none"; + expect(harnessObservationsEnabled()).toBe(false); + }); + + it("indexes observations by run and leaves gaps on capture failure", async () => { + let call = 0; + const recorder = new ObservationRecorder(async () => { + call += 1; + if (call === 2) throw new Error("probe failed"); + return { url: `https://example.com/${call}` }; + }); + await recorder.record(); + await recorder.record(); + await recorder.record(); + expect(recorder.drain().map((o) => [o.runIndex, o.evidence.url])).toEqual([ + [0, "https://example.com/1"], + [2, "https://example.com/3"], + ]); + expect(recorder.drain()).toEqual([]); + }); + + it("drops empty artifacts", async () => { + const recorder = new ObservationRecorder(async () => ({})); + await recorder.record(); + expect(recorder.drain()).toEqual([]); + }); +}); + +describe("per-step observations in trajectories", () => { + it("attaches claude_code observations to run-tool steps only", () => { + const messages = [ + assistantToolUse("u1", "Bash", { command: "ls" }), + toolResult("u1", "ok"), + assistantToolUse("u2", AGENT_RUN_TOOL_NAME, { code: "await page.goto(startUrl)" }), + toolResult("u2", "done"), + assistantToolUse("u3", AGENT_RUN_TOOL_NAME, { code: "await page.title()" }), + toolResult("u3", "Example"), + ]; + const trajectory = claudeCodeAdapter.fromHarnessResult( + { + messages, + stepObservations: [ + { runIndex: 0, evidence: { url: "https://example.com/a" } }, + { runIndex: 1, evidence: { url: "https://example.com/b" } }, + ], + finalObservation: { + url: "https://example.com/final", + screenshot: Buffer.from("final"), + }, + }, + TASK_SPEC, + ); + expect(trajectory.steps.map((s) => s.probeEvidence.url)).toEqual([ + undefined, + "https://example.com/a", + "https://example.com/b", + ]); + expect(trajectory.finalObservation?.url).toBe("https://example.com/final"); + }); + + it("attaches codex observations to bridge-run steps only", () => { + const events = [ + commandExecution("cat notes.txt"), + commandExecution("node browser_run.mjs snippet.js"), + commandExecution("node browser_run.mjs snippet2.js"), + ]; + const trajectory = codexAdapter.fromHarnessResult( + { + events, + stepObservations: [{ runIndex: 1, evidence: { url: "https://example.com/second" } }], + }, + TASK_SPEC, + ); + expect(trajectory.steps.map((s) => s.probeEvidence.url)).toEqual([ + undefined, + undefined, + "https://example.com/second", + ]); + }); + + it("maps the Nth codex observation to the Nth bridge run", () => { + const events = [ + commandExecution("node browser_run.mjs a.js"), + commandExecution("ls"), + commandExecution("node browser_run.mjs b.js"), + ]; + const trajectory = codexAdapter.fromHarnessResult( + { + events, + stepObservations: [ + { runIndex: 0, evidence: { url: "https://example.com/a" } }, + { runIndex: 1, evidence: { url: "https://example.com/b" } }, + ], + }, + TASK_SPEC, + ); + expect(trajectory.steps.map((s) => s.probeEvidence.url)).toEqual([ + "https://example.com/a", + undefined, + "https://example.com/b", + ]); + }); + + it("attaches no codex observations when bridge runs outnumber matched steps", () => { + // Two recorded bridge runs but only one command matches the filter — + // ordinals could be shifted, so misattribution must be refused. + const events = [commandExecution("node browser_run.mjs a.js"), commandExecution("ls")]; + const trajectory = codexAdapter.fromHarnessResult( + { + events, + stepObservations: [ + { runIndex: 0, evidence: { url: "https://example.com/a" } }, + { runIndex: 1, evidence: { url: "https://example.com/b" } }, + ], + }, + TASK_SPEC, + ); + expect(trajectory.steps.every((s) => s.probeEvidence.url === undefined)).toBe(true); + }); +}); + +describe("verifiability gate", () => { + afterEach(() => { + delete process.env.EVAL_MAX_UNVERIFIABLE_CRITERIA; + }); + + it("aggregates unverifiable criteria per arm and skips ungraded runs", () => { + const arms = summarizeArmVerifiability( + [ + row("model-a", "stagehand_code", { criterionCount: 4, evidenceInsufficient: ["c1"] }), + row("model-a", "stagehand_code", { criterionCount: 3, evidenceInsufficient: [] }), + row("model-a", "playwright_code", { + criterionCount: 5, + evidenceInsufficient: ["c1", "c2"], + }), + row("model-a", "stagehand_code", {}), + ], + "claude_code", + ); + expect(arms).toEqual([ + { + arm: "claude_code × stagehand_code × model-a", + gradedRuns: 2, + unverifiableCriteria: 1, + totalCriteria: 7, + }, + { + arm: "claude_code × playwright_code × model-a", + gradedRuns: 1, + unverifiableCriteria: 2, + totalCriteria: 5, + }, + ]); + }); + + it("gates arms over the limit; unset env reports only", () => { + expect(resolveUnverifiableCriteriaLimit()).toBeUndefined(); + process.env.EVAL_MAX_UNVERIFIABLE_CRITERIA = "1"; + expect(resolveUnverifiableCriteriaLimit()).toBe(1); + const arms = [ + { arm: "a", gradedRuns: 1, unverifiableCriteria: 1, totalCriteria: 4 }, + { arm: "b", gradedRuns: 1, unverifiableCriteria: 2, totalCriteria: 4 }, + ]; + expect(armsOverLimit(arms, 1).map((a) => a.arm)).toEqual(["b"]); + }); + + it("treats malformed limit values as report-only", () => { + for (const raw of ["1.5", "10foo", "-2", "", " "]) { + process.env.EVAL_MAX_UNVERIFIABLE_CRITERIA = raw; + expect(resolveUnverifiableCriteriaLimit()).toBeUndefined(); + } + process.env.EVAL_MAX_UNVERIFIABLE_CRITERIA = " 3 "; + expect(resolveUnverifiableCriteriaLimit()).toBe(3); + }); +}); + +function assistantToolUse( + id: string, + name: string, + input: Record, +): Record { + return { + type: "assistant", + message: { content: [{ type: "tool_use", id, name, input }] }, + }; +} + +function toolResult(toolUseId: string, text: string): Record { + return { + type: "user", + message: { + content: [{ type: "tool_result", tool_use_id: toolUseId, content: text }], + }, + }; +} + +function commandExecution(command: string): Record { + return { + type: "item.completed", + item: { + type: "command_execution", + command, + aggregated_output: "ok", + exit_code: 0, + status: "completed", + }, + }; +} + +function row( + modelName: string, + toolSurface: string, + output: Record, +): { + input: { name: string; modelName: never; params: Record }; + output: Record; +} { + return { + input: { + name: "task", + modelName: modelName as never, + params: { toolSurface }, + }, + output, + }; +} diff --git a/packages/evals/tui/commands/run.ts b/packages/evals/tui/commands/run.ts index b2cce37ac..32c1d4dac 100644 --- a/packages/evals/tui/commands/run.ts +++ b/packages/evals/tui/commands/run.ts @@ -20,6 +20,11 @@ import type { ResolvedRunOptions } from "./parse.js"; import { withEnvOverrides } from "./parse.js"; import { getRuntimeTasksRoot } from "../../runtimePaths.js"; import { isExecutableBenchHarness, type Harness } from "../../framework/benchTypes.js"; +import { + armsOverLimit, + resolveUnverifiableCriteriaLimit, + summarizeArmVerifiability, +} from "../../framework/verifierGate.js"; type RunProgressEvent = { type: "planned" | "started" | "passed" | "failed" | "error"; @@ -303,6 +308,39 @@ export async function runCommand( printModelSummary(result.results); } + const arms = summarizeArmVerifiability(result.results, options.harness); + const unverifiableLimit = resolveUnverifiableCriteriaLimit(); + if (arms.length > 0) { + for (const arm of arms) { + console.log( + dim( + ` Verifiability: ${arm.arm} — ${arm.unverifiableCriteria}/${arm.totalCriteria} criteria unverifiable across ${arm.gradedRuns} graded runs`, + ), + ); + } + if (unverifiableLimit !== undefined) { + const over = armsOverLimit(arms, unverifiableLimit); + for (const arm of over) { + console.error( + ` ✗ verifiability gate: ${arm.arm} has ${arm.unverifiableCriteria} unverifiable criteria (limit ${unverifiableLimit})`, + ); + } + if (over.length > 0) { + process.exitCode = 1; + } + } + } else if ( + unverifiableLimit !== undefined && + result.results.some((row) => row.output.verifierError !== undefined) + ) { + // A configured gate must never be silently bypassed: verifier-backed + // runs happened, but none produced a graded arm to measure. + console.error( + ` ✗ verifiability gate: EVAL_MAX_UNVERIFIABLE_CRITERIA=${unverifiableLimit} is set but no runs were graded`, + ); + process.exitCode = 1; + } + console.log(dim(` Experiment: ${result.experimentName}`)); console.log(""); } catch (error) {