From 327f36a9b0fe5ae8d0df08e234e65b8364b8779f Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 14 Aug 2026 02:48:20 +0000 Subject: [PATCH 1/3] Add attach(), the handle the agent classes will retire into MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `attach({ browser, client })` binds a Kernel browser to CUA's execution resources and compiles (model, tools) pairs into plain pi objects: the model carrying the transport its tools derive, executables materialized against the handle's pool, a `Models` collection adding provider retry, required headers, the catalog's payload transforms and the tool-result image bound, and an `install(harness)` for the behaviors that are pi event handlers rather than constructor options. The handle is what persists — the Kernel client and browser, the canonical translator, the lazily created raw-CDP executor, ref and frame state — so a spec materializes exactly once across repeat compiles. That is the lifetime a one-shot function would have misrepresented. The agent classes keep their behavior and now share these internals rather than owning private copies, so the two cannot drift while both exist. Tests drive a plain pi `Agent` and a plain pi `AgentHarness` from a handle with no CUA agent class involved, which is the seam the classes will retire into. --- packages/agent/CHANGELOG.md | 15 ++ packages/agent/src/agent.ts | 174 ++------------- packages/agent/src/attach.ts | 331 +++++++++++++++++++++++++++++ packages/agent/src/index.ts | 11 +- packages/agent/test/attach.test.ts | 135 ++++++++++++ 5 files changed, 505 insertions(+), 161 deletions(-) create mode 100644 packages/agent/src/attach.ts create mode 100644 packages/agent/test/attach.test.ts diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index 2025f15..826b5d8 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## 0.15.0 - 2026-08-14 + +- Add `attach({ browser, client })`, returning a handle that compiles + (model, tools) pairs into plain pi objects: the model carrying the transport + its tools derive, executables materialized against the handle's browser pool, + a `Models` collection adding provider retry, required headers, the catalog's + payload transforms and the tool-result image bound, and an `install(harness)` + for the behaviors that are pi event handlers rather than constructor options. + The handle owns what actually persists — the Kernel client and browser, the + translator, the raw-CDP executor, ref and frame state — so a spec materializes + once across repeat compiles. +- `CuaAgent` and `CuaAgentHarness` are unchanged and now share their internals + with `attach()` rather than owning private copies. They are slated to retire + in favor of the handle. + ## 0.14.0 - 2026-08-14 - `CuaAgentHarness` no longer refuses a model ref that is absent from its diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index 5dcf0fd..6e38b5f 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -45,25 +45,25 @@ import { } from "./provider-retry"; import { CuaExecutionResources, type CuaExecutionDetails } from "./resources"; import { CuaToolManager, type CuaAgentTool, type CuaHarnessTool } from "./tool-manager"; +import { + type CuaEmptyResponseRecoveryOptions, + type CuaModelInput, + defaultCuaStream, + hasExecutionError, + isEmptyAssistantResponse, + modelTransportChanged, + projectToolResultImages, + requiredImageToolNames, + resolveEmptyResponseRecovery, + resolveModelFromCollection, + resolveResponseThreading, + resolveToolResultImageReplayLimit, + type ToolResultImageReplayLimit, + turnFailureStopMessage, + withCatalogModels, +} from "./attach"; import type { KernelBrowser } from "./translator/translator"; -/** A registered CUA model reference or an already resolved pi model. */ -export type CuaModelInput = CuaModelRef | Model; - -const DEFAULT_TOOL_RESULT_IMAGE_REPLAY_LIMIT = 4; -const OMITTED_TOOL_RESULT_IMAGES = "[stale tool-result images omitted]"; - -/** Maximum recent tool-result images retained in model context, or `false` to retain all images. Provider-required native tool images are always retained. */ -export type ToolResultImageReplayLimit = number | false; - -/** Optional follow-up policy for otherwise empty successful assistant responses. */ -export interface CuaEmptyResponseRecoveryOptions { - /** User message queued to ask the model to continue. */ - followUp: string; - /** Maximum automatic follow-ups per prompt. */ - maxAttempts: number; -} - /** Mutable conversation state exposed by {@link CuaAgent}. */ export interface CuaAgentState { systemPrompt: string; @@ -493,143 +493,3 @@ export class CuaAgentHarness< await this.tools.resources.dispose(); } } - -const defaultCuaStream: StreamFn = (model, context, options) => cuaModels().streamSimple(model, context, options); - -function resolveModelFromCollection(ref: CuaModelRef, models: Models): Model { - const { provider, model: id } = parseCuaModelRef(ref); - return models.getModel(provider, id) ?? getCuaModel(ref); -} - -/** Whether a tools-only recompile actually changed the model pi streams with, so `setTools()` only pushes `setModel()` (and its session/event side effects) when the derived transport moved. */ -function modelTransportChanged(previous: Model, next: Model): boolean { - return previous.provider !== next.provider || previous.id !== next.id || previous.api !== next.api; -} - -function withCatalogModels( - models: Models, - manager: CuaToolManager, - imageReplayLimit: ToolResultImageReplayLimit, - responseThreading: boolean, -): Models { - const contextFor = (context: Context) => projectModelContext( - context, - imageReplayLimit, - requiredImageToolNames(manager.catalog.incoming), - ); - const optionsFor = (options: T): T => { - const catalog = manager.catalog; - const callerOnPayload = options?.onPayload; - return { - ...options, - headers: catalog.headers.merge(options?.headers), - disableResponseThreading: responseThreading ? undefined : true, - cuaIncomingToolPlan: catalog.incoming, - onPayload: async (payload: unknown, model: Model) => { - const generated = await catalog.payload.apply(payload, model); - return callerOnPayload ? (await callerOnPayload(generated, model)) ?? generated : generated; - }, - } as T; - }; - return { - getProviders: () => models.getProviders(), - getProvider: (id) => models.getProvider(id), - getModels: (provider) => models.getModels(provider), - getModel: (provider, id) => models.getModel(provider, id), - refresh: (provider) => models.refresh(provider), - getAuth: (input, overrides) => models.getAuth(input as never, overrides), - checkAuth: (providerId) => models.checkAuth(providerId), - getAvailable: (providerId) => models.getAvailable(providerId), - login: (providerId, type, interaction) => models.login(providerId, type, interaction), - logout: (providerId) => models.logout(providerId), - stream: (model, context, options) => models.stream(model, contextFor(context), optionsFor(options)), - complete: (model, context, options) => models.complete(model, contextFor(context), optionsFor(options)), - streamSimple: (model, context, options) => models.streamSimple(model, contextFor(context), optionsFor(options)), - completeSimple: (model, context, options) => models.completeSimple(model, contextFor(context), optionsFor(options)), - }; -} - -function resolveToolResultImageReplayLimit(limit: ToolResultImageReplayLimit | undefined): ToolResultImageReplayLimit { - if (limit === undefined) return DEFAULT_TOOL_RESULT_IMAGE_REPLAY_LIMIT; - if (limit !== false && (!Number.isFinite(limit) || !Number.isInteger(limit) || limit < 0)) { - throw new TypeError("toolResultImageReplayLimit must be a finite non-negative integer or false"); - } - return limit; -} - -/** Native computer tool names whose screenshot history the provider protocol requires in full, regardless of the image replay limit. */ -function requiredImageToolNames(incoming: CuaIncomingToolPlan): ReadonlySet { - return new Set(incoming.openaiComputerName ? [incoming.openaiComputerName] : []); -} - -function projectToolResultImages( - messages: TMessage[], - limit: ToolResultImageReplayLimit, - requiredToolNames: ReadonlySet = new Set(), -): TMessage[] { - if (limit === false) return messages; - let imageCount = 0; - for (const message of messages) { - if (message.role === "toolResult" && !requiredToolNames.has(message.toolName)) { - imageCount += message.content.filter((block) => block.type === "image").length; - } - } - if (imageCount <= limit) return messages; - const firstRetainedImage = Math.max(0, imageCount - limit); - let imageOrdinal = 0; - return messages.map((message) => { - if (message.role !== "toolResult" || requiredToolNames.has(message.toolName)) return message; - let changed = false; - let markerInserted = false; - const content = [] as typeof message.content; - for (const block of message.content) { - if (block.type !== "image" || imageOrdinal++ >= firstRetainedImage) { - content.push(block); - continue; - } - changed = true; - if (!markerInserted) { - content.push({ type: "text", text: OMITTED_TOOL_RESULT_IMAGES }); - markerInserted = true; - } - } - return changed ? { ...message, content } as TMessage : message; - }); -} - -function projectModelContext( - context: Context, - imageReplayLimit: ToolResultImageReplayLimit, - requiredToolNames: ReadonlySet, -): Context { - const messages = projectToolResultImages(context.messages, imageReplayLimit, requiredToolNames); - return messages === context.messages ? context : { ...context, messages }; -} - -function resolveEmptyResponseRecovery(options: CuaEmptyResponseRecoveryOptions | undefined): CuaEmptyResponseRecoveryOptions | undefined { - if (!options) return undefined; - if (options.followUp.trim().length === 0) throw new Error("emptyResponseRecovery.followUp must not be blank"); - if (!Number.isInteger(options.maxAttempts) || options.maxAttempts < 0) throw new Error("emptyResponseRecovery.maxAttempts must be a non-negative finite integer"); - return { followUp: options.followUp, maxAttempts: options.maxAttempts }; -} - -function resolveResponseThreading(value: boolean | undefined): boolean { - if (value !== undefined && typeof value !== "boolean") throw new TypeError("responseThreading must be a boolean"); - return value ?? true; -} - -function isEmptyAssistantResponse(message: AgentMessage): boolean { - return message.role === "assistant" && message.stopReason === "stop" && message.content.length === 0; -} - -function hasExecutionError(details: unknown): boolean { - return Boolean(details && typeof details === "object" && (details as CuaExecutionDetails).isError === true); -} - -function turnFailureStopMessage(manager: CuaToolManager): string | undefined { - for (const entry of manager.catalog.entries) { - const execution = manager.specFor(entry.identity)?.execution; - if (execution?.kind === "actions" && execution.stopTurnOnFailureMessage) return execution.stopTurnOnFailureMessage; - } - return undefined; -} diff --git a/packages/agent/src/attach.ts b/packages/agent/src/attach.ts new file mode 100644 index 0000000..e340ed2 --- /dev/null +++ b/packages/agent/src/attach.ts @@ -0,0 +1,331 @@ +import type { + AgentHarness, + AgentHarnessTool, + AgentMessage, + AgentTool, + StreamFn, +} from "@earendil-works/pi-agent-core"; +import { + type Api, + type Context, + cuaModels, + type CuaIncomingToolPlan, + type CuaModelRef, + getCuaModel, + parseCuaModelRef, + type CuaSimpleStreamOptions, + type Model, + type Models, + type SimpleStreamOptions, +} from "@onkernel/cua-ai"; +import type Kernel from "@onkernel/sdk"; +import { resolveProviderRetryPolicy, type CuaRetryOptions, withProviderRetryModels } from "./provider-retry"; +import { CuaExecutionResources, type CuaExecutionDetails } from "./resources"; +import { CuaToolManager, type CuaHarnessTool } from "./tool-manager"; +import type { KernelBrowser } from "./translator/translator"; + +/** A registered CUA model reference or an already resolved pi model. */ +export type CuaModelInput = CuaModelRef | Model; + +const DEFAULT_TOOL_RESULT_IMAGE_REPLAY_LIMIT = 4; +const OMITTED_TOOL_RESULT_IMAGES = "[stale tool-result images omitted]"; + +/** Maximum recent tool-result images retained in model context, or `false` to retain all images. Provider-required native tool images are always retained. */ +export type ToolResultImageReplayLimit = number | false; + +/** Optional follow-up policy for otherwise empty successful assistant responses. */ +export interface CuaEmptyResponseRecoveryOptions { + /** User message queued to ask the model to continue. */ + followUp: string; + /** Maximum automatic follow-ups per prompt. */ + maxAttempts: number; +} + +/** What a Kernel browser handle needs to know to stream and execute. */ +export interface CuaAttachOptions { + browser: KernelBrowser; + client: Kernel; + /** Defaults to the shared {@link cuaModels} collection. */ + models?: Models; + retry?: CuaRetryOptions; + toolResultImageReplayLimit?: ToolResultImageReplayLimit; + responseThreading?: boolean; + emptyResponseRecovery?: CuaEmptyResponseRecoveryOptions; + onPayload?: SimpleStreamOptions["onPayload"]; +} + +/** One compiled (model, tools) pair, ready to hand to pi. */ +export interface CuaCompiled { + /** The model to stream with, carrying the transport its tools derive. */ + readonly model: Model; + /** Executable tools, materialized once against the handle's browser pool. */ + readonly tools: readonly AgentHarnessTool[]; + /** Same tools viewed as pi `AgentTool`s, for the low-level `Agent`. */ + readonly agentTools: readonly AgentTool[]; + /** + * A `Models` collection that adds what CUA owns per request: provider retry, + * required headers, the catalog's payload transforms, and the tool-result + * image bound. + */ + readonly models: Models; + /** + * Register the behaviors that are pi event handlers rather than constructor + * options: marking failed tool results, blocking a turn's remaining calls + * after one fails, and empty-response recovery. Returns an unsubscribe. + */ + install(harness: AgentHarness): () => void; +} + +/** + * A Kernel browser bound to CUA's execution resources. + * + * The handle is what persists: the Kernel client and browser, the canonical + * computer translator, the lazily created raw-CDP executor, element-ref and + * frame state, and screenshot and Playwright capability all outlive any model + * or tool change. `compile()` is called again for each new (model, tools) pair, + * and a spec materializes exactly once per handle, so repeat compiles keep tool + * identity stable. + */ +export interface CuaBrowserHandle { + compile(options: { + model: CuaModelInput; + tools: readonly CuaHarnessTool[]; + }): CuaCompiled; + /** The shared execution pool, for callers that need it directly. */ + readonly resources: CuaExecutionResources; + dispose(): Promise; +} + +/** + * Bind a Kernel browser to CUA's execution resources and return a handle that + * compiles (model, tools) pairs into plain pi objects. + * + * ```ts + * const kb = attach({ browser, client }); + * const { model, tools, models } = kb.compile({ model: "openai:gpt-5.6-sol", tools: [...] }); + * const harness = new AgentHarness({ model, tools, models, activeToolNames: tools.map((t) => t.name), session }); + * ``` + */ +export function attach(options: CuaAttachOptions): CuaBrowserHandle { + const resources = new CuaExecutionResources({ browser: options.browser, client: options.client }); + const imageReplayLimit = resolveToolResultImageReplayLimit(options.toolResultImageReplayLimit); + const useResponseThreading = resolveResponseThreading(options.responseThreading); + const recovery = resolveEmptyResponseRecovery(options.emptyResponseRecovery); + const retrying = withProviderRetryModels(options.models ?? cuaModels(), resolveProviderRetryPolicy(options.retry)); + + return { + resources, + dispose: () => resources.dispose(), + compile(request: { + model: CuaModelInput; + tools: readonly CuaHarnessTool[]; + }): CuaCompiled { + const manager = new CuaToolManager>( + resources, + request.model, + request.tools, + (ref) => resolveModelFromCollection(ref, retrying), + ); + const models = withCatalogModels(retrying, manager, imageReplayLimit, useResponseThreading, options.onPayload); + return { + model: manager.catalog.model, + tools: manager.harnessTools() as readonly AgentHarnessTool[], + agentTools: manager.agentTools(), + models, + install: (harness) => installCuaBehaviors(harness, manager, recovery), + }; + }, + }; +} + +/** + * Wire the pi event handlers CUA owns. Kept separate from `compile()` because + * they are handlers on a constructed harness, not constructor options. + */ +export function installCuaBehaviors( + harness: AgentHarness, + manager: CuaToolManager, + recovery: CuaEmptyResponseRecoveryOptions | undefined, +): () => void { + let turnFailed = false; + let hasPendingQueue = false; + let recoveryAttempts = 0; + const offs: Array<() => void> = []; + + offs.push(harness.on("tool_result", (event: { details?: unknown }) => (hasExecutionError(event.details) ? { isError: true } : undefined))); + offs.push(harness.on("tool_call", () => + turnFailed && turnFailureStopMessage(manager) ? { block: true, reason: turnFailureStopMessage(manager) } : undefined)); + offs.push(harness.on("before_agent_start", () => { + turnFailed = false; + recoveryAttempts = 0; + hasPendingQueue = false; + return undefined; + })); + offs.push(harness.subscribe((event: any, signal?: AbortSignal) => { + if (event.type === "message_end" && event.message.role === "assistant") turnFailed = false; + else if (event.type === "tool_execution_end" && event.isError) turnFailed = true; + else if (event.type === "queue_update") hasPendingQueue = event.steer.length > 0 || event.followUp.length > 0; + if (!recovery || recovery.maxAttempts <= 0) return; + if (event.type !== "turn_end" || !isEmptyAssistantResponse(event.message)) return; + if (signal?.aborted || recoveryAttempts >= recovery.maxAttempts || hasPendingQueue) return; + recoveryAttempts += 1; + return harness.followUp(recovery.followUp); + })); + + return () => { + for (const off of offs) off(); + }; +} + +/** @internal shared with the agent classes until they retire. */ +export const defaultCuaStream: StreamFn = (model, context, options) => cuaModels().streamSimple(model, context, options); + +/** @internal shared with the agent classes until they retire. */ +export function resolveModelFromCollection(ref: CuaModelRef, models: Models): Model { + const { provider, model: id } = parseCuaModelRef(ref); + return models.getModel(provider, id) ?? getCuaModel(ref); +} + +/** Whether a tools-only recompile actually changed the model pi streams with, so `setTools()` only pushes `setModel()` (and its session/event side effects) when the derived transport moved. */ +/** @internal shared with the agent classes until they retire. */ +export function modelTransportChanged(previous: Model, next: Model): boolean { + return previous.provider !== next.provider || previous.id !== next.id || previous.api !== next.api; +} + +/** @internal shared with the agent classes until they retire. */ +export function withCatalogModels( + models: Models, + manager: CuaToolManager, + imageReplayLimit: ToolResultImageReplayLimit, + responseThreading: boolean, + handleOnPayload?: SimpleStreamOptions["onPayload"], +): Models { + const contextFor = (context: Context) => projectModelContext( + context, + imageReplayLimit, + requiredImageToolNames(manager.catalog.incoming), + ); + const optionsFor = (options: T): T => { + const catalog = manager.catalog; + const callerOnPayload = options?.onPayload ?? handleOnPayload; + return { + ...options, + headers: catalog.headers.merge(options?.headers), + disableResponseThreading: responseThreading ? undefined : true, + cuaIncomingToolPlan: catalog.incoming, + onPayload: async (payload: unknown, model: Model) => { + const generated = await catalog.payload.apply(payload, model); + return callerOnPayload ? (await callerOnPayload(generated, model)) ?? generated : generated; + }, + } as T; + }; + return { + getProviders: () => models.getProviders(), + getProvider: (id) => models.getProvider(id), + getModels: (provider) => models.getModels(provider), + getModel: (provider, id) => models.getModel(provider, id), + refresh: (provider) => models.refresh(provider), + getAuth: (input, overrides) => models.getAuth(input as never, overrides), + checkAuth: (providerId) => models.checkAuth(providerId), + getAvailable: (providerId) => models.getAvailable(providerId), + login: (providerId, type, interaction) => models.login(providerId, type, interaction), + logout: (providerId) => models.logout(providerId), + stream: (model, context, options) => models.stream(model, contextFor(context), optionsFor(options)), + complete: (model, context, options) => models.complete(model, contextFor(context), optionsFor(options)), + streamSimple: (model, context, options) => models.streamSimple(model, contextFor(context), optionsFor(options)), + completeSimple: (model, context, options) => models.completeSimple(model, contextFor(context), optionsFor(options)), + }; +} + +/** @internal shared with the agent classes until they retire. */ +export function resolveToolResultImageReplayLimit(limit: ToolResultImageReplayLimit | undefined): ToolResultImageReplayLimit { + if (limit === undefined) return DEFAULT_TOOL_RESULT_IMAGE_REPLAY_LIMIT; + if (limit !== false && (!Number.isFinite(limit) || !Number.isInteger(limit) || limit < 0)) { + throw new TypeError("toolResultImageReplayLimit must be a finite non-negative integer or false"); + } + return limit; +} + +/** Native computer tool names whose screenshot history the provider protocol requires in full, regardless of the image replay limit. */ +/** @internal shared with the agent classes until they retire. */ +export function requiredImageToolNames(incoming: CuaIncomingToolPlan): ReadonlySet { + return new Set(incoming.openaiComputerName ? [incoming.openaiComputerName] : []); +} + +/** @internal shared with the agent classes until they retire. */ +export function projectToolResultImages( + messages: TMessage[], + limit: ToolResultImageReplayLimit, + requiredToolNames: ReadonlySet = new Set(), +): TMessage[] { + if (limit === false) return messages; + let imageCount = 0; + for (const message of messages) { + if (message.role === "toolResult" && !requiredToolNames.has(message.toolName)) { + imageCount += message.content.filter((block) => block.type === "image").length; + } + } + if (imageCount <= limit) return messages; + const firstRetainedImage = Math.max(0, imageCount - limit); + let imageOrdinal = 0; + return messages.map((message) => { + if (message.role !== "toolResult" || requiredToolNames.has(message.toolName)) return message; + let changed = false; + let markerInserted = false; + const content = [] as typeof message.content; + for (const block of message.content) { + if (block.type !== "image" || imageOrdinal++ >= firstRetainedImage) { + content.push(block); + continue; + } + changed = true; + if (!markerInserted) { + content.push({ type: "text", text: OMITTED_TOOL_RESULT_IMAGES }); + markerInserted = true; + } + } + return changed ? { ...message, content } as TMessage : message; + }); +} + +function projectModelContext( + context: Context, + imageReplayLimit: ToolResultImageReplayLimit, + requiredToolNames: ReadonlySet, +): Context { + const messages = projectToolResultImages(context.messages, imageReplayLimit, requiredToolNames); + return messages === context.messages ? context : { ...context, messages }; +} + +/** @internal shared with the agent classes until they retire. */ +export function resolveEmptyResponseRecovery(options: CuaEmptyResponseRecoveryOptions | undefined): CuaEmptyResponseRecoveryOptions | undefined { + if (!options) return undefined; + if (options.followUp.trim().length === 0) throw new Error("emptyResponseRecovery.followUp must not be blank"); + if (!Number.isInteger(options.maxAttempts) || options.maxAttempts < 0) throw new Error("emptyResponseRecovery.maxAttempts must be a non-negative finite integer"); + return { followUp: options.followUp, maxAttempts: options.maxAttempts }; +} + +/** @internal shared with the agent classes until they retire. */ +export function resolveResponseThreading(value: boolean | undefined): boolean { + if (value !== undefined && typeof value !== "boolean") throw new TypeError("responseThreading must be a boolean"); + return value ?? true; +} + +/** @internal shared with the agent classes until they retire. */ +export function isEmptyAssistantResponse(message: AgentMessage): boolean { + return message.role === "assistant" && message.stopReason === "stop" && message.content.length === 0; +} + +/** @internal shared with the agent classes until they retire. */ +export function hasExecutionError(details: unknown): boolean { + return Boolean(details && typeof details === "object" && (details as CuaExecutionDetails).isError === true); +} + +/** @internal shared with the agent classes until they retire. */ +export function turnFailureStopMessage(manager: CuaToolManager): string | undefined { + for (const entry of manager.catalog.entries) { + const execution = manager.specFor(entry.identity)?.execution; + if (execution?.kind === "actions" && execution.stopTurnOnFailureMessage) return execution.stopTurnOnFailureMessage; + } + return undefined; +} diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index c96d9fd..d890993 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -32,11 +32,14 @@ export type { BrowserWaitReason, } from "./translator/types"; export { CuaAgent, CuaAgentHarness } from "./agent"; +export { attach } from "./attach"; export type { - CuaAgentHarnessOptions, - CuaAgentOptions, - CuaAgentState, + CuaAttachOptions, + CuaBrowserHandle, + CuaCompiled, CuaEmptyResponseRecoveryOptions, + CuaModelInput, ToolResultImageReplayLimit, -} from "./agent"; +} from "./attach"; +export type { CuaAgentHarnessOptions, CuaAgentOptions, CuaAgentState } from "./agent"; export type { CuaRetryOptions } from "./provider-retry"; diff --git a/packages/agent/test/attach.test.ts b/packages/agent/test/attach.test.ts new file mode 100644 index 0000000..faf2c61 --- /dev/null +++ b/packages/agent/test/attach.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from "vitest"; +import { + createAssistantMessageEventStream, + createCuaModels, + cua, + GOOGLE_CUA_INTERACTIONS_API, + type AssistantMessage, + type Context, + type Model, +} from "@onkernel/cua-ai"; +import type Kernel from "@onkernel/sdk"; +import { Agent, AgentHarness, attach, InMemorySessionRepo, type KernelBrowser, type StreamFn } from "../src/index"; + +const browser = { session_id: "browser_123", viewport: { width: 1440, height: 900 } } as KernelBrowser; +const client = {} as Kernel; + +function assistant(model: Model): AssistantMessage { + return { + role: "assistant", + content: [], + api: model.api, + provider: model.provider, + model: model.id, + usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + stopReason: "stop", + timestamp: Date.now(), + }; +} + +function recordingStream(seen: { model: Model; context: Context }[]): StreamFn { + return (model, context) => { + seen.push({ model, context: { ...context, tools: context.tools?.slice() } }); + const stream = createAssistantMessageEventStream(); + const message = assistant(model); + stream.push({ type: "start", partial: message }); + stream.push({ type: "done", reason: "stop", message }); + stream.end(message); + return stream; + }; +} + +describe("attach", () => { + it("compiles a (model, tools) pair into plain pi objects", () => { + const handle = attach({ browser, client }); + const compiled = handle.compile({ + model: "openai:gpt-5.5", + tools: [cua.tools.browser.snapshot(), cua.tools.browser.click()], + }); + + expect(compiled.model.api).toBe("openai-responses"); + expect(compiled.tools.map((tool) => tool.name)).toEqual(["browser_snapshot", "browser_click"]); + expect(compiled.agentTools.every((tool) => typeof tool.execute === "function")).toBe(true); + expect(typeof compiled.models.streamSimple).toBe("function"); + }); + + it("derives the transport from the selected tools", () => { + const handle = attach({ browser, client }); + const cdp = handle.compile({ model: "google:gemini-3.6-flash", tools: [cua.tools.browser.snapshot()] }); + const native = handle.compile({ model: "google:gemini-3.6-flash", tools: cua.providers.google.toolsets.browser() }); + + expect(cdp.model.api).toBe("google-generative-ai"); + expect(native.model.api).toBe(GOOGLE_CUA_INTERACTIONS_API); + }); + + it("materializes a spec once per handle, across compiles", () => { + const handle = attach({ browser, client }); + const snapshot = cua.tools.browser.snapshot(); + handle.compile({ model: "openai:gpt-5.5", tools: [snapshot] }); + handle.compile({ model: "openai:gpt-5.5", tools: [snapshot, cua.tools.browser.click()] }); + + // The executable is cached per pool and per spec. Each compile wraps it to + // install the execution scope, so the wrapper differs while the tool + // underneath — and the implementation identity pi keys cache decisions on — + // stays stable. + expect(handle.resources.materialize(snapshot)).toBe(handle.resources.materialize(snapshot)); + }); + + it("drives a plain pi Agent with no CUA agent class", async () => { + const seen: { model: Model; context: Context }[] = []; + const handle = attach({ browser, client }); + const compiled = handle.compile({ model: "openai:gpt-5.5", tools: [cua.tools.browser.snapshot()] }); + + const agent = new Agent({ + streamFn: recordingStream(seen), + initialState: { model: compiled.model, tools: [...compiled.agentTools], systemPrompt: "" }, + }); + await agent.prompt("go"); + + expect(seen).toHaveLength(1); + expect(seen[0]!.model.api).toBe("openai-responses"); + expect(seen[0]!.context.tools?.map((tool) => tool.name)).toEqual(["browser_snapshot"]); + }); + + it("drives a plain pi AgentHarness, with CUA's behaviors installed", async () => { + const seen: { model: Model; context: Context }[] = []; + const handle = attach({ browser, client, models: modelsFromStream(recordingStream(seen)) }); + const compiled = handle.compile({ model: "openai:gpt-5.5", tools: [cua.tools.browser.snapshot()] }); + const session = await new InMemorySessionRepo().create(); + + const harness = new AgentHarness({ + session, + model: compiled.model, + models: compiled.models, + tools: [...compiled.tools], + activeToolNames: compiled.tools.map((tool) => tool.name), + } as never); + const uninstall = compiled.install(harness); + + await harness.prompt("go"); + expect(seen).toHaveLength(1); + expect(seen[0]!.context.tools?.map((tool) => tool.name)).toEqual(["browser_snapshot"]); + expect(typeof uninstall).toBe("function"); + uninstall(); + }); + + it("disposes the shared execution pool once, not per compile", async () => { + const handle = attach({ browser, client }); + handle.compile({ model: "openai:gpt-5.5", tools: [] }); + handle.compile({ model: "anthropic:claude-opus-5", tools: [] }); + await expect(handle.dispose()).resolves.toBeUndefined(); + }); +}); + +function modelsFromStream(streamFn: StreamFn) { + const models = createCuaModels(); + models.setProvider({ + id: "openai", + name: "scripted", + auth: { apiKey: { name: "test", resolve: async () => ({ auth: { apiKey: "test" } }) } }, + getModels: () => [], + stream: streamFn, + streamSimple: streamFn, + } as never); + return models; +} From c025d401d7f402b33e9dee253e72850b7eda0cd6 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 14 Aug 2026 03:02:39 +0000 Subject: [PATCH 2/3] Collapse per-merge changelog headings into Unreleased Nothing above cua-ai/cua-agent 0.10.0 and cua-cli 0.9.0 has been published, so the version headings above them claimed releases that never happened. Merging the sections exposed entries that contradicted each other or described states that never shipped, so each Unreleased section now reads as the net change from the last released version. --- .agents/skills/release/SKILL.md | 21 +++- packages/agent/CHANGELOG.md | 79 ++++++--------- packages/ai/CHANGELOG.md | 172 ++++++++++++-------------------- packages/cli/CHANGELOG.md | 63 ++++-------- 4 files changed, 135 insertions(+), 200 deletions(-) diff --git a/.agents/skills/release/SKILL.md b/.agents/skills/release/SKILL.md index 1598729..8d2f9c0 100644 --- a/.agents/skills/release/SKILL.md +++ b/.agents/skills/release/SKILL.md @@ -93,25 +93,38 @@ and every version returned by `npm view versions --json`. ## Changelog -Update only the changelog for packages being released: +Changes land under a `## Unreleased` heading as they merge, so every changelog +has at most one unreleased section: - `packages/ai/CHANGELOG.md` - `packages/agent/CHANGELOG.md` - `packages/cli/CHANGELOG.md` -Add a new top entry: +Releasing renames that heading in place — do not add a second top entry: ```markdown ## - YYYY-MM-DD - -- ... ``` +Then read the section as a whole before tagging. It accumulated over several +merges, so it can carry entries that contradict each other or describe a state +that never shipped: an API added and then removed, or a note that a provider +"keeps" a behavior when a later entry deletes that provider. Consumers upgrade +from the previous release, not through the intermediate steps, so collapse +those into the net change and drop what nobody can observe. Cross-package +"update `@onkernel/cua-ai` to X" notes belong here too — the version is not +known until this step. + Write customer-facing changes. Do not dump commit subjects, internal issue names, Slack context, or vague entries like "misc improvements." Group details only when it improves readability. If the release is only metadata or docs, say that plainly. +Merges between releases add to `## Unreleased`, creating it directly under +`# Changelog` when it is absent. Never invent a version heading for a merge: +package versions are chosen at release time from the accumulated changes, and a +per-merge heading claims a release that never happened. + ## Edit Release Metadata Set versions explicitly: diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index 826b5d8..188fc2d 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 0.15.0 - 2026-08-14 +## Unreleased - Add `attach({ browser, client })`, returning a handle that compiles (model, tools) pairs into plain pi objects: the model carrying the transport @@ -14,61 +14,42 @@ - `CuaAgent` and `CuaAgentHarness` are unchanged and now share their internals with `attach()` rather than owning private copies. They are slated to retire in favor of the handle. - -## 0.14.0 - 2026-08-14 - -- `CuaAgentHarness` no longer refuses a model ref that is absent from its - supplied `Models` collection: it falls back to the registry, and an id the - registry lacks is synthesized. Update `@onkernel/cua-ai` to 0.14.0. - -## 0.13.0 - 2026-08-13 - -Breaking: Tzafon and Yutori support is removed. - -- Update `@onkernel/cua-ai` to 0.13.0. Constructing a `CuaAgent` or - `CuaAgentHarness` with a Tzafon or Yutori model ref now fails to resolve the - model, and `cua.providers.tzafon` / `cua.providers.yutori` no longer exist. -- The tool-result image replay limit now exempts only OpenAI's native computer - tool, whose protocol requires every `computer_call_output` to carry a - screenshot. Tzafon's native computer results were exempt for the same reason - and are gone with the provider. -- Remove `CuaExecutionResources.viewport`. It only fed the removed catalog - viewport option; the same value is still on `resources.browser.viewport`. - -## 0.12.0 - 2026-08-13 - - Add `CuaAgentHarness.setModelAndTools()`. A model switch that also swaps interaction tools has to compile as one pair now that the selected tools decide the transport: staging the two in sequence produces an intermediate - catalog whose derived transport differs from both the old and the new one, - and records a model change for a transport nothing ever streamed with. - -- Update `@onkernel/cua-ai` to 0.12.0. The model streamed for a Google model - now depends on which tools `CuaAgent`/`CuaAgentHarness` were constructed or - mutated with: selecting Google's native browser toolset still compiles to - the CUA-owned Interactions API, but a Google model selected with only CDP - browser tools now streams through pi's builtin Google transport instead of - always carrying the CUA-owned api. This applies uniformly across - construction, `setTools()`, and `setModel()`, since all three feed the same - compiled `catalog.model` into pi. + catalog whose derived transport differs from both the old and the new one, and + records a model change for a transport nothing ever streamed with. +- `CuaAgentHarness` no longer refuses a model ref that is absent from its + supplied `Models` collection: it falls back to the registry, and an id the + registry lacks is synthesized. +- The model streamed for a Google model now depends on which tools `CuaAgent` / + `CuaAgentHarness` were constructed or mutated with: selecting Google's native + browser toolset still compiles to the CUA-owned Interactions API, but a Google + model selected with only CDP browser tools now streams through pi's builtin + Google transport instead of always carrying the CUA-owned api. This applies + uniformly across construction, `setTools()`, and `setModel()`, since all three + feed the same compiled `catalog.model` into pi. - Fix `setTools()` recompiling from the previously *compiled* model instead of the caller's model selection: dropping a native toolset that had derived a - tool-selection-dependent api (e.g. Google's Interactions API) left - subsequent tools-only recompiles stuck on that api even though the new - selection no longer required it. `CuaToolManager` now recompiles tools-only - changes from the model input the caller last selected. - -## 0.11.0 - 2026-08-13 - + tool-selection-dependent api (e.g. Google's Interactions API) left subsequent + tools-only recompiles stuck on that api even though the new selection no + longer required it. `CuaToolManager` now recompiles tools-only changes from + the model input the caller last selected. - `responseThreading` (`CuaAgentOptions`/`CuaAgentHarnessOptions`) no longer affects OpenAI models: OpenAI now streams through pi-ai's builtin Responses - transport and its automatic prompt caching regardless of this flag. The - option still governs Google and Tzafon's `previous_response_id`-style - continuation. -- Exempt OpenAI's native computer tool from the tool-result image replay - limit, alongside Tzafon: its `computer_call_output` items must each carry a - screenshot, and stateless replay no longer leaves them in provider-stored - state. + transport and its automatic prompt caching regardless of this flag. The option + still governs Google's `previous_response_id`-style continuation. +- Exempt OpenAI's native computer tool from the tool-result image replay limit. + Its `computer_call_output` items must each carry a screenshot, and stateless + replay no longer leaves them in provider-stored state. + +Breaking: Tzafon and Yutori support is removed. + +- Constructing a `CuaAgent` or `CuaAgentHarness` with a Tzafon or Yutori model + ref now fails to resolve the model, and `cua.providers.tzafon` / + `cua.providers.yutori` no longer exist. +- Remove `CuaExecutionResources.viewport`. It only fed the removed catalog + viewport option; the same value is still on `resources.browser.viewport`. ## 0.10.0 - 2026-08-04 diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 64c0d79..8510796 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 0.15.0 - 2026-08-14 +## Unreleased - Add `cuaToolMenu(model, selected)`: every tool CUA can offer for a model, each marked available or not with the compiler's own reason when it is not. It @@ -10,8 +10,6 @@ selection, because two providers' native surfaces cannot coexist and a native surface pins the transport. -## 0.14.0 - 2026-08-14 - Breaking: the model allowlist is removed. - `listCuaModels()` returns pi-ai's whole catalog — 37 providers, ~1,150 models — @@ -34,121 +32,83 @@ Breaking: the model allowlist is removed. `cuaNativeSurfaces(model)` and `cuaModelQuirks(model)` are exported for menus and diagnostics. -## 0.13.0 - 2026-08-13 - -- Remove the Meta provider. pi-ai ships no `meta` provider, so cua hand-wrote a - model entry and pointed pi's own `openai-responses` transport at - `api.meta.ai`. Meta was the last user of the model-override mechanism, so - `CUA_MODEL_OVERRIDES`, `cuaOverrideModels()`, and `META_API_KEY` go with it, - and `getCuaModel` no longer has a "supported but not registered" fallback. - Muse Spark remains available through pi's OpenRouter catalog as - `openrouter:meta/muse-spark-1.1`, annotated with explicit capabilities - because OpenRouter's provider-level defaults are conservative. - -- Remove `routeCuaApi`. With Tzafon and Yutori gone it routed no transport at - all, and its last branch only patched grok-4.5's thinking-level map, price - tiers, and compat flags onto pi-ai's registry entry. Live runs on grok-4.5 - at the default, `off`, and `xhigh` thinking levels behave identically - without it, so model resolution now returns pi-ai's data unmodified. The - only lost detail is a >200k-token price tier that pi's registry does not - carry, which affects `usage.cost` reporting for long requests and nothing - else. - -Breaking: Tzafon and Yutori support is removed. - -- Remove the `tzafon` and `yutori` providers: their `CuaProvider` members, - model annotations and overrides, `cua.providers.tzafon`, - `cua.providers.yutori`, `TZAFON_API_KEY`/`YUTORI_API_KEY`, and the exported - `TZAFON_RESPONSES_API`, `YUTORI_CHAT_COMPLETIONS_API`, - `streamTzafonResponses`, `streamSimpleTzafonResponses`, `streamYutori`, and - `streamSimpleYutori` stream functions. `createCuaModels()` no longer - registers either provider, and refs like `tzafon:tzafon.northstar-cua-fast` - or `yutori:n1.5-latest` now fail to resolve. -- `CuaProviderBinding` loses its `tzafon-native` and `yutori-native` variants, - and `CuaIncomingToolPlan` loses `tzafonComputerName` and `yutoriNames`. The - Yutori-only rule rejecting a partial n1 native action set is gone with them; - every surviving native toolset can be selected in part. -- `CompileCuaToolCatalogOptions.viewport` is removed. It existed only to fill - Tzafon's `display_width`/`display_height` declaration defaults, and no - surviving declaration reads it. -- `routeCuaApi` no longer routes any transport. Every remaining provider gets - its transport either from pi-ai's registry or from the selected tools' - `requiresApi`; what is left is grok-4.5's cost/compat/thinking-level - overrides. -- Drop the `@tzafon/lightcone` dependency. - -## 0.12.0 - 2026-08-13 - -- Route OpenAI's native computer adapter through its own stream function, - `streamOpenAICuaComputer`, selected by `model.api`. `streamOpenAIResponses` - no longer inspects the incoming tool plan to decide which adapter runs: the - compiled api id is the only dispatch key, and the provider wrapper is the - only place that reads it. -- Remove the unreferenced Yutori n1.5 expanded action declarations - (`YUTORI_N15_EXPANDED_ACTION_TYPES`, `YUTORI_N15_EXPANDED_TOOL_SET`, - `YUTORI_N15_ACTION_TYPES`) and the canonical-action type aliases nothing - consumed. None were exported from the package root. The expanded set was - scaffolding for a ref/DOM execution path that does not exist. - -Breaking: Google's api id is now derived from selected tools, not stamped on -every Google model. +Breaking: a model's transport is derived from the tools selected with it, rather +than stamped on the model. - `compileCuaToolCatalog` derives the compiled model's `api` from the selected tools' provider bindings: a `CuaProviderBinding` may declare `requiresApi`, - and the returned `catalog.model` carries that transport. Selecting tools - whose bindings require different transports fails to compile with a named - catalog error. This makes transport a function of `(model, selected tools)` - instead of `(model)` alone. + and the returned `catalog.model` carries that transport. Selecting tools whose + bindings require different transports fails to compile with a named catalog + error. This makes transport a function of `(model, selected tools)` instead of + `(model)` alone. - `getCuaModel("google:...")` no longer forces `google-cua-interactions`. A Google model resolved without Google's native browser toolset selected now keeps pi-ai's builtin `google-generative-ai` transport; selecting `cua.providers.google.toolsets.browser()` still compiles to - `google-cua-interactions` as before. `routeCuaApi` no longer touches Google - at all — it's now scoped to genuinely model-shaped routing (Tzafon and - Yutori, which pi ships no transport for at all, and grok-4.5's cost/compat - overrides). -- Add `OPENAI_CUA_COMPUTER_API` (`"openai-cua-computer"`). A model compiled - with `cua.providers.openai.tools.computer()` selected now carries this api; - the OpenAI provider wrapper dispatches to the CUA adapter on `model.api` - alone for that case. The one remaining request-shape check, - `requiresCuaOpenAINamespaceAdapter` (renamed from `requiresCuaOpenAIAdapter`, - which also tested for the native computer tool), covers only the case that - cannot be derived from the model: a transcript carrying a deferred - tool-search addition or a replayed function-call namespace, which pi-ai's - builtin transport does not round-trip. - -## 0.11.0 - 2026-08-13 + `google-cua-interactions` as before. +- Add `OPENAI_CUA_COMPUTER_API` (`"openai-cua-computer"`). A model compiled with + `cua.providers.openai.tools.computer()` selected carries this api, and the + OpenAI provider wrapper dispatches to the CUA adapter on `model.api` alone. + The one remaining request-shape check, `requiresCuaOpenAINamespaceAdapter`, + covers only what cannot be derived from the model: a transcript carrying a + deferred tool-search addition or a replayed function-call namespace, neither + of which pi-ai's builtin transport round-trips. +- Remove `routeCuaApi`. Every provider now takes its transport from pi-ai's + registry or from the selected tools' `requiresApi`, so model resolution + returns pi-ai's data unmodified. Its last remaining branch patched grok-4.5's + thinking-level map, price tiers, and compat flags onto pi-ai's registry entry; + live runs on grok-4.5 at the default, `off`, and `xhigh` thinking levels + behave identically without it. The only lost detail is a >200k-token price + tier that pi's registry does not carry, which affects `usage.cost` reporting + for long requests and nothing else. Breaking: OpenAI models no longer carry a CUA-owned api id. -- OpenAI models now resolve to pi-ai's builtin `"openai-responses"` api - instead of the removed `openai-cua-responses`, and stream through pi's - builtin Responses transport (`store: false`, automatic prompt-cache-key - matching) by default. `OPENAI_CUA_RESPONSES_API` and the OpenAI adapter's - `previous_response_id` threading are removed; `previous_response_id` and - `store: true` no longer appear on any OpenAI request. -- The CUA-owned OpenAI adapter is retained but now dispatches on request - shape rather than a rerouted api id: it only intercepts requests that select - OpenAI's native computer tool, or whose transcript carries a deferred - tool-search addition or a replayed function-call namespace (pi-ai 0.83.0's - builtin transport does not round-trip either). +- OpenAI models resolve to pi-ai's builtin `"openai-responses"` api instead of + the removed `openai-cua-responses`, and stream through pi's builtin Responses + transport (`store: false`, automatic prompt-cache-key matching) by default. + `OPENAI_CUA_RESPONSES_API` and the OpenAI adapter's `previous_response_id` + threading are removed; `previous_response_id` and `store: true` no longer + appear on any OpenAI request. - OpenAI's native computer adapter now sends the same `prompt_cache_key`, - `prompt_cache_retention`, `prompt_cache_options`, and session-affinity - headers as the function-tool path. It previously relied on stored response - state for context reuse and sent no cache key of its own. -- Remove the xAI and Meta Responses forks. Both existed only to thread + `prompt_cache_retention`, `prompt_cache_options`, and session-affinity headers + as the function-tool path. It previously relied on stored response state for + context reuse and sent no cache key of its own. +- Remove the xAI Responses fork. It existed only to thread `previous_response_id` and to set `parallel_tool_calls: false`, which the tool - catalog already emits for those providers. `xai-cua-responses` and - `meta-responses` are gone: Grok streams through pi's builtin xAI provider, and - Meta registers pi's builtin Responses transport against its own base URL and - credentials. `XAI_CUA_RESPONSES_API`, `META_RESPONSES_API`, - `streamXaiResponses`, `streamSimpleXaiResponses`, `streamMetaResponses`, and - `streamSimpleMetaResponses` are no longer exported. Meta also regains pi's - stateless encrypted-reasoning replay, which the fork deleted because it relied - on stored response state. -- Google and Tzafon keep their continuation protocols: each threads a - provider-specific field with no builtin equivalent, and the shared helpers in - `providers/common.ts` are unchanged for them. + catalog already emits for the provider. `xai-cua-responses` is gone and Grok + streams through pi's builtin xAI provider; `XAI_CUA_RESPONSES_API`, + `streamXaiResponses`, and `streamSimpleXaiResponses` are no longer exported. +- Google keeps its continuation protocol: it threads a provider-specific field + with no builtin equivalent, and the shared helpers in `providers/common.ts` + are unchanged for it. + +Breaking: the Tzafon, Yutori, and Meta providers are removed. + +- Remove the `tzafon` and `yutori` providers: their model annotations and + overrides, `cua.providers.tzafon`, `cua.providers.yutori`, + `TZAFON_API_KEY`/`YUTORI_API_KEY`, and the exported `TZAFON_RESPONSES_API`, + `YUTORI_CHAT_COMPLETIONS_API`, `streamTzafonResponses`, + `streamSimpleTzafonResponses`, `streamYutori`, and `streamSimpleYutori` stream + functions. `createCuaModels()` no longer registers either provider, and refs + like `tzafon:tzafon.northstar-cua-fast` or `yutori:n1.5-latest` now fail to + resolve. Drops the `@tzafon/lightcone` dependency. +- Remove the `meta` provider. pi-ai ships no `meta` provider, so cua hand-wrote + a model entry and pointed pi's own Responses transport at `api.meta.ai`. + `META_API_KEY`, `META_RESPONSES_API`, `streamMetaResponses`, and + `streamSimpleMetaResponses` go with it. Meta was the last user of the + model-override mechanism, so `CUA_MODEL_OVERRIDES` and `cuaOverrideModels()` + are gone too, and `getCuaModel` no longer has a "supported but not registered" + fallback. Muse Spark remains available through pi's OpenRouter catalog as + `openrouter:meta/muse-spark-1.1`, annotated with explicit capabilities because + OpenRouter's provider-level defaults are conservative. +- `CuaProviderBinding` loses its `tzafon-native` and `yutori-native` variants, + and `CuaIncomingToolPlan` loses `tzafonComputerName` and `yutoriNames`. The + Yutori-only rule rejecting a partial n1 native action set is gone with them; + every surviving native toolset can be selected in part. +- `CompileCuaToolCatalogOptions.viewport` is removed. It existed only to fill + Tzafon's `display_width`/`display_height` declaration defaults, and no + surviving declaration reads it. ## 0.10.0 - 2026-08-04 diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 004b652..a87266a 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 0.14.0 - 2026-08-14 +## Unreleased - `/tools` now offers the model's whole tool menu instead of filtering the list the CLI composed. Tools the CLI did not choose — `playwright_execute`, the @@ -9,60 +9,41 @@ the selection is staged. `ctrl+r` still restores the model's defaults. - Add `cua tools`, which prints that menu for a model without the TUI, with `--json` for scripting. - -## 0.13.0 - 2026-08-14 - - `cua models` lists every model pi-ai carries, not a curated subset, and `-p` accepts any provider it carries. - `-m` accepts any model id. A bare id that several providers carry now resolves to the first-party provider rather than erroring, since gateways resell the same ids; pass a qualified `provider:model` ref to reach a specific one. -- The API-key preflight now runs only for providers CUA documents variable - names for. Any other pi-ai provider is still selectable; pi resolves its - credential when it streams, and failing up front would refuse a model that - works. +- The API-key preflight now runs only for providers CUA documents variable names + for. Any other pi-ai provider is still selectable; pi resolves its credential + when it streams, and failing up front would refuse a model that works. - The default interaction toolset is chosen from the model rather than its provider: a model with a native browser surface gets it, and everything else gets CUA's CDP tools, with `browser_act` included only where the model accepts - its schema. - -## 0.12.0 - 2026-08-13 - + its schema. A provider can front several model families — Kimi K3 rejects + `browser_act`'s schema while Muse Spark accepts it — so one answer per + provider was never right. +- `--print -o jsonl` schema bumps to version 2: every assistant message now also + emits an `assistant_usage` event (`turn`, `model`, `api`, `input`, `output`, + `cache_read`, `cache_write`, `reasoning`, `total_tokens`, and a derived + `cache_hit_ratio`), including tool-only turns with no text. +- `defaultInteractionTools` still selects Google's native browser toolset, so + `assistant_usage.api` for a Google model is unchanged as long as that toolset + stays selected. A `/tools` selection that drops it now reports pi's builtin + `google-generative-ai` api instead of the CUA-owned one, since the selected + tools decide the transport. + +Breaking: the Tzafon, Yutori, and Meta providers are removed. + +- Refs like `-m tzafon:…` and `-m yutori:…` are no longer accepted, `cua models` + no longer lists either provider, and `TZAFON_API_KEY`/`YUTORI_API_KEY` are no + longer read. - `-m meta:muse-spark-1.1` is removed; use `-m openrouter:meta/muse-spark-1.1`. `META_API_KEY` is no longer read. -- The default interaction toolset for OpenRouter models is now chosen per model - rather than per provider. OpenRouter fronts several model families, and Kimi - K3 rejects `browser_act`'s schema while Muse Spark accepts it, so the CLI - asks the model's capabilities instead of assuming one answer per provider. - -Breaking: Tzafon and Yutori support is removed. - -- Update `@onkernel/cua-ai` and `@onkernel/cua-agent` to 0.13.0. Refs like - `-m tzafon:…` and `-m yutori:…` are no longer accepted, `cua models` no - longer lists either provider, and `TZAFON_API_KEY`/`YUTORI_API_KEY` are no - longer read. - The `/tools` picker no longer has atomic tool groups. They existed only for Yutori's n1 action set, which the catalog compiler refused to accept as a partial selection; every remaining tool toggles on its own. -## 0.11.0 - 2026-08-13 - -- Update `@onkernel/cua-ai` and `@onkernel/cua-agent` to 0.12.0. The default - Google interaction catalog is unchanged (`defaultInteractionTools` still - selects Google's native browser toolset), so the default `cua` model and - `--print -o jsonl`'s `assistant_usage.api` field for Google are unaffected - as long as that native toolset stays selected, including across an in-session - `/model` switch. Only a `/tools` selection that drops Google's native toolset - now reports pi's builtin `google-generative-ai` api instead of the CUA-owned - one. - -## 0.10.0 - 2026-08-13 - -- `--print -o jsonl` schema bumps to version 2: every assistant message now - also emits an `assistant_usage` event (`turn`, `model`, `api`, `input`, - `output`, `cache_read`, `cache_write`, `reasoning`, `total_tokens`, and a - derived `cache_hit_ratio`), including tool-only turns with no text. - ## 0.9.0 - 2026-08-04 Breaking: upgrade the pi stack to 0.83.0 (`pi-ai`, `pi-agent-core`, From a7de6f86c5f095cd0cbaca3f69df025b6c4b755c Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Fri, 14 Aug 2026 03:12:50 +0000 Subject: [PATCH 3/3] Spend an empty-response retry only once the follow-up is queued installCuaBehaviors incremented the attempt counter before awaiting followUp, so a rejected queue consumed a retry that the agent classes would not have spent. Match them: await first, then count. --- packages/agent/src/attach.ts | 6 ++++-- packages/agent/test/attach.test.ts | 30 ++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/packages/agent/src/attach.ts b/packages/agent/src/attach.ts index e340ed2..faefc04 100644 --- a/packages/agent/src/attach.ts +++ b/packages/agent/src/attach.ts @@ -161,15 +161,17 @@ export function installCuaBehaviors( hasPendingQueue = false; return undefined; })); - offs.push(harness.subscribe((event: any, signal?: AbortSignal) => { + offs.push(harness.subscribe(async (event: any, signal?: AbortSignal) => { if (event.type === "message_end" && event.message.role === "assistant") turnFailed = false; else if (event.type === "tool_execution_end" && event.isError) turnFailed = true; else if (event.type === "queue_update") hasPendingQueue = event.steer.length > 0 || event.followUp.length > 0; if (!recovery || recovery.maxAttempts <= 0) return; if (event.type !== "turn_end" || !isEmptyAssistantResponse(event.message)) return; if (signal?.aborted || recoveryAttempts >= recovery.maxAttempts || hasPendingQueue) return; + // Count the attempt only once the follow-up is actually queued: a rejected + // queue leaves the turn exactly as it was, so it must not consume a retry. + await harness.followUp(recovery.followUp); recoveryAttempts += 1; - return harness.followUp(recovery.followUp); })); return () => { diff --git a/packages/agent/test/attach.test.ts b/packages/agent/test/attach.test.ts index faf2c61..fbffca7 100644 --- a/packages/agent/test/attach.test.ts +++ b/packages/agent/test/attach.test.ts @@ -10,6 +10,7 @@ import { } from "@onkernel/cua-ai"; import type Kernel from "@onkernel/sdk"; import { Agent, AgentHarness, attach, InMemorySessionRepo, type KernelBrowser, type StreamFn } from "../src/index"; +import { installCuaBehaviors } from "../src/attach"; const browser = { session_id: "browser_123", viewport: { width: 1440, height: 900 } } as KernelBrowser; const client = {} as Kernel; @@ -113,6 +114,35 @@ describe("attach", () => { uninstall(); }); + it("spends an empty-response retry only when the follow-up is queued", async () => { + const attempted: string[] = []; + let reject = true; + let emit!: (event: unknown, signal?: AbortSignal) => Promise; + const stub = { + on: () => () => {}, + subscribe: (handler: (event: unknown, signal?: AbortSignal) => Promise) => { + emit = handler; + return () => {}; + }, + followUp: async (message: string) => { + attempted.push(message); + if (reject) throw new Error("queue closed"); + }, + }; + const manager = { catalog: { entries: [] }, specFor: () => undefined }; + installCuaBehaviors(stub as never, manager as never, { followUp: "continue", maxAttempts: 1 }); + + const emptyTurn = { type: "turn_end", message: { role: "assistant", stopReason: "stop", content: [] } }; + await expect(emit(emptyTurn)).rejects.toThrow("queue closed"); + reject = false; + await emit(emptyTurn); + // The rejected queue left the turn untouched, so it must not have spent the + // single attempt; the second follow-up does, and a third is refused. + await emit(emptyTurn); + + expect(attempted).toEqual(["continue", "continue"]); + }); + it("disposes the shared execution pool once, not per compile", async () => { const handle = attach({ browser, client }); handle.compile({ model: "openai:gpt-5.5", tools: [] });