diff --git a/docs/architecture.md b/docs/architecture.md index 1b3dfe81..f42bf74b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -153,8 +153,9 @@ catalog: - Ordinary function tools stay ordinary. - Anthropic native browser/computer declarations replace only their own placeholders and merge required beta headers with caller headers. -- OpenAI native computer uses a CUA-owned Responses adapter and can coexist with - ordinary functions. +- OpenAI streams through pi's builtin Responses transport and its automatic + prompt caching; a CUA-owned adapter handles OpenAI's native computer tool and + tool-search namespace round-trips. - Tzafon replaces only the selected computer identity and fills declaration dimensions from the actual viewport. - Anthropic's native browser tool falls back to an equivalent function-tool diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index 78f9f332..54f3fda5 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 0.11.0 - 2026-08-13 + +- `responseThreading` (`CuaAgentOptions`/`CuaAgentHarnessOptions`) no longer + affects OpenAI models: OpenAI now streams through pi-ai's builtin Responses + transport and its automatic prompt caching regardless of this flag. The + option still governs Google and Tzafon's `previous_response_id`-style + continuation. +- Exempt OpenAI's native computer tool from the tool-result image replay + limit, alongside Tzafon: its `computer_call_output` items must each carry a + screenshot, and stateless replay no longer leaves them in provider-stored + state. + ## 0.10.0 - 2026-08-04 Breaking: upgrade `@earendil-works/pi-agent-core` and `@earendil-works/pi-ai` diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index 9e68b96c..89886cea 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -92,6 +92,7 @@ export type CuaAgentOptions = Omit & streamFn?: AgentOptions["streamFn"]; emptyResponseRecovery?: CuaEmptyResponseRecoveryOptions; toolResultImageReplayLimit?: ToolResultImageReplayLimit; + /** Governs Google and Tzafon's `previous_response_id`-style continuation. Every other provider streams through pi's transports and their automatic prompt caching regardless of this flag. Defaults to `true`. */ responseThreading?: boolean; retry?: CuaRetryOptions; }; @@ -112,6 +113,7 @@ type CuaAgentHarnessOptionsBase< onPayload?: SimpleStreamOptions["onPayload"]; emptyResponseRecovery?: CuaEmptyResponseRecoveryOptions; toolResultImageReplayLimit?: ToolResultImageReplayLimit; + /** Governs Google and Tzafon's `previous_response_id`-style continuation. Every other provider streams through pi's transports and their automatic prompt caching regardless of this flag. Defaults to `true`. */ responseThreading?: boolean; retry?: CuaRetryOptions; }; @@ -203,7 +205,7 @@ export class CuaAgent { transformContext: async (messages, signal) => projectToolResultImages( transformContext ? await transformContext(messages, signal) : messages, imageReplayLimit, - manager.catalog.incoming.tzafonComputerName, + requiredImageToolNames(manager.catalog.incoming), ), prepareNextTurnWithContext: async (context, signal) => { const update = prepareNextTurnWithContext @@ -483,7 +485,7 @@ function withCatalogModels( const contextFor = (context: Context) => projectModelContext( context, imageReplayLimit, - manager.catalog.incoming.tzafonComputerName, + requiredImageToolNames(manager.catalog.incoming), ); const optionsFor = (options: T): T => { const catalog = manager.catalog; @@ -525,15 +527,20 @@ function resolveToolResultImageReplayLimit(limit: ToolResultImageReplayLimit | u return limit; } +/** Native computer tool names whose screenshot history the provider protocol requires in full, regardless of the image replay limit. */ +function requiredImageToolNames(incoming: CuaIncomingToolPlan): ReadonlySet { + return new Set([incoming.tzafonComputerName, incoming.openaiComputerName].filter((name): name is string => !!name)); +} + function projectToolResultImages( messages: TMessage[], limit: ToolResultImageReplayLimit, - requiredImageToolName?: string, + requiredToolNames: ReadonlySet = new Set(), ): TMessage[] { if (limit === false) return messages; let imageCount = 0; for (const message of messages) { - if (message.role === "toolResult" && message.toolName !== requiredImageToolName) { + if (message.role === "toolResult" && !requiredToolNames.has(message.toolName)) { imageCount += message.content.filter((block) => block.type === "image").length; } } @@ -541,7 +548,7 @@ function projectToolResultImages( const firstRetainedImage = Math.max(0, imageCount - limit); let imageOrdinal = 0; return messages.map((message) => { - if (message.role !== "toolResult" || message.toolName === requiredImageToolName) return message; + if (message.role !== "toolResult" || requiredToolNames.has(message.toolName)) return message; let changed = false; let markerInserted = false; const content = [] as typeof message.content; @@ -563,9 +570,9 @@ function projectToolResultImages( function projectModelContext( context: Context, imageReplayLimit: ToolResultImageReplayLimit, - requiredImageToolName?: string, + requiredToolNames: ReadonlySet, ): Context { - const messages = projectToolResultImages(context.messages, imageReplayLimit, requiredImageToolName); + const messages = projectToolResultImages(context.messages, imageReplayLimit, requiredToolNames); return messages === context.messages ? context : { ...context, messages }; } diff --git a/packages/agent/test/agent.test.ts b/packages/agent/test/agent.test.ts index 395e23ad..ddd61f61 100644 --- a/packages/agent/test/agent.test.ts +++ b/packages/agent/test/agent.test.ts @@ -132,6 +132,34 @@ describe("CuaAgent explicit tools", () => { expect(results[1]!.content).toEqual([{ type: "text", text: "[stale tool-result images omitted]" }]); }); + it("retains protocol-required OpenAI native computer screenshot results outside the image replay limit", async () => { + const contexts: Context[] = []; + const model = getCuaModel("openai:gpt-5.5"); + const nativeComputer = cua.providers.openai.tools.computer(); + const ordinaryTool = callerTool("ordinary"); + const image = { type: "image" as const, data: "c2NyZWVuc2hvdA==", mimeType: "image/png" }; + const messages: AgentMessage[] = [ + assistant(model, [{ type: "toolCall", id: "native-shot", name: "computer", arguments: { action: { type: "screenshot" } } }], "toolUse"), + { role: "toolResult", toolCallId: "native-shot", toolName: "computer", content: [image], isError: false, timestamp: 1 }, + assistant(model, [{ type: "toolCall", id: "ordinary-shot", name: "ordinary", arguments: {} }], "toolUse"), + { role: "toolResult", toolCallId: "ordinary-shot", toolName: "ordinary", content: [image], isError: false, timestamp: 2 }, + ]; + const agent = new CuaAgent({ + browser, + client, + tools: [nativeComputer, ordinaryTool], + streamFn: scriptedStream([(selectedModel) => assistant(selectedModel)], contexts), + toolResultImageReplayLimit: 0, + initialState: { model, messages }, + }); + + await agent.prompt("continue"); + + const results = contexts[0]!.messages.filter((message) => message.role === "toolResult"); + expect(results[0]!.content).toEqual([image]); + expect(results[1]!.content).toEqual([{ type: "text", text: "[stale tool-result images omitted]" }]); + }); + it("installs exact native-only, native-plus-CUA, Playwright-only, and browser-act-only catalogs", () => { const custom = callerTool("customer_lookup"); const cases = [ diff --git a/packages/agent/test/openai-deferred-tools.test.ts b/packages/agent/test/openai-deferred-tools.test.ts index 60ba0bfc..962725d8 100644 --- a/packages/agent/test/openai-deferred-tools.test.ts +++ b/packages/agent/test/openai-deferred-tools.test.ts @@ -86,7 +86,6 @@ describe("OpenAI deferred tool namespace continuation", () => { tools: [loader], initialState: { model: "openai:gpt-5.5" }, getApiKey: () => "test", - responseThreading: false, }); await agent.prompt("load and run the added tool"); diff --git a/packages/agent/vitest.config.ts b/packages/agent/vitest.config.ts index 039b5887..ab19cb94 100644 --- a/packages/agent/vitest.config.ts +++ b/packages/agent/vitest.config.ts @@ -8,5 +8,9 @@ export default defineConfig({ globals: true, environment: "node", testTimeout: 30000, + // pi-ai ships real ESM and imports "openai" itself; both need to run + // through Vitest's module graph (not Node's native loader) for + // vi.mock("openai") to intercept requests pi's builtin transport makes. + server: { deps: { inline: [/@earendil-works\/pi-ai/, /^openai$/] } }, }, }); diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 2d336ad6..aa0487c5 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -1,5 +1,38 @@ # Changelog +## 0.11.0 - 2026-08-13 + +Breaking: OpenAI models no longer carry a CUA-owned api id. + +- OpenAI models now resolve to pi-ai's builtin `"openai-responses"` api + instead of the removed `openai-cua-responses`, and stream through pi's + builtin Responses transport (`store: false`, automatic prompt-cache-key + matching) by default. `OPENAI_CUA_RESPONSES_API` and the OpenAI adapter's + `previous_response_id` threading are removed; `previous_response_id` and + `store: true` no longer appear on any OpenAI request. +- The CUA-owned OpenAI adapter is retained but now dispatches on request + shape rather than a rerouted api id: it only intercepts requests that select + OpenAI's native computer tool, or whose transcript carries a deferred + tool-search addition or a replayed function-call namespace (pi-ai 0.83.0's + builtin transport does not round-trip either). +- OpenAI's native computer adapter now sends the same `prompt_cache_key`, + `prompt_cache_retention`, `prompt_cache_options`, and session-affinity + headers as the function-tool path. It previously relied on stored response + state for context reuse and sent no cache key of its own. +- Remove the xAI and Meta Responses forks. Both existed only to thread + `previous_response_id` and to set `parallel_tool_calls: false`, which the tool + catalog already emits for those providers. `xai-cua-responses` and + `meta-responses` are gone: Grok streams through pi's builtin xAI provider, and + Meta registers pi's builtin Responses transport against its own base URL and + credentials. `XAI_CUA_RESPONSES_API`, `META_RESPONSES_API`, + `streamXaiResponses`, `streamSimpleXaiResponses`, `streamMetaResponses`, and + `streamSimpleMetaResponses` are no longer exported. Meta also regains pi's + stateless encrypted-reasoning replay, which the fork deleted because it relied + on stored response state. +- Google and Tzafon keep their continuation protocols: each threads a + provider-specific field with no builtin equivalent, and the shared helpers in + `providers/common.ts` are unchanged for them. + ## 0.10.0 - 2026-08-04 Breaking: upgrade `@earendil-works/pi-ai` to 0.83.0. diff --git a/packages/ai/README.md b/packages/ai/README.md index 23542c50..aef7db5e 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -240,8 +240,9 @@ additions through pi's active-tool change entries. ## Provider behavior -- **OpenAI**: CUA-owned Responses transport for native computer plus ordinary - function composition and response threading. +- **OpenAI**: streams through pi's builtin Responses transport and its + automatic prompt caching by default. A CUA-owned adapter is used only for + OpenAI's native computer tool and for tool-search namespace round-trips. - **Anthropic**: exact native declarations, beta-header composition, and adaptive model preparation. - **Google**: a CUA-owned Interactions API adapter plus the current predefined diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 1823aad1..fca2e3c2 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -4,22 +4,15 @@ export { createCuaModels, cuaModels, GOOGLE_CUA_INTERACTIONS_API, - META_RESPONSES_API, - OPENAI_CUA_RESPONSES_API, streamGoogleInteractions, - streamMetaResponses, streamOpenAIResponses, streamSimpleGoogleInteractions, - streamSimpleMetaResponses, streamSimpleOpenAIResponses, streamSimpleTzafonResponses, - streamSimpleXaiResponses, streamSimpleYutori, streamTzafonResponses, - streamXaiResponses, streamYutori, TZAFON_RESPONSES_API, - XAI_CUA_RESPONSES_API, YUTORI_CHAT_COMPLETIONS_API, } from "./providers"; export * from "./models"; diff --git a/packages/ai/src/models.ts b/packages/ai/src/models.ts index 07049201..cf167abd 100644 --- a/packages/ai/src/models.ts +++ b/packages/ai/src/models.ts @@ -1,9 +1,6 @@ import type { Api, Model } from "@earendil-works/pi-ai"; import { getBuiltinModel, getBuiltinModels } from "@earendil-works/pi-ai/providers/all"; import { GOOGLE_CUA_INTERACTIONS_API } from "./providers/google/provider"; -import { META_RESPONSES_API } from "./providers/meta/provider"; -import { OPENAI_CUA_RESPONSES_API } from "./providers/openai/provider"; -import { XAI_CUA_RESPONSES_API } from "./providers/xai/provider"; /** Providers with curated computer-use model support. */ export type CuaProvider = "openai" | "anthropic" | "google" | "meta" | "xai" | "moonshotai" | "openrouter" | "tzafon" | "yutori"; @@ -229,22 +226,18 @@ export function getCuaModel(ref: CuaModelRef): Model { throw new Error(`CUA model "${ref}" is supported but not registered. Add it to pi-ai (models.dev) or CUA_MODEL_OVERRIDES.`); } -// Route CUA models to provider-specific transports. Registry-resolved models -// otherwise carry pi-ai's builtin API ids. +// Route CUA models to provider-specific transports. OpenAI keeps pi-ai's +// builtin "openai-responses" api and streams through its automatic prompt +// caching; the CUA adapter dispatches on request shape (see +// requiresCuaOpenAIAdapter), not on a rerouted api id. Other registry-resolved +// models otherwise carry pi-ai's builtin API ids too. export function routeCuaApi(model: Model): Model { if (model.provider === "google" && model.api !== GOOGLE_CUA_INTERACTIONS_API) { return { ...model, api: GOOGLE_CUA_INTERACTIONS_API }; } - if (model.provider === "openai" && model.api !== OPENAI_CUA_RESPONSES_API) { - return { ...model, api: OPENAI_CUA_RESPONSES_API }; - } - if (model.provider === "meta" && model.api !== META_RESPONSES_API) { - return { ...model, api: META_RESPONSES_API }; - } if (model.provider === "xai" && model.id === "grok-4.5") { return { ...model, - api: XAI_CUA_RESPONSES_API, thinkingLevelMap: { off: "low", minimal: "low", xhigh: "high" }, cost: { ...model.cost, @@ -328,7 +321,7 @@ function cuaModel(provider: "meta" | "tzafon" | "yutori", id: string, name: stri // computer-use cookbook configures 128,000 maximum output tokens. return { ...base, - api: META_RESPONSES_API, + api: "openai-responses", baseUrl: "https://api.meta.ai/v1", thinkingLevelMap: { off: null, xhigh: "xhigh" }, cost: { input: 1.25, output: 4.25, cacheRead: 0.15, cacheWrite: 0 }, diff --git a/packages/ai/src/providers.ts b/packages/ai/src/providers.ts index b51862aa..fd6de719 100644 --- a/packages/ai/src/providers.ts +++ b/packages/ai/src/providers.ts @@ -10,15 +10,17 @@ import { type SimpleStreamOptions, type StreamOptions, } from "@earendil-works/pi-ai"; +import { + stream as piStreamOpenAIResponses, + streamSimple as piStreamSimpleOpenAIResponses, +} from "@earendil-works/pi-ai/api/openai-responses"; import { builtinModels } from "@earendil-works/pi-ai/providers/all"; import { cuaApiKeyEnvVarsForProvider } from "./api-keys"; import { cuaOverrideModels } from "./models"; import { withAnthropicBrowserFallback } from "./providers/anthropic/browser-fallback"; import { GOOGLE_CUA_INTERACTIONS_API, streamGoogleInteractions, streamSimpleGoogleInteractions } from "./providers/google/provider"; -import { META_RESPONSES_API, streamMetaResponses, streamSimpleMetaResponses } from "./providers/meta/provider"; -import { OPENAI_CUA_RESPONSES_API, streamOpenAIResponses, streamSimpleOpenAIResponses } from "./providers/openai/provider"; +import { requiresCuaOpenAIAdapter, streamOpenAIResponses, streamSimpleOpenAIResponses } from "./providers/openai/provider"; import { streamSimpleTzafonResponses, streamTzafonResponses, TZAFON_RESPONSES_API } from "./providers/tzafon/provider"; -import { streamSimpleXaiResponses, streamXaiResponses, XAI_CUA_RESPONSES_API } from "./providers/xai/provider"; import { streamSimpleYutori, streamYutori, YUTORI_CHAT_COMPLETIONS_API } from "./providers/yutori/provider"; /** @@ -27,17 +29,20 @@ import { streamSimpleYutori, streamYutori, YUTORI_CHAT_COMPLETIONS_API } from ". * * - `anthropic` retries an inaccessible native browser beta through the * selected tool's equivalent function declaration. - * - `openai` intercepts the `openai-cua-responses` api that - * {@link getCuaModel} routes OpenAI models to, threading - * `previous_response_id`; every other api falls through to pi's builtin - * provider. + * - `openai` streams through pi's builtin `openai-responses` transport and + * its automatic prompt caching by default. The CUA adapter only intercepts + * requests that need it: OpenAI's native computer tool, or a transcript + * carrying a deferred tool-search addition or a replayed function-call + * namespace (see {@link requiresCuaOpenAIAdapter}). * - `google` intercepts `google-cua-interactions` for current native computer * use and resolves API keys from `GOOGLE_API_KEY` or `GEMINI_API_KEY`. - * - `xai` intercepts `xai-cua-responses` so Grok can use stateful Responses - * tool loops while preserving pi's builtin xAI auth and catalog. + * - `xai` is pi's builtin provider untouched: Grok streams through pi's + * Responses transport, and the catalog supplies its serial-tool-call field. * - `moonshotai` is pi's builtin provider untouched: Kimi streams through the * plain OpenAI-compatible chat completions transport with `MOONSHOT_API_KEY`. * - `meta`, `tzafon`, and `yutori` are CUA-only providers pi does not ship. + * `meta` speaks the OpenAI Responses wire protocol, so it registers pi's + * builtin transport against Meta's base URL and credentials. * * Each call returns an independent collection; register additional providers * or credentials on it freely. Use {@link cuaModels} for the shared default. @@ -47,11 +52,9 @@ export function createCuaModels(options?: CreateModelsOptions): MutableModels { const anthropic = models.getProvider("anthropic"); if (anthropic) models.setProvider(withAnthropicBrowserFallback(anthropic)); const openai = models.getProvider("openai"); - if (openai) models.setProvider(withOpenAICuaResponses(openai)); + if (openai) models.setProvider(withOpenAICuaComputerAdapter(openai)); const google = models.getProvider("google"); if (google) models.setProvider(withGoogleCuaInteractions(google)); - const xai = models.getProvider("xai"); - if (xai) models.setProvider(withXaiCuaResponses(xai)); models.setProvider(metaProvider()); models.setProvider(tzafonProvider()); models.setProvider(yutoriProvider()); @@ -72,18 +75,18 @@ export function cuaModels(): MutableModels { return (defaultCuaModels ??= createCuaModels()); } -// pi's builtin openai provider only streams its own api ids. CUA routes -// OpenAI models to OPENAI_CUA_RESPONSES_API (see routeCuaApi), so the -// registered provider must dispatch that api to cua's threading stream fns. -function withOpenAICuaResponses(base: Provider): Provider { +// OpenAI models keep pi-ai's builtin "openai-responses" api id. Only requests +// that need cua-ai's adapter (see requiresCuaOpenAIAdapter) are intercepted; +// everything else falls through to pi's builtin provider. +function withOpenAICuaComputerAdapter(base: Provider): Provider { return { ...base, stream: (model: Model, context: Context, options?: StreamOptions) => - model.api === OPENAI_CUA_RESPONSES_API + model.api === "openai-responses" && requiresCuaOpenAIAdapter(context, options) ? streamOpenAIResponses(model as never, context, options) : base.stream(model, context, options), streamSimple: (model: Model, context: Context, options?: SimpleStreamOptions) => - model.api === OPENAI_CUA_RESPONSES_API + model.api === "openai-responses" && requiresCuaOpenAIAdapter(context, options) ? streamSimpleOpenAIResponses(model as never, context, options) : base.streamSimple(model, context, options), }; @@ -104,19 +107,6 @@ function withGoogleCuaInteractions(base: Provider): Provider { }; } -function withXaiCuaResponses(base: Provider): Provider { - return { - ...base, - stream: (model: Model, context: Context, options?: StreamOptions) => - model.api === XAI_CUA_RESPONSES_API - ? streamXaiResponses(model as never, context, options) - : base.stream(model, context, options), - streamSimple: (model: Model, context: Context, options?: SimpleStreamOptions) => - model.api === XAI_CUA_RESPONSES_API - ? streamSimpleXaiResponses(model as never, context, options) - : base.streamSimple(model, context, options), - }; -} function metaProvider(): Provider { return createProvider({ @@ -125,7 +115,7 @@ function metaProvider(): Provider { baseUrl: "https://api.meta.ai/v1", auth: { apiKey: envApiKeyAuth("Meta Model API key", cuaApiKeyEnvVarsForProvider("meta")) }, models: cuaOverrideModels("meta"), - api: { stream: streamMetaResponses, streamSimple: streamSimpleMetaResponses }, + api: { stream: piStreamOpenAIResponses, streamSimple: piStreamSimpleOpenAIResponses }, }); } @@ -152,8 +142,6 @@ function yutoriProvider(): Provider { } export { GOOGLE_CUA_INTERACTIONS_API, streamGoogleInteractions, streamSimpleGoogleInteractions }; -export { META_RESPONSES_API, streamMetaResponses, streamSimpleMetaResponses }; -export { OPENAI_CUA_RESPONSES_API, streamOpenAIResponses, streamSimpleOpenAIResponses }; +export { streamOpenAIResponses, streamSimpleOpenAIResponses }; export { TZAFON_RESPONSES_API, streamSimpleTzafonResponses, streamTzafonResponses }; -export { XAI_CUA_RESPONSES_API, streamSimpleXaiResponses, streamXaiResponses }; export { YUTORI_CHAT_COMPLETIONS_API, streamSimpleYutori, streamYutori }; diff --git a/packages/ai/src/providers/meta/provider.ts b/packages/ai/src/providers/meta/provider.ts deleted file mode 100644 index dffa9e15..00000000 --- a/packages/ai/src/providers/meta/provider.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { - type Context, - type OpenAIResponsesOptions as PiOpenAIResponsesOptions, - type StreamFunction, -} from "@earendil-works/pi-ai"; -import { - stream as piStreamOpenAIResponses, - streamSimple as piStreamSimpleOpenAIResponses, -} from "@earendil-works/pi-ai/api/openai-responses"; -import { - type CuaSimpleStreamOptions, - type ResponsesThreadingOptions, - threadResponsesRequest, -} from "../common"; - -export const META_RESPONSES_API = "meta-responses"; - -/** - * Stream options for Meta's OpenAI-compatible Responses API. - */ -export interface MetaResponsesOptions extends PiOpenAIResponsesOptions, ResponsesThreadingOptions {} - -/** - * Apply Meta's computer-use payload constraints on top of shared Responses threading. - */ -export function threadMetaRequest(context: Context, options: ResponsesThreadingOptions | undefined) { - const callerOnPayload = options?.onPayload; - const threaded = threadResponsesRequest(context, META_RESPONSES_API, { - ...options, - onPayload: async (payload, model) => { - const constrained: Record = { ...(payload as Record), parallel_tool_calls: false }; - delete constrained.include; - return callerOnPayload ? ((await callerOnPayload(constrained, model)) ?? constrained) : constrained; - }, - }); - const onPayload: typeof threaded.onPayload = async (payload, model) => { - const prepared = await threaded.onPayload(payload, model); - const sanitized: Record = { - ...(prepared as Record), - store: true, - parallel_tool_calls: false, - }; - delete sanitized.previous_response_id; - if (threaded.previousResponseId) sanitized.previous_response_id = threaded.previousResponseId; - // CUA uses stored response state instead of stateless encrypted reasoning replay. - delete sanitized.include; - return sanitized; - }; - return { context: threaded.context, onPayload }; -} - -// Meta implements the OpenAI Responses wire protocol. The model carries a -// distinct api id so response ids can never be threaded across providers. -export const streamMetaResponses: StreamFunction = (model, context, options) => { - const threaded = threadMetaRequest(context, options); - return piStreamOpenAIResponses(model as never, threaded.context, { ...options, onPayload: threaded.onPayload }); -}; - -export const streamSimpleMetaResponses: StreamFunction = (model, context, options) => { - const threaded = threadMetaRequest(context, options); - return piStreamSimpleOpenAIResponses(model as never, threaded.context, { ...options, onPayload: threaded.onPayload }); -}; diff --git a/packages/ai/src/providers/openai/provider.ts b/packages/ai/src/providers/openai/provider.ts index 6e4047e2..78c6ec6e 100644 --- a/packages/ai/src/providers/openai/provider.ts +++ b/packages/ai/src/providers/openai/provider.ts @@ -19,40 +19,50 @@ import { convertResponsesTools, processResponsesStream, } from "@earendil-works/pi-ai/api/openai-responses-shared"; +import { createGrammarToolInputProperties } from "@earendil-works/pi-ai/api/constrained-sampling"; import { clampOpenAIPromptCacheKey } from "@earendil-works/pi-ai/api/openai-prompt-cache"; import { buildBaseOptions } from "@earendil-works/pi-ai/api/simple-options"; import type { CuaIncomingToolPlan } from "../../tool-catalog"; -import { - type CuaSimpleStreamOptions, - type ResponsesThreadingOptions, - threadResponsesRequest, -} from "../common"; - -export const OPENAI_CUA_RESPONSES_API = "openai-cua-responses"; +import type { CuaSimpleStreamOptions } from "../common"; -export interface OpenAIResponsesOptions extends PiOpenAIResponsesOptions, ResponsesThreadingOptions { +export interface OpenAIResponsesOptions extends PiOpenAIResponsesOptions { /** @internal Identity-addressed native dispatch compiled from selected tools. */ cuaIncomingToolPlan?: CuaIncomingToolPlan; } -export function threadRequest(context: Context, options: ResponsesThreadingOptions | undefined) { - const { context: threadedContext, onPayload } = threadResponsesRequest(context, OPENAI_CUA_RESPONSES_API, options); - return { context: threadedContext, onPayload }; +/** The request fields both the function-tool and native-computer paths read, whichever option shape the caller passed. */ +type OpenAIRequestOptions = Pick; + +/** + * Whether a request needs cua-ai's OpenAI Responses adapter instead of pi-ai's + * builtin `openai-responses` transport: OpenAI's native computer tool is + * selected, or the transcript carries state (a deferred tool-search addition, + * or a replayed function-call namespace) that only this adapter round-trips. + */ +export function requiresCuaOpenAIAdapter(context: Context, options?: unknown): boolean { + // pi's Provider.stream signature hides cua's added option, so the plan is + // read structurally here rather than at both call sites. + const cuaIncomingToolPlan = (options as { cuaIncomingToolPlan?: CuaIncomingToolPlan } | undefined)?.cuaIncomingToolPlan; + if (cuaIncomingToolPlan?.openaiComputerName) return true; + for (const message of context.messages) { + if (message.role === "toolResult" && (message.addedToolNames?.length ?? 0) > 0) return true; + if (message.role === "assistant" && message.content.some((part) => part.type === "toolCall" && toolCallNamespace(part))) return true; + } + return false; } -export const streamOpenAIResponses: StreamFunction = (model, context, options) => { +export const streamOpenAIResponses: StreamFunction<"openai-responses", OpenAIResponsesOptions> = (model, context, options) => { if (options?.cuaIncomingToolPlan?.openaiComputerName) return streamOpenAINativeComputer(model, context, options); return streamOpenAIFunctionTools(model, context, options); }; -export const streamSimpleOpenAIResponses: StreamFunction = (model, context, options) => { +export const streamSimpleOpenAIResponses: StreamFunction<"openai-responses", CuaSimpleStreamOptions> = (model, context, options) => { if (options?.cuaIncomingToolPlan?.openaiComputerName) return streamOpenAINativeComputer(model, context, options); const base = buildBaseOptions(model, context, options, options?.apiKey); const clampedReasoning = options?.reasoning ? clampThinkingLevel(model, options.reasoning) : undefined; return streamOpenAIFunctionTools(model, context, { ...base, reasoningEffort: clampedReasoning === "off" ? undefined : clampedReasoning, - disableResponseThreading: options?.disableResponseThreading, cuaIncomingToolPlan: options?.cuaIncomingToolPlan, }); }; @@ -60,10 +70,10 @@ export const streamSimpleOpenAIResponses: StreamFunction, + model: Model<"openai-responses">, context: Context, options: OpenAIResponsesOptions | undefined, ) { @@ -72,11 +82,10 @@ function streamOpenAIFunctionTools( void (async () => { try { const apiKey = openAIApiKey(options); - const threaded = threadRequest(context, options); - const compat = model.compat as { supportsToolSearch?: boolean } | undefined; - const placement = splitDeferredTools(threaded.context, compat?.supportsToolSearch === true); - let payload = buildFunctionPayload(model, threaded.context, options, placement); - payload = (await threaded.onPayload(payload, model)) as Record; + const placement = splitDeferredTools(context, model.compat?.supportsToolSearch === true); + const grammarToolInputProperties = createGrammarToolInputProperties(context.tools, model.compat?.supportsOpenAIGrammarTools === true); + let payload = buildFunctionPayload(model, context, options, placement, grammarToolInputProperties); + payload = ((await options?.onPayload?.(payload, model)) ?? payload) as Record; const client = createOpenAIClient(model, options, apiKey); const request = client.responses.create(payload as never, { ...(options?.signal ? { signal: options.signal } : {}), @@ -102,6 +111,7 @@ function streamOpenAIFunctionTools( { serviceTier: options?.serviceTier, applyServiceTierPricing: (usage, tier) => applyServiceTierPricing(usage, tier, model), + grammarToolInputProperties, }, ); applyToolCallNamespaces(output, namespaces); @@ -121,27 +131,31 @@ function streamOpenAIFunctionTools( } function buildFunctionPayload( - model: Model, + model: Model<"openai-responses">, context: Context, options: OpenAIResponsesOptions | undefined, placement: { immediate: Tool[]; deferred: Map }, + grammarToolInputProperties: ReadonlyMap, ): Record { - const input = convertResponsesMessages(model, context, new Set(["openai"]), { deferredTools: placement.deferred }); + const compat = model.compat; + const toolOptions = { supportsStrictMode: compat?.supportsStrictMode, supportsOpenAIGrammarTools: compat?.supportsOpenAIGrammarTools }; + const input = convertResponsesMessages(model, context, new Set(["openai"]), { + deferredTools: placement.deferred, + toolOptions, + grammarToolInputProperties, + }); applyTranscriptNamespaces(input as unknown as Array>, context.messages); - const retention = cacheRetention(options); - const compat = model.compat as { supportsLongCacheRetention?: boolean } | undefined; const payload: Record = { model: model.id, input, stream: true, - prompt_cache_key: retention === "none" ? undefined : clampOpenAIPromptCacheKey(options?.sessionId), - prompt_cache_retention: retention === "long" && compat?.supportsLongCacheRetention !== false ? "24h" : undefined, + ...promptCacheFields(model, options), store: false, }; if (options?.maxTokens) payload.max_output_tokens = Math.max(options.maxTokens, 16); if (options?.temperature !== undefined) payload.temperature = options.temperature; if (options?.serviceTier !== undefined) payload.service_tier = options.serviceTier; - if (placement.immediate.length > 0) payload.tools = convertResponsesTools(placement.immediate); + if (placement.immediate.length > 0) payload.tools = convertResponsesTools(placement.immediate, toolOptions); if (options?.toolChoice !== undefined) payload.tool_choice = options.toolChoice; if (model.reasoning) { if (options?.reasoningEffort || options?.reasoningSummary) { @@ -157,9 +171,20 @@ function buildFunctionPayload( return payload; } +/** Prompt-cache request fields, matching what pi's builtin Responses transport sends. */ +function promptCacheFields(model: Model<"openai-responses">, options: OpenAIRequestOptions | undefined): Record { + const compat = model.compat; + const retention = cacheRetention(options); + return { + prompt_cache_key: retention === "none" ? undefined : clampOpenAIPromptCacheKey(options?.sessionId), + prompt_cache_retention: retention === "long" && compat?.supportsLongCacheRetention !== false ? "24h" : undefined, + prompt_cache_options: retention === "none" && compat?.supportsExplicitPromptCacheMode ? { mode: "explicit" } : undefined, + }; +} + function createOpenAIClient( - model: Model, - options: OpenAIResponsesOptions | undefined, + model: Model<"openai-responses">, + options: OpenAIRequestOptions | undefined, apiKey: string, ): OpenAI { const headers: Record = { ...model.headers, ...options?.headers }; @@ -174,7 +199,7 @@ function createOpenAIClient( }); } -function openAIApiKey(options: OpenAIResponsesOptions | undefined): string { +function openAIApiKey(options: OpenAIRequestOptions | undefined): string { const apiKey = options?.apiKey || options?.env?.OPENAI_API_KEY || process.env.OPENAI_API_KEY; if (apiKey) return apiKey; const hasAuthorization = Object.entries(options?.headers ?? {}).some(([name, value]) => @@ -184,19 +209,26 @@ function openAIApiKey(options: OpenAIResponsesOptions | undefined): string { throw new Error("No API key for provider: openai"); } -function cacheRetention(options: OpenAIResponsesOptions | undefined): "none" | "short" | "long" { +function cacheRetention(options: OpenAIRequestOptions | undefined): "none" | "short" | "long" { if (options?.cacheRetention) return options.cacheRetention; return (options?.env?.PI_CACHE_RETENTION ?? process.env.PI_CACHE_RETENTION) === "long" ? "long" : "short"; } function applyTranscriptNamespaces(input: Array>, messages: readonly Context["messages"][number][]): void { - const calls = messages.flatMap((message) => message.role === "assistant" - ? message.content.filter((part): part is ToolCall => part.type === "toolCall") - : []); - let callIndex = 0; + const namespaces = new Map(); + for (const message of messages) { + if (message.role !== "assistant") continue; + for (const part of message.content) { + if (part.type !== "toolCall") continue; + const namespace = toolCallNamespace(part); + if (!namespace) continue; + const callId = part.id.split("|", 1)[0]!; + namespaces.set(callId, namespace); + } + } for (const item of input) { - if (item.type !== "function_call") continue; - const namespace = toolCallNamespace(calls[callIndex++]); + if (item.type !== "function_call" || typeof item.call_id !== "string") continue; + const namespace = namespaces.get(item.call_id); if (namespace) item.namespace = namespace; } } @@ -239,7 +271,7 @@ function toolCallNamespace(call: ToolCall | undefined): string | undefined { function applyServiceTierPricing( usage: AssistantMessage["usage"], serviceTier: OpenAIResponsesOptions["serviceTier"], - model: Model, + model: Model<"openai-responses">, ): void { const multiplier = serviceTier === "flex" ? 0.5 : serviceTier === "priority" ? (model.id === "gpt-5.5" ? 2.5 : 2) : 1; if (multiplier === 1) return; @@ -252,7 +284,7 @@ function applyServiceTierPricing( /** Responses adapter used only when the selected catalog contains OpenAI's native computer tool. */ function streamOpenAINativeComputer( - model: Model, + model: Model<"openai-responses">, context: Context, options: OpenAIResponsesOptions | CuaSimpleStreamOptions | undefined, ) { @@ -260,25 +292,21 @@ function streamOpenAINativeComputer( const output = initialAssistantMessage(model); void (async () => { try { - const apiKey = options?.apiKey || process.env.OPENAI_API_KEY; - if (!apiKey) throw new Error("No API key for provider: openai"); + const apiKey = openAIApiKey(options); const nativeName = options?.cuaIncomingToolPlan?.openaiComputerName; if (!nativeName) throw new Error("OpenAI native computer incoming plan is missing"); const placement = splitDeferredTools(context); - const threaded = threadRequest(context, options); let payload: Record = { model: model.id, - instructions: threaded.context.systemPrompt, - input: convertMessages(threaded.context.messages, nativeName, placement.deferred), + instructions: context.systemPrompt, + input: convertMessages(context.messages, nativeName, placement.deferred), tools: convertTools(placement.immediate), max_output_tokens: options?.maxTokens ?? model.maxTokens, + ...promptCacheFields(model, options), + store: false, }; - payload = (await threaded.onPayload(payload, model)) as Record; - const client = new OpenAI({ - apiKey, - baseURL: model.baseUrl || "https://api.openai.com/v1", - defaultHeaders: { ...model.headers, ...options?.headers }, - }); + payload = ((await options?.onPayload?.(payload, model)) ?? payload) as Record; + const client = createOpenAIClient(model, options, apiKey); const request = client.responses.create(payload as never, { signal: options?.signal }); const { data: response, response: rawResponse } = await request.withResponse(); await options?.onResponse?.({ status: rawResponse.status, headers: headersToRecord(rawResponse.headers) }, model); diff --git a/packages/ai/src/providers/xai/provider.ts b/packages/ai/src/providers/xai/provider.ts deleted file mode 100644 index 24b64828..00000000 --- a/packages/ai/src/providers/xai/provider.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { - type Context, - type OpenAIResponsesOptions as PiOpenAIResponsesOptions, - type StreamFunction, -} from "@earendil-works/pi-ai"; -import { - stream as piStreamOpenAIResponses, - streamSimple as piStreamSimpleOpenAIResponses, -} from "@earendil-works/pi-ai/api/openai-responses"; -import { - type CuaSimpleStreamOptions, - type ResponsesThreadingOptions, - threadResponsesRequest, -} from "../common"; - -export const XAI_CUA_RESPONSES_API = "xai-cua-responses"; - -/** - * Stream options for xAI's OpenAI-compatible Responses API. - */ -export interface XaiResponsesOptions extends PiOpenAIResponsesOptions, ResponsesThreadingOptions {} - -/** - * Apply xAI's serial computer-use constraints on top of shared Responses threading. - */ -export function threadXaiRequest(context: Context, options: ResponsesThreadingOptions | undefined) { - const callerOnPayload = options?.onPayload; - const threaded = threadResponsesRequest(context, XAI_CUA_RESPONSES_API, { - ...options, - onPayload: async (payload, model) => { - const constrained = { ...(payload as Record), parallel_tool_calls: false }; - return callerOnPayload ? ((await callerOnPayload(constrained, model)) ?? constrained) : constrained; - }, - }); - const onPayload: typeof threaded.onPayload = async (payload, model) => { - const prepared = await threaded.onPayload(payload, model); - const constrained: Record = { - ...(prepared as Record), - store: true, - parallel_tool_calls: false, - }; - delete constrained.previous_response_id; - if (threaded.previousResponseId) constrained.previous_response_id = threaded.previousResponseId; - return constrained; - }; - return { context: threaded.context, onPayload }; -} - -export const streamXaiResponses: StreamFunction = (model, context, options) => { - const threaded = threadXaiRequest(context, options); - return piStreamOpenAIResponses(model as never, threaded.context, { ...options, onPayload: threaded.onPayload }); -}; - -export const streamSimpleXaiResponses: StreamFunction = (model, context, options) => { - const threaded = threadXaiRequest(context, options); - return piStreamSimpleOpenAIResponses(model as never, threaded.context, { ...options, onPayload: threaded.onPayload }); -}; diff --git a/packages/ai/test/meta-threading.test.ts b/packages/ai/test/meta-threading.test.ts deleted file mode 100644 index 43f18a16..00000000 --- a/packages/ai/test/meta-threading.test.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { describe, expect, it } from "vitest"; -import type { Context, Message, Model } from "@earendil-works/pi-ai"; -import { META_RESPONSES_API, threadMetaRequest } from "../src/providers/meta/provider"; - -const model = {} as Model; - -function multiTurnContext(): Context { - const messages: Message[] = [ - { role: "user", content: "inspect the browser", timestamp: 0 }, - { - role: "assistant", - content: [{ type: "toolCall", id: "call_1|fc_1", name: "screenshot", arguments: {} }], - api: META_RESPONSES_API, - provider: "meta", - model: "muse-spark-1.1", - responseId: "resp_meta_1", - usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, - stopReason: "toolUse", - timestamp: 0, - }, - { - role: "toolResult", - toolCallId: "call_1|fc_1", - toolName: "screenshot", - content: [{ type: "image", mimeType: "image/png", data: "screenshot" }], - isError: false, - timestamp: 0, - }, - ]; - return { messages, tools: [], systemPrompt: "control the browser" }; -} - -describe("Meta Responses threading", () => { - it("threads the latest response and applies serial computer-use defaults", async () => { - const { context, onPayload } = threadMetaRequest(multiTurnContext(), undefined); - expect(context.messages).toHaveLength(1); - expect(await onPayload({ input: [] }, model)).toEqual({ - input: [], - store: true, - parallel_tool_calls: false, - previous_response_id: "resp_meta_1", - }); - }); - - it("omits encrypted reasoning replay on the initial request", async () => { - const context: Context = { messages: [{ role: "user", content: "hello", timestamp: 0 }] }; - const { onPayload } = threadMetaRequest(context, undefined); - expect(await onPayload({ input: [], include: ["reasoning.encrypted_content"] }, model)).toEqual({ - input: [], - store: true, - parallel_tool_calls: false, - }); - }); - - it("sends full history without previous_response_id when threading is disabled", async () => { - const context = multiTurnContext(); - const prepared = threadMetaRequest(context, { - disableResponseThreading: true, - onPayload: (payload) => ({ ...(payload as object), previous_response_id: "foreign-response" }), - }); - expect(prepared.context).toBe(context); - expect(await prepared.onPayload({ input: [] }, model)).toEqual({ - input: [], - store: true, - parallel_tool_calls: false, - }); - }); - - it("passes the prepared request to a caller payload hook", async () => { - const { onPayload } = threadMetaRequest(multiTurnContext(), { - onPayload: (payload) => ({ ...(payload as object), caller_field: true }), - }); - expect(await onPayload({}, model)).toEqual({ - caller_field: true, - store: true, - parallel_tool_calls: false, - previous_response_id: "resp_meta_1", - }); - }); - - it("reapplies Meta constraints after caller payload hooks", async () => { - const { onPayload } = threadMetaRequest(multiTurnContext(), { - onPayload: (payload) => ({ - ...(payload as object), - store: false, - parallel_tool_calls: true, - previous_response_id: "wrong-response", - include: ["reasoning.encrypted_content"], - }), - }); - expect(await onPayload({}, model)).toEqual({ - store: true, - parallel_tool_calls: false, - previous_response_id: "resp_meta_1", - }); - }); -}); diff --git a/packages/ai/test/models.test.ts b/packages/ai/test/models.test.ts index ddea4ba5..6f5628f0 100644 --- a/packages/ai/test/models.test.ts +++ b/packages/ai/test/models.test.ts @@ -9,10 +9,7 @@ import { getCuaModel, GOOGLE_CUA_INTERACTIONS_API, listCuaModels, - META_RESPONSES_API, - OPENAI_CUA_RESPONSES_API, parseCuaModelRef, - XAI_CUA_RESPONSES_API, } from "../src/index"; describe("CUA model refs", () => { @@ -79,7 +76,7 @@ describe("CUA model refs", () => { const muse = getCuaModel("meta:muse-spark-1.1"); expect(muse.provider).toBe("meta"); - expect(muse.api).toBe(META_RESPONSES_API); + expect(muse.api).toBe("openai-responses"); expect(muse.baseUrl).toBe("https://api.meta.ai/v1"); expect(muse.contextWindow).toBe(1_048_576); expect(muse.maxTokens).toBe(128_000); @@ -90,7 +87,7 @@ describe("CUA model refs", () => { expect(cuaOverrideModels("xai")).toEqual([]); const grok = getCuaModel("xai:grok-4.5"); expect(grok.provider).toBe("xai"); - expect(grok.api).toBe(XAI_CUA_RESPONSES_API); + expect(grok.api).toBe("openai-responses"); expect(grok.baseUrl).toBe("https://api.x.ai/v1"); expect(grok.contextWindow).toBe(500_000); expect(grok.maxTokens).toBe(500_000); @@ -134,15 +131,16 @@ describe("CUA model refs", () => { expect(getCuaModel("yutori:n1.5-latest").api).toBe("yutori-chat-completions"); }); - it("routes Responses models to provider-specific threading APIs", () => { - // Registry models carry pi-ai's builtin "openai-responses" api and must - // be routed to provider-specific previous_response_id transports. - expect(getCuaModel("openai:gpt-5.6-sol").api).toBe(OPENAI_CUA_RESPONSES_API); - expect(getCuaModel("openai:gpt-5.5").api).toBe(OPENAI_CUA_RESPONSES_API); - expect(getCuaModel("openai:gpt-5.4-mini").api).toBe(OPENAI_CUA_RESPONSES_API); + it("keeps every Responses model on pi's builtin transport except Google's Interactions API", () => { + // A CUA-owned api id now exists only where pi ships no equivalent + // transport. OpenAI, Meta, and xAI all speak the Responses protocol pi + // already implements, including its automatic prompt caching. + expect(getCuaModel("openai:gpt-5.6-sol").api).toBe("openai-responses"); + expect(getCuaModel("openai:gpt-5.5").api).toBe("openai-responses"); + expect(getCuaModel("openai:gpt-5.4-mini").api).toBe("openai-responses"); expect(getCuaModel("google:gemini-3.6-flash").api).toBe(GOOGLE_CUA_INTERACTIONS_API); - expect(getCuaModel("meta:muse-spark-1.1").api).toBe(META_RESPONSES_API); - expect(getCuaModel("xai:grok-4.5").api).toBe(XAI_CUA_RESPONSES_API); + expect(getCuaModel("meta:muse-spark-1.1").api).toBe("openai-responses"); + expect(getCuaModel("xai:grok-4.5").api).toBe("openai-responses"); }); it("rejects supported model IDs that are not in pi-ai or overrides", () => { diff --git a/packages/ai/test/openai-adapter-routing.test.ts b/packages/ai/test/openai-adapter-routing.test.ts new file mode 100644 index 00000000..74dcc56e --- /dev/null +++ b/packages/ai/test/openai-adapter-routing.test.ts @@ -0,0 +1,187 @@ +import { describe, expect, it, vi } from "vitest"; +import type { ToolCall } from "@earendil-works/pi-ai"; +import { createCuaModels, getCuaModel } from "../src/index"; + +const { responsesCreate } = vi.hoisted(() => ({ responsesCreate: vi.fn() })); + +vi.mock("openai", () => ({ + default: class { + responses = { + create: (...args: unknown[]) => ({ + withResponse: async () => ({ data: responsePayload(responsesCreate(...args)), response: { status: 200, headers: new Headers() } }), + }), + }; + }, +})); + +async function* responseEvents(response: Record) { + yield { type: "response.created", response: { id: response.id } }; + for (const [outputIndex, item] of ((response.output as unknown[]) ?? []).entries()) { + yield { type: "response.output_item.added", output_index: outputIndex, item }; + yield { type: "response.output_item.done", output_index: outputIndex, item }; + } + yield { type: "response.completed", response }; +} + +// pi's builtin transport iterates an event stream; the CUA native-computer +// adapter reads the raw response object's fields directly. Serve both from +// one mock so a single test file can exercise either dispatch path. +function responsePayload(response: Record): AsyncIterable & Record { + return Object.assign(responseEvents(response), response); +} + +const model = getCuaModel("openai:gpt-5.5"); +const tools = [{ name: "lookup", description: "lookup", parameters: { type: "object" } as never }]; + +describe("OpenAI adapter routing", () => { + it("streams a plain function-tool context through pi's builtin transport", async () => { + responsesCreate.mockReturnValueOnce({ + id: "resp_1", + status: "completed", + usage: { input_tokens: 1, output_tokens: 1 }, + output: [{ + type: "function_call", + id: "fc_1", + call_id: "call_1", + name: "lookup", + namespace: "deferred_tools", + arguments: "{}", + status: "completed", + }], + }); + const message = await createCuaModels().streamSimple(model, { + messages: [{ role: "user", content: "look it up", timestamp: 1 }], + tools, + }, { apiKey: "test", sessionId: "session_1" }).result(); + + // pi-ai 0.83.0's builtin Responses path does not parse the namespace + // field on function_call items, unlike the CUA adapter. + const call = message.content.find((part): part is ToolCall => part.type === "toolCall") as (ToolCall & { namespace?: string }) | undefined; + expect(call?.name).toBe("lookup"); + expect(call?.namespace).toBeUndefined(); + + // The measurement A/B depends on the default path relying on prompt + // caching instead of `previous_response_id` threading. + const payload = responsesCreate.mock.calls[0]![0] as Record; + expect(payload.store).toBe(false); + expect(payload.prompt_cache_key).toBe("session_1"); + expect(payload.previous_response_id).toBeUndefined(); + }); + + it("reaches the CUA adapter when the incoming plan selects OpenAI's native computer tool", async () => { + responsesCreate.mockReturnValueOnce({ + id: "resp_2", + output: [{ + type: "computer_call", + call_id: "computer_1", + action: { type: "click", x: 10, y: 20 }, + }], + }); + const message = await createCuaModels().streamSimple(model, { + messages: [{ role: "user", content: "click it", timestamp: 1 }], + tools: [{ name: "computer", description: "placeholder", parameters: { type: "object" } as never }], + }, { + apiKey: "test", + cuaIncomingToolPlan: { openaiComputerName: "computer", yutoriNames: {}, googleNames: {}, googleExcludedNames: [], nativeToolNames: ["computer"] }, + } as never).result(); + + // Only the CUA native-computer adapter understands computer_call items; + // pi's builtin transport has no case for them and would emit nothing. + const call = message.content.find((part): part is ToolCall => part.type === "toolCall"); + expect(call).toMatchObject({ name: "computer", arguments: { action: { type: "click", x: 10, y: 20 } } }); + }); + + it("reaches the CUA adapter when the transcript carries a deferred tool-search addition", async () => { + responsesCreate.mockReturnValueOnce({ + id: "resp_3", + status: "completed", + usage: { input_tokens: 1, output_tokens: 1 }, + output: [{ + type: "function_call", + id: "fc_3", + call_id: "call_3", + name: "lookup", + namespace: "deferred_tools", + arguments: "{}", + status: "completed", + }], + }); + const message = await createCuaModels().streamSimple(model, { + messages: [ + { role: "user", content: "load and look it up", timestamp: 1 }, + { + role: "toolResult", + toolCallId: "call_loader", + toolName: "loader", + content: [{ type: "text", text: "loaded" }], + isError: false, + addedToolNames: ["lookup"], + timestamp: 2, + }, + ], + tools, + }, { apiKey: "test" } as never).result(); + + // The CUA adapter round-trips the namespace pi's builtin drops. + const call = message.content.find((part): part is ToolCall => part.type === "toolCall") as (ToolCall & { namespace?: string }) | undefined; + expect(call?.namespace).toBe("deferred_tools"); + }); + + it("pairs replayed namespaces by call id, not ordinal, across an aborted assistant turn", async () => { + responsesCreate.mockReturnValueOnce({ id: "resp_4", usage: {}, output: [] }); + await createCuaModels().streamSimple(model, { + messages: [ + { role: "user", content: "load and look it up", timestamp: 1 }, + { + role: "assistant", + content: [{ type: "toolCall", id: "call_loader|fc_loader", name: "loader", arguments: {} }], + api: model.api, + provider: model.provider, + model: model.id, + usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + stopReason: "aborted", + timestamp: 2, + }, + { + role: "toolResult", + toolCallId: "call_loader|fc_loader", + toolName: "loader", + content: [{ type: "text", text: "loaded" }], + isError: false, + addedToolNames: ["lookup"], + timestamp: 3, + }, + { + role: "assistant", + content: [{ type: "toolCall", id: "call_lookup|fc_lookup", name: "lookup", arguments: {}, namespace: "deferred_tools" } as ToolCall], + api: model.api, + provider: model.provider, + model: model.id, + usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + stopReason: "toolUse", + timestamp: 4, + }, + { + role: "toolResult", + toolCallId: "call_lookup|fc_lookup", + toolName: "lookup", + content: [{ type: "text", text: "ok" }], + isError: false, + timestamp: 5, + }, + ], + tools, + }, { apiKey: "test" } as never).result(); + + // pi drops the aborted "loader" assistant message from the replayed + // transcript entirely, so pairing by ordinal would shift "lookup"'s + // namespace onto the missing "loader" call instead. + const payload = responsesCreate.mock.calls.at(-1)?.[0] as { input: Array> }; + expect(payload.input).toContainEqual(expect.objectContaining({ + type: "function_call", + call_id: "call_lookup", + name: "lookup", + namespace: "deferred_tools", + })); + }); +}); diff --git a/packages/ai/test/openai-native-provider.test.ts b/packages/ai/test/openai-native-provider.test.ts index 48c60c0a..d4cd523e 100644 --- a/packages/ai/test/openai-native-provider.test.ts +++ b/packages/ai/test/openai-native-provider.test.ts @@ -15,7 +15,7 @@ vi.mock("openai", () => ({ }, })); -const model = getCuaModel("openai:gpt-5.5") as Model; +const model = getCuaModel("openai:gpt-5.5") as Model<"openai-responses">; const incoming = { openaiComputerName: "computer", yutoriNames: {}, googleNames: {}, googleExcludedNames: [], nativeToolNames: ["computer"] }; describe("OpenAI native computer Responses adapter", () => { @@ -47,6 +47,22 @@ describe("OpenAI native computer Responses adapter", () => { pending_safety_checks: [{ id: "check_1", code: "malicious_instructions" }], }, }); + + const payload = responsesCreate.mock.calls.at(-1)?.[0] as Record; + expect(payload.store).toBe(false); + expect(payload.previous_response_id).toBeUndefined(); + }); + + it("sends the same prompt-cache fields as the function-tool path", async () => { + responsesCreate.mockReturnValueOnce({ id: "resp_cache", usage: {}, output: [] }); + await openai.streamOpenAIResponses(model, { + messages: [{ role: "user", content: "go", timestamp: 1 }], + tools: [{ name: "computer", description: "placeholder", parameters: { type: "object" } as never }], + }, { apiKey: "test", sessionId: "session_native", cuaIncomingToolPlan: incoming }).result(); + + const payload = responsesCreate.mock.calls.at(-1)?.[0] as Record; + expect(payload.prompt_cache_key).toBe("session_native"); + expect(payload.store).toBe(false); }); it("round-trips function-call namespaces beside the native computer adapter", async () => { @@ -91,15 +107,17 @@ describe("OpenAI native computer Responses adapter", () => { { name: "computer", description: "placeholder", parameters: { type: "object" } as never }, { name: "lookup", description: "lookup", parameters: { type: "object" } as never }, ], - }, { apiKey: "test", cuaIncomingToolPlan: incoming, disableResponseThreading: true }).result(); + }, { apiKey: "test", cuaIncomingToolPlan: incoming }).result(); - const payload = responsesCreate.mock.calls.at(-1)?.[0] as { input: Array> }; + const payload = responsesCreate.mock.calls.at(-1)?.[0] as { input: Array>; store?: unknown; previous_response_id?: unknown }; expect(payload.input).toContainEqual(expect.objectContaining({ type: "function_call", call_id: "call_lookup", name: "lookup", namespace: "deferred_tools", })); + expect(payload.store).toBe(false); + expect(payload.previous_response_id).toBeUndefined(); }); it("serializes native results as computer_call_output and ordinary results as function output", async () => { @@ -133,11 +151,10 @@ describe("OpenAI native computer Responses adapter", () => { }, { apiKey: "test", cuaIncomingToolPlan: incoming, - disableResponseThreading: true, onPayload: (payload) => ({ ...(payload as Record), tools: [{ type: "computer" }] }), }).result(); - const payload = responsesCreate.mock.calls.at(-1)?.[0] as { input: Array> }; + const payload = responsesCreate.mock.calls.at(-1)?.[0] as { input: Array>; store?: unknown; previous_response_id?: unknown }; expect(payload.input).toEqual(expect.arrayContaining([ expect.objectContaining({ type: "computer_call", call_id: "computer_1" }), expect.objectContaining({ @@ -147,5 +164,7 @@ describe("OpenAI native computer Responses adapter", () => { output: expect.objectContaining({ type: "computer_screenshot", image_url: expect.stringContaining("data:image/png;base64,") }), }), ])); + expect(payload.store).toBe(false); + expect(payload.previous_response_id).toBeUndefined(); }); }); diff --git a/packages/ai/test/openai-threading.test.ts b/packages/ai/test/openai-threading.test.ts deleted file mode 100644 index d41cbd07..00000000 --- a/packages/ai/test/openai-threading.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { describe, expect, it } from "vitest"; -import type { Context, Message, Model } from "@earendil-works/pi-ai"; -import { OPENAI_CUA_RESPONSES_API, threadRequest } from "../src/providers/openai/provider"; - -const TURNS = 6; -const model = {} as Model; - -/** Multi-turn context where each assistant turn carries a distinct responseId followed by a screenshot tool result. */ -function multiTurnContext(): Context { - const messages: Message[] = [{ role: "user", content: "book a flight", timestamp: 0 }]; - for (let turn = 0; turn < TURNS; turn += 1) { - messages.push({ - role: "assistant", - content: [{ type: "toolCall", id: `call_${turn}`, name: "click", arguments: { x: turn, y: turn } }], - api: OPENAI_CUA_RESPONSES_API, - provider: "openai", - model: "gpt-5.5", - responseId: `resp_${turn}`, - usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, - stopReason: "toolUse", - timestamp: 0, - }); - messages.push({ - role: "toolResult", - toolCallId: `call_${turn}`, - toolName: "click", - content: [{ type: "image", mimeType: "image/png", data: `screenshot-${turn}` }], - isError: false, - timestamp: 0, - }); - } - return { messages, tools: [], systemPrompt: "control the browser" }; -} - -describe("openai threadRequest", () => { - it("prunes to the delta and injects store + previous_response_id when threading (default)", async () => { - const { context, onPayload } = threadRequest(multiTurnContext(), undefined); - // Only the latest tool result (after the last assistant turn) is sent; the rest lives server-side. - expect(context.messages).toHaveLength(1); - expect((context.messages[0] as { toolCallId?: string }).toolCallId).toBe(`call_${TURNS - 1}`); - expect(await onPayload({ input: [] }, model)).toEqual({ input: [], store: true, previous_response_id: `resp_${TURNS - 1}` }); - }); - - it("sends full history with store but no previous_response_id when disabled by option", async () => { - const ctx = multiTurnContext(); - const { context, onPayload } = threadRequest(ctx, { disableResponseThreading: true }); - expect(context).toBe(ctx); - expect(await onPayload({}, model)).toEqual({ store: true }); - }); - - it("falls back to full history when the latest assistant turn lacks a responseId", async () => { - const ctx = multiTurnContext(); - ctx.messages.push({ - role: "assistant", - content: [{ type: "text", text: "request failed" }], - api: OPENAI_CUA_RESPONSES_API, - provider: "openai", - model: "gpt-5.5", - usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, - stopReason: "error", - timestamp: 0, - }); - const { context, onPayload } = threadRequest(ctx, undefined); - expect(context).toBe(ctx); - expect(((await onPayload({}, model)) as Record).previous_response_id).toBeUndefined(); - }); - - it("ignores a responseId from an errored turn so it never anchors previous_response_id", async () => { - const ctx = multiTurnContext(); - // An error after response.created can capture a responseId for a response the server never stored. - ctx.messages.push({ - role: "assistant", - content: [{ type: "text", text: "request failed" }], - api: OPENAI_CUA_RESPONSES_API, - provider: "openai", - model: "gpt-5.5", - responseId: "resp_failed", - usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, - stopReason: "error", - timestamp: 0, - }); - const { context, onPayload } = threadRequest(ctx, undefined); - expect(context).toBe(ctx); - expect(((await onPayload({}, model)) as Record).previous_response_id).toBeUndefined(); - }); - - it("never anchors previous_response_id on an assistant turn from a different api", async () => { - const ctx = multiTurnContext(); - // A mid-session -m provider switch leaves the prior provider's turn (and its foreign id) as the anchor. - ctx.messages.push({ - role: "assistant", - content: [{ type: "text", text: "done" }], - api: "anthropic-messages", - provider: "anthropic", - model: "claude-opus-4-8", - responseId: "msg_anthropic", - usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, - stopReason: "stop", - timestamp: 0, - }); - const { context, onPayload } = threadRequest(ctx, undefined); - expect(context).toBe(ctx); - expect(((await onPayload({}, model)) as Record).previous_response_id).toBeUndefined(); - }); - - it("composes a caller onPayload on top of the threaded payload", async () => { - const { onPayload } = threadRequest(multiTurnContext(), { - onPayload: (payload) => ({ wrapped: payload }), - }); - expect(await onPayload({ input: [] }, model)).toEqual({ - wrapped: { input: [], store: true, previous_response_id: `resp_${TURNS - 1}` }, - }); - }); -}); diff --git a/packages/ai/test/providers.test.ts b/packages/ai/test/providers.test.ts index 2faf9e71..ea651aba 100644 --- a/packages/ai/test/providers.test.ts +++ b/packages/ai/test/providers.test.ts @@ -2,7 +2,6 @@ import { describe, expect, it } from "vitest"; import { createCuaModels, cuaModels, - OPENAI_CUA_RESPONSES_API, TZAFON_RESPONSES_API, YUTORI_CHAT_COMPLETIONS_API, } from "../src/index"; @@ -20,7 +19,7 @@ describe("createCuaModels", () => { it("lists CUA provider catalogs", () => { const models = createCuaModels(); - expect(models.getModel("meta", "muse-spark-1.1")?.api).toBe("meta-responses"); + expect(models.getModel("meta", "muse-spark-1.1")?.api).toBe("openai-responses"); expect(models.getModel("xai", "grok-4.5")?.api).toBe("openai-responses"); expect(models.getModel("moonshotai", "kimi-k3")?.api).toBe("openai-completions"); expect(models.getModel("openrouter", "moonshotai/kimi-k3")?.api).toBe("openai-completions"); @@ -41,9 +40,10 @@ describe("createCuaModels", () => { const models = createCuaModels(); const openaiIds = models.getModels("openai").map((m) => m.id); expect(openaiIds).toContain("gpt-5.4"); - // The catalog keeps pi's api ids; getCuaModel() routes to - // openai-cua-responses, which the wrapped provider dispatches. - expect(models.getModel("openai", "gpt-5.4")?.api).not.toBe(OPENAI_CUA_RESPONSES_API); + // OpenAI models keep pi's builtin "openai-responses" api id on both the + // collection and getCuaModel(); the wrapped provider only intercepts + // requests that need the CUA adapter (see requiresCuaOpenAIAdapter). + expect(models.getModel("openai", "gpt-5.4")?.api).toBe("openai-responses"); const xaiIds = models.getModels("xai").map((m) => m.id); expect(xaiIds).toContain("grok-4.3"); diff --git a/packages/ai/test/xai-threading.test.ts b/packages/ai/test/xai-threading.test.ts deleted file mode 100644 index 1907aca6..00000000 --- a/packages/ai/test/xai-threading.test.ts +++ /dev/null @@ -1,87 +0,0 @@ -import type { Context, Message, Model } from "@earendil-works/pi-ai"; -import { describe, expect, it } from "vitest"; -import { threadXaiRequest, XAI_CUA_RESPONSES_API } from "../src/providers/xai/provider"; - -const model = {} as Model; - -function multiTurnContext(): Context { - const messages: Message[] = [ - { role: "user", content: "inspect the browser", timestamp: 0 }, - { - role: "assistant", - content: [{ type: "toolCall", id: "call_1|fc_1", name: "screenshot", arguments: {} }], - api: XAI_CUA_RESPONSES_API, - provider: "xai", - model: "grok-4.5", - responseId: "response_xai_1", - usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, - stopReason: "toolUse", - timestamp: 0, - }, - { - role: "toolResult", - toolCallId: "call_1|fc_1", - toolName: "screenshot", - content: [{ type: "image", mimeType: "image/png", data: "screenshot" }], - isError: false, - timestamp: 0, - }, - ]; - return { messages, tools: [], systemPrompt: "control the browser" }; -} - -describe("xAI Responses threading", () => { - it("threads the latest response and applies serial computer-use defaults", async () => { - const { context, onPayload } = threadXaiRequest(multiTurnContext(), undefined); - expect(context.messages).toHaveLength(1); - expect(await onPayload({ input: [] }, model)).toEqual({ - input: [], - store: true, - parallel_tool_calls: false, - previous_response_id: "response_xai_1", - }); - }); - - it("preserves encrypted reasoning replay", async () => { - const context: Context = { messages: [{ role: "user", content: "hello", timestamp: 0 }] }; - const { onPayload } = threadXaiRequest(context, undefined); - expect(await onPayload({ input: [], include: ["reasoning.encrypted_content"] }, model)).toEqual({ - input: [], - include: ["reasoning.encrypted_content"], - store: true, - parallel_tool_calls: false, - }); - }); - - it("sends full history without previous_response_id when threading is disabled", async () => { - const context = multiTurnContext(); - const prepared = threadXaiRequest(context, { - disableResponseThreading: true, - onPayload: (payload) => ({ ...(payload as object), previous_response_id: "foreign-response" }), - }); - expect(prepared.context).toBe(context); - expect(await prepared.onPayload({ input: [] }, model)).toEqual({ - input: [], - store: true, - parallel_tool_calls: false, - }); - }); - - it("reapplies xAI constraints after caller payload hooks", async () => { - const { onPayload } = threadXaiRequest(multiTurnContext(), { - onPayload: (payload) => ({ - ...(payload as object), - store: false, - parallel_tool_calls: true, - previous_response_id: "wrong-response", - include: ["reasoning.encrypted_content"], - }), - }); - expect(await onPayload({}, model)).toEqual({ - store: true, - parallel_tool_calls: false, - previous_response_id: "response_xai_1", - include: ["reasoning.encrypted_content"], - }); - }); -}); diff --git a/packages/ai/vitest.config.ts b/packages/ai/vitest.config.ts index 219048aa..25582a14 100644 --- a/packages/ai/vitest.config.ts +++ b/packages/ai/vitest.config.ts @@ -11,5 +11,9 @@ export default defineConfig({ // Unit runs cover every test file except the opt-in suites; use // vitest.integration.config.ts to run those. exclude: [...configDefaults.exclude, "**/*.integration.test.ts", "**/*.live.test.ts"], + // pi-ai ships real ESM and imports "openai" itself; both need to run + // through Vitest's module graph (not Node's native loader) for + // vi.mock("openai") to intercept requests pi's builtin transport makes. + server: { deps: { inline: [/@earendil-works\/pi-ai/, /^openai$/] } }, }, }); diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 94b8418f..873ed864 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 0.10.0 - 2026-08-13 + +- `--print -o jsonl` schema bumps to version 2: every assistant message now + also emits an `assistant_usage` event (`turn`, `model`, `api`, `input`, + `output`, `cache_read`, `cache_write`, `reasoning`, `total_tokens`, and a + derived `cache_hit_ratio`), including tool-only turns with no text. + ## 0.9.0 - 2026-08-04 Breaking: upgrade the pi stack to 0.83.0 (`pi-ai`, `pi-agent-core`, diff --git a/packages/cli/README.md b/packages/cli/README.md index 134727b0..e08e49ad 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -204,10 +204,18 @@ Add `--jsonl-include-deltas` for assistant-token deltas and The first event of every `--print -o jsonl` run is `session_created` with a `schema_version` field. The current schema -version is `1`. The `model` field carries a provider-qualified ref +version is `2`. The `model` field carries a provider-qualified ref (e.g. `openai:gpt-5.6-sol`); use `parseCuaModelRef` from `@onkernel/cua-ai` if you only need the bare model id. +Every assistant message also emits an `assistant_usage` event (including +tool-only turns with no text): `turn`, `model`, `api`, `input`, `output`, +`cache_read`, `cache_write`, `reasoning`, `total_tokens`, and +`cache_hit_ratio`. OpenAI's billed prompt tokens are `input + cache_read + +cache_write` (the provider already subtracts cached and cache-write tokens +out of `input`), so `cache_hit_ratio` is `cache_read` over that total, +reported as `0` when the total is `0`. + ## Sessions and transcripts `--print`, the interactive TUI, and any `-s ` invocation persist diff --git a/packages/cli/src/output/harness-jsonl.ts b/packages/cli/src/output/harness-jsonl.ts index c3bf79a2..f4ab623c 100644 --- a/packages/cli/src/output/harness-jsonl.ts +++ b/packages/cli/src/output/harness-jsonl.ts @@ -2,13 +2,14 @@ import type { AgentHarnessEvent, KernelBrowser, } from "@onkernel/cua-agent"; +import type { Usage } from "@onkernel/cua-ai"; import type { CuaCliHarness } from "../harness"; /** * Schema version stamped on every `session_created` event. Bump when the * jsonl shape changes in a way external consumers need to detect. */ -export const CUA_JSONL_SCHEMA_VERSION = 1; +export const CUA_JSONL_SCHEMA_VERSION = 2; export interface JsonlSinkOptions { harness: CuaCliHarness; @@ -90,6 +91,7 @@ export function attachHarnessJsonlSink(opts: JsonlSinkOptions): () => void { } else if (msg.role === "assistant") { const text = textOf(msg.content); if (text) emit({ type: "assistant_text_done", text, ts: Date.now() }); + emit({ type: "assistant_usage", turn, model: msg.model, api: msg.api, ...usageFields(msg.usage), ts: Date.now() }); } return; } @@ -155,6 +157,24 @@ export function attachHarnessJsonlSink(opts: JsonlSinkOptions): () => void { }); } +/** + * OpenAI's `input_tokens` (and pi-ai's `Usage.input`) already excludes cached + * and cache-write tokens, so the billed prompt is `input + cacheRead + + * cacheWrite` and the cache hit ratio is `cacheRead` over that total. + */ +function usageFields(usage: Usage): Record { + const billedPrompt = usage.input + usage.cacheRead + usage.cacheWrite; + return { + input: usage.input, + output: usage.output, + cache_read: usage.cacheRead, + cache_write: usage.cacheWrite, + reasoning: usage.reasoning, + total_tokens: usage.totalTokens, + cache_hit_ratio: billedPrompt > 0 ? usage.cacheRead / billedPrompt : 0, + }; +} + function textOf(content: unknown): string { if (typeof content === "string") return content; if (!Array.isArray(content)) return ""; diff --git a/packages/cli/test/fixtures/scripted-provider.ts b/packages/cli/test/fixtures/scripted-provider.ts index d6763fa1..6c2ff2fe 100644 --- a/packages/cli/test/fixtures/scripted-provider.ts +++ b/packages/cli/test/fixtures/scripted-provider.ts @@ -7,6 +7,7 @@ import { type CuaSimpleStreamOptions, type Model, type MutableModels, + type Usage, } from "@onkernel/cua-ai"; /** One scripted step replayed when the harness asks the provider for a turn. */ @@ -23,6 +24,8 @@ export interface ScriptedTurn { * calls and to "toolUse" otherwise. */ stopReason?: "stop" | "toolUse" | "length"; + /** Overrides the zeroed default usage on the resulting assistant message. */ + usage?: Partial; } export interface ScriptedProviderHandle { @@ -88,7 +91,7 @@ export function createScriptedCuaModels(providerId: string, turns: ScriptedTurn[ function buildStream(model: Model, turn: ScriptedTurn | undefined, signal?: AbortSignal) { const stream = createAssistantMessageEventStream(); void (async () => { - const message = baseAssistantMessage(model); + const message = baseAssistantMessage(model, turn?.usage); if (!turn) { message.stopReason = "stop"; stream.push({ type: "start", partial: message }); @@ -222,7 +225,7 @@ async function waitForAbort(signal?: AbortSignal): Promise { }); } -function baseAssistantMessage(model: Model): AssistantMessage { +function baseAssistantMessage(model: Model, usage?: Partial): AssistantMessage { return { role: "assistant", content: [], @@ -236,6 +239,7 @@ function baseAssistantMessage(model: Model): AssistantMessage { cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + ...usage, }, stopReason: "stop", timestamp: Date.now(), diff --git a/packages/cli/test/print.test.ts b/packages/cli/test/print.test.ts index f022a691..01f1e8f2 100644 --- a/packages/cli/test/print.test.ts +++ b/packages/cli/test/print.test.ts @@ -34,7 +34,30 @@ describe("runPrint", () => { expect(types).toContain("assistant_text_done"); expect(types).toContain("turn_done"); expect(types).toContain("run_complete"); - expect((events[0] as { schema_version: number }).schema_version).toBe(1); + expect((events[0] as { schema_version: number }).schema_version).toBe(2); + }); + + it("emits assistant_usage with the billed-prompt cache hit ratio", async () => { + fixture = await buildTestHarness({ + turns: [ + { + steps: [{ type: "text", text: "ok" }], + usage: { input: 100, output: 20, cacheRead: 300, cacheWrite: 50, totalTokens: 420 }, + }, + ], + }); + const events = await runPrintAsJsonl(fixture, "go"); + const usage = events.find((e) => e.type === "assistant_usage") as Record; + expect(usage).toMatchObject({ + turn: 1, + input: 100, + output: 20, + cache_read: 300, + cache_write: 50, + total_tokens: 420, + }); + // billed prompt = input + cache_read + cache_write = 450; ratio = cache_read / billed prompt. + expect(usage.cache_hit_ratio).toBeCloseTo(300 / 450); }); it("returns exit code 1 when the provider emits an error", async () => {