From 85ed1f233de1f7905f2e10ddf24d9b3c70fa8fd4 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 13 Aug 2026 15:12:08 +0000 Subject: [PATCH 1/3] Stream OpenAI through pi-ai's builtin Responses transport OpenAI models were routed to a CUA-owned `openai-cua-responses` api that threaded `previous_response_id` with `store: true` and sent only the messages after the last stored response. That threading did not reduce billed prompt tokens: a threaded request still bills the reconstructed prompt, with the matched prefix charged at the cache-read rate. pi-ai's builtin Responses transport gets the same caching from `prompt_cache_key` and `prompt_cache_retention` with `store: false`, and replays reasoning through `reasoning.encrypted_content`. OpenAI models now keep pi-ai's builtin `openai-responses` api id and stream through its transport by default. The CUA adapter is retained but dispatches on request shape instead of a rerouted api id: OpenAI's native computer tool, or a transcript carrying a deferred tool-search addition or a replayed function-call namespace, neither of which pi-ai 0.83.0 round-trips. Escalated requests now send pi's `prompt_cache_options` and strict-mode/grammar tool conversion so the cached prefix survives a mid-conversation switch between the two paths. Google, Meta, xAI, and Tzafon threading is unchanged; those providers either own their whole transport or thread a provider-specific continuation field with no builtin equivalent. Two consequences handled here: - Stateless replay means OpenAI's native computer results are no longer held in provider-stored state, so `computer` joins Tzafon in the exemption from the tool-result image replay limit. Its `computer_call_output` items must each carry a screenshot. - Transcript namespaces are now paired by `call_id` rather than by position, because a dropped aborted assistant message shifted every later namespace once the full transcript is converted. `--print -o jsonl` gains an `assistant_usage` event (schema version 2) with input, output, cache read/write, reasoning, and a derived cache hit ratio, so the caching change can be measured. No live A/B was run: this environment has no OpenAI or Kernel credentials. --- docs/architecture.md | 5 +- packages/agent/CHANGELOG.md | 12 ++ packages/agent/src/agent.ts | 21 +- packages/agent/test/agent.test.ts | 28 +++ .../agent/test/openai-deferred-tools.test.ts | 1 - packages/agent/vitest.config.ts | 4 + packages/ai/CHANGELOG.md | 18 ++ packages/ai/README.md | 5 +- packages/ai/src/index.ts | 1 - packages/ai/src/models.ts | 11 +- packages/ai/src/providers.ts | 27 +-- packages/ai/src/providers/openai/provider.ts | 97 +++++---- packages/ai/test/models.test.ts | 14 +- .../ai/test/openai-adapter-routing.test.ts | 187 ++++++++++++++++++ .../ai/test/openai-native-provider.test.ts | 17 +- packages/ai/test/openai-threading.test.ts | 114 ----------- packages/ai/test/providers.test.ts | 8 +- packages/ai/vitest.config.ts | 4 + packages/cli/CHANGELOG.md | 7 + packages/cli/README.md | 10 +- packages/cli/src/output/harness-jsonl.ts | 22 ++- .../cli/test/fixtures/scripted-provider.ts | 8 +- packages/cli/test/print.test.ts | 25 ++- 23 files changed, 441 insertions(+), 205 deletions(-) create mode 100644 packages/ai/test/openai-adapter-routing.test.ts delete mode 100644 packages/ai/test/openai-threading.test.ts diff --git a/docs/architecture.md b/docs/architecture.md index 1b3dfe81..f42bf74b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -153,8 +153,9 @@ catalog: - Ordinary function tools stay ordinary. - Anthropic native browser/computer declarations replace only their own placeholders and merge required beta headers with caller headers. -- OpenAI native computer uses a CUA-owned Responses adapter and can coexist with - ordinary functions. +- OpenAI streams through pi's builtin Responses transport and its automatic + prompt caching; a CUA-owned adapter handles OpenAI's native computer tool and + tool-search namespace round-trips. - Tzafon replaces only the selected computer identity and fills declaration dimensions from the actual viewport. - Anthropic's native browser tool falls back to an equivalent function-tool diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index 78f9f332..134d2d13 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 0.11.0 - 2026-08-13 + +- `responseThreading` (`CuaAgentOptions`/`CuaAgentHarnessOptions`) no longer + affects OpenAI models: OpenAI now streams through pi-ai's builtin Responses + transport and its automatic prompt caching regardless of this flag. The + option still governs Google, Meta, xAI, and Tzafon's + `previous_response_id`-style continuation. +- Exempt OpenAI's native computer tool from the tool-result image replay + limit, alongside Tzafon: its `computer_call_output` items must each carry a + screenshot, and stateless replay no longer leaves them in provider-stored + state. + ## 0.10.0 - 2026-08-04 Breaking: upgrade `@earendil-works/pi-agent-core` and `@earendil-works/pi-ai` diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index 9e68b96c..d3317fdd 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -92,6 +92,7 @@ export type CuaAgentOptions = Omit & streamFn?: AgentOptions["streamFn"]; emptyResponseRecovery?: CuaEmptyResponseRecoveryOptions; toolResultImageReplayLimit?: ToolResultImageReplayLimit; + /** Governs Google, Meta, xAI, and Tzafon's `previous_response_id`-style continuation. OpenAI streams through pi's automatic prompt caching regardless of this flag. Defaults to `true`. */ responseThreading?: boolean; retry?: CuaRetryOptions; }; @@ -112,6 +113,7 @@ type CuaAgentHarnessOptionsBase< onPayload?: SimpleStreamOptions["onPayload"]; emptyResponseRecovery?: CuaEmptyResponseRecoveryOptions; toolResultImageReplayLimit?: ToolResultImageReplayLimit; + /** Governs Google, Meta, xAI, and Tzafon's `previous_response_id`-style continuation. OpenAI streams through pi's automatic prompt caching regardless of this flag. Defaults to `true`. */ responseThreading?: boolean; retry?: CuaRetryOptions; }; @@ -203,7 +205,7 @@ export class CuaAgent { transformContext: async (messages, signal) => projectToolResultImages( transformContext ? await transformContext(messages, signal) : messages, imageReplayLimit, - manager.catalog.incoming.tzafonComputerName, + requiredImageToolNames(manager.catalog.incoming), ), prepareNextTurnWithContext: async (context, signal) => { const update = prepareNextTurnWithContext @@ -483,7 +485,7 @@ function withCatalogModels( const contextFor = (context: Context) => projectModelContext( context, imageReplayLimit, - manager.catalog.incoming.tzafonComputerName, + requiredImageToolNames(manager.catalog.incoming), ); const optionsFor = (options: T): T => { const catalog = manager.catalog; @@ -525,15 +527,20 @@ function resolveToolResultImageReplayLimit(limit: ToolResultImageReplayLimit | u return limit; } +/** Native computer tool names whose screenshot history the provider protocol requires in full, regardless of the image replay limit. */ +function requiredImageToolNames(incoming: CuaIncomingToolPlan): ReadonlySet { + return new Set([incoming.tzafonComputerName, incoming.openaiComputerName].filter((name): name is string => !!name)); +} + function projectToolResultImages( messages: TMessage[], limit: ToolResultImageReplayLimit, - requiredImageToolName?: string, + requiredToolNames: ReadonlySet = new Set(), ): TMessage[] { if (limit === false) return messages; let imageCount = 0; for (const message of messages) { - if (message.role === "toolResult" && message.toolName !== requiredImageToolName) { + if (message.role === "toolResult" && !requiredToolNames.has(message.toolName)) { imageCount += message.content.filter((block) => block.type === "image").length; } } @@ -541,7 +548,7 @@ function projectToolResultImages( const firstRetainedImage = Math.max(0, imageCount - limit); let imageOrdinal = 0; return messages.map((message) => { - if (message.role !== "toolResult" || message.toolName === requiredImageToolName) return message; + if (message.role !== "toolResult" || requiredToolNames.has(message.toolName)) return message; let changed = false; let markerInserted = false; const content = [] as typeof message.content; @@ -563,9 +570,9 @@ function projectToolResultImages( function projectModelContext( context: Context, imageReplayLimit: ToolResultImageReplayLimit, - requiredImageToolName?: string, + requiredToolNames: ReadonlySet, ): Context { - const messages = projectToolResultImages(context.messages, imageReplayLimit, requiredImageToolName); + const messages = projectToolResultImages(context.messages, imageReplayLimit, requiredToolNames); return messages === context.messages ? context : { ...context, messages }; } diff --git a/packages/agent/test/agent.test.ts b/packages/agent/test/agent.test.ts index 395e23ad..ddd61f61 100644 --- a/packages/agent/test/agent.test.ts +++ b/packages/agent/test/agent.test.ts @@ -132,6 +132,34 @@ describe("CuaAgent explicit tools", () => { expect(results[1]!.content).toEqual([{ type: "text", text: "[stale tool-result images omitted]" }]); }); + it("retains protocol-required OpenAI native computer screenshot results outside the image replay limit", async () => { + const contexts: Context[] = []; + const model = getCuaModel("openai:gpt-5.5"); + const nativeComputer = cua.providers.openai.tools.computer(); + const ordinaryTool = callerTool("ordinary"); + const image = { type: "image" as const, data: "c2NyZWVuc2hvdA==", mimeType: "image/png" }; + const messages: AgentMessage[] = [ + assistant(model, [{ type: "toolCall", id: "native-shot", name: "computer", arguments: { action: { type: "screenshot" } } }], "toolUse"), + { role: "toolResult", toolCallId: "native-shot", toolName: "computer", content: [image], isError: false, timestamp: 1 }, + assistant(model, [{ type: "toolCall", id: "ordinary-shot", name: "ordinary", arguments: {} }], "toolUse"), + { role: "toolResult", toolCallId: "ordinary-shot", toolName: "ordinary", content: [image], isError: false, timestamp: 2 }, + ]; + const agent = new CuaAgent({ + browser, + client, + tools: [nativeComputer, ordinaryTool], + streamFn: scriptedStream([(selectedModel) => assistant(selectedModel)], contexts), + toolResultImageReplayLimit: 0, + initialState: { model, messages }, + }); + + await agent.prompt("continue"); + + const results = contexts[0]!.messages.filter((message) => message.role === "toolResult"); + expect(results[0]!.content).toEqual([image]); + expect(results[1]!.content).toEqual([{ type: "text", text: "[stale tool-result images omitted]" }]); + }); + it("installs exact native-only, native-plus-CUA, Playwright-only, and browser-act-only catalogs", () => { const custom = callerTool("customer_lookup"); const cases = [ diff --git a/packages/agent/test/openai-deferred-tools.test.ts b/packages/agent/test/openai-deferred-tools.test.ts index 60ba0bfc..962725d8 100644 --- a/packages/agent/test/openai-deferred-tools.test.ts +++ b/packages/agent/test/openai-deferred-tools.test.ts @@ -86,7 +86,6 @@ describe("OpenAI deferred tool namespace continuation", () => { tools: [loader], initialState: { model: "openai:gpt-5.5" }, getApiKey: () => "test", - responseThreading: false, }); await agent.prompt("load and run the added tool"); diff --git a/packages/agent/vitest.config.ts b/packages/agent/vitest.config.ts index 039b5887..ab19cb94 100644 --- a/packages/agent/vitest.config.ts +++ b/packages/agent/vitest.config.ts @@ -8,5 +8,9 @@ export default defineConfig({ globals: true, environment: "node", testTimeout: 30000, + // pi-ai ships real ESM and imports "openai" itself; both need to run + // through Vitest's module graph (not Node's native loader) for + // vi.mock("openai") to intercept requests pi's builtin transport makes. + server: { deps: { inline: [/@earendil-works\/pi-ai/, /^openai$/] } }, }, }); diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 2d336ad6..43acec14 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## 0.11.0 - 2026-08-13 + +Breaking: OpenAI models no longer carry a CUA-owned api id. + +- OpenAI models now resolve to pi-ai's builtin `"openai-responses"` api + instead of the removed `openai-cua-responses`, and stream through pi's + builtin Responses transport (`store: false`, automatic prompt-cache-key + matching) by default. `OPENAI_CUA_RESPONSES_API` and the OpenAI adapter's + `previous_response_id` threading are removed; `previous_response_id` and + `store: true` no longer appear on any OpenAI request. +- The CUA-owned OpenAI adapter is retained but now dispatches on request + shape rather than a rerouted api id: it only intercepts requests that select + OpenAI's native computer tool, or whose transcript carries a deferred + tool-search addition or a replayed function-call namespace (pi-ai 0.83.0's + builtin transport does not round-trip either). +- Google, Meta, xAI, and Tzafon are unaffected: their `previous_response_id` + threading helpers in `providers/common.ts` are unchanged. + ## 0.10.0 - 2026-08-04 Breaking: upgrade `@earendil-works/pi-ai` to 0.83.0. diff --git a/packages/ai/README.md b/packages/ai/README.md index 23542c50..aef7db5e 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -240,8 +240,9 @@ additions through pi's active-tool change entries. ## Provider behavior -- **OpenAI**: CUA-owned Responses transport for native computer plus ordinary - function composition and response threading. +- **OpenAI**: streams through pi's builtin Responses transport and its + automatic prompt caching by default. A CUA-owned adapter is used only for + OpenAI's native computer tool and for tool-search namespace round-trips. - **Anthropic**: exact native declarations, beta-header composition, and adaptive model preparation. - **Google**: a CUA-owned Interactions API adapter plus the current predefined diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index 1823aad1..cd4e8c04 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -5,7 +5,6 @@ export { cuaModels, GOOGLE_CUA_INTERACTIONS_API, META_RESPONSES_API, - OPENAI_CUA_RESPONSES_API, streamGoogleInteractions, streamMetaResponses, streamOpenAIResponses, diff --git a/packages/ai/src/models.ts b/packages/ai/src/models.ts index 07049201..34fffca2 100644 --- a/packages/ai/src/models.ts +++ b/packages/ai/src/models.ts @@ -2,7 +2,6 @@ import type { Api, Model } from "@earendil-works/pi-ai"; import { getBuiltinModel, getBuiltinModels } from "@earendil-works/pi-ai/providers/all"; import { GOOGLE_CUA_INTERACTIONS_API } from "./providers/google/provider"; import { META_RESPONSES_API } from "./providers/meta/provider"; -import { OPENAI_CUA_RESPONSES_API } from "./providers/openai/provider"; import { XAI_CUA_RESPONSES_API } from "./providers/xai/provider"; /** Providers with curated computer-use model support. */ @@ -229,15 +228,15 @@ export function getCuaModel(ref: CuaModelRef): Model { throw new Error(`CUA model "${ref}" is supported but not registered. Add it to pi-ai (models.dev) or CUA_MODEL_OVERRIDES.`); } -// Route CUA models to provider-specific transports. Registry-resolved models -// otherwise carry pi-ai's builtin API ids. +// Route CUA models to provider-specific transports. OpenAI keeps pi-ai's +// builtin "openai-responses" api and streams through its automatic prompt +// caching; the CUA adapter dispatches on request shape (see +// requiresCuaOpenAIAdapter), not on a rerouted api id. Other registry-resolved +// models otherwise carry pi-ai's builtin API ids too. export function routeCuaApi(model: Model): Model { if (model.provider === "google" && model.api !== GOOGLE_CUA_INTERACTIONS_API) { return { ...model, api: GOOGLE_CUA_INTERACTIONS_API }; } - if (model.provider === "openai" && model.api !== OPENAI_CUA_RESPONSES_API) { - return { ...model, api: OPENAI_CUA_RESPONSES_API }; - } if (model.provider === "meta" && model.api !== META_RESPONSES_API) { return { ...model, api: META_RESPONSES_API }; } diff --git a/packages/ai/src/providers.ts b/packages/ai/src/providers.ts index b51862aa..37835409 100644 --- a/packages/ai/src/providers.ts +++ b/packages/ai/src/providers.ts @@ -16,7 +16,7 @@ import { cuaOverrideModels } from "./models"; import { withAnthropicBrowserFallback } from "./providers/anthropic/browser-fallback"; import { GOOGLE_CUA_INTERACTIONS_API, streamGoogleInteractions, streamSimpleGoogleInteractions } from "./providers/google/provider"; import { META_RESPONSES_API, streamMetaResponses, streamSimpleMetaResponses } from "./providers/meta/provider"; -import { OPENAI_CUA_RESPONSES_API, streamOpenAIResponses, streamSimpleOpenAIResponses } from "./providers/openai/provider"; +import { requiresCuaOpenAIAdapter, streamOpenAIResponses, streamSimpleOpenAIResponses } from "./providers/openai/provider"; import { streamSimpleTzafonResponses, streamTzafonResponses, TZAFON_RESPONSES_API } from "./providers/tzafon/provider"; import { streamSimpleXaiResponses, streamXaiResponses, XAI_CUA_RESPONSES_API } from "./providers/xai/provider"; import { streamSimpleYutori, streamYutori, YUTORI_CHAT_COMPLETIONS_API } from "./providers/yutori/provider"; @@ -27,10 +27,11 @@ import { streamSimpleYutori, streamYutori, YUTORI_CHAT_COMPLETIONS_API } from ". * * - `anthropic` retries an inaccessible native browser beta through the * selected tool's equivalent function declaration. - * - `openai` intercepts the `openai-cua-responses` api that - * {@link getCuaModel} routes OpenAI models to, threading - * `previous_response_id`; every other api falls through to pi's builtin - * provider. + * - `openai` streams through pi's builtin `openai-responses` transport and + * its automatic prompt caching by default. The CUA adapter only intercepts + * requests that need it: OpenAI's native computer tool, or a transcript + * carrying a deferred tool-search addition or a replayed function-call + * namespace (see {@link requiresCuaOpenAIAdapter}). * - `google` intercepts `google-cua-interactions` for current native computer * use and resolves API keys from `GOOGLE_API_KEY` or `GEMINI_API_KEY`. * - `xai` intercepts `xai-cua-responses` so Grok can use stateful Responses @@ -47,7 +48,7 @@ export function createCuaModels(options?: CreateModelsOptions): MutableModels { const anthropic = models.getProvider("anthropic"); if (anthropic) models.setProvider(withAnthropicBrowserFallback(anthropic)); const openai = models.getProvider("openai"); - if (openai) models.setProvider(withOpenAICuaResponses(openai)); + if (openai) models.setProvider(withOpenAICuaComputerAdapter(openai)); const google = models.getProvider("google"); if (google) models.setProvider(withGoogleCuaInteractions(google)); const xai = models.getProvider("xai"); @@ -72,18 +73,18 @@ export function cuaModels(): MutableModels { return (defaultCuaModels ??= createCuaModels()); } -// pi's builtin openai provider only streams its own api ids. CUA routes -// OpenAI models to OPENAI_CUA_RESPONSES_API (see routeCuaApi), so the -// registered provider must dispatch that api to cua's threading stream fns. -function withOpenAICuaResponses(base: Provider): Provider { +// OpenAI models keep pi-ai's builtin "openai-responses" api id. Only requests +// that need cua-ai's adapter (see requiresCuaOpenAIAdapter) are intercepted; +// everything else falls through to pi's builtin provider. +function withOpenAICuaComputerAdapter(base: Provider): Provider { return { ...base, stream: (model: Model, context: Context, options?: StreamOptions) => - model.api === OPENAI_CUA_RESPONSES_API + model.api === "openai-responses" && requiresCuaOpenAIAdapter(context, options) ? streamOpenAIResponses(model as never, context, options) : base.stream(model, context, options), streamSimple: (model: Model, context: Context, options?: SimpleStreamOptions) => - model.api === OPENAI_CUA_RESPONSES_API + model.api === "openai-responses" && requiresCuaOpenAIAdapter(context, options) ? streamSimpleOpenAIResponses(model as never, context, options) : base.streamSimple(model, context, options), }; @@ -153,7 +154,7 @@ function yutoriProvider(): Provider { export { GOOGLE_CUA_INTERACTIONS_API, streamGoogleInteractions, streamSimpleGoogleInteractions }; export { META_RESPONSES_API, streamMetaResponses, streamSimpleMetaResponses }; -export { OPENAI_CUA_RESPONSES_API, streamOpenAIResponses, streamSimpleOpenAIResponses }; +export { streamOpenAIResponses, streamSimpleOpenAIResponses }; export { TZAFON_RESPONSES_API, streamSimpleTzafonResponses, streamTzafonResponses }; export { XAI_CUA_RESPONSES_API, streamSimpleXaiResponses, streamXaiResponses }; export { YUTORI_CHAT_COMPLETIONS_API, streamSimpleYutori, streamYutori }; diff --git a/packages/ai/src/providers/openai/provider.ts b/packages/ai/src/providers/openai/provider.ts index 6e4047e2..1fb0e85d 100644 --- a/packages/ai/src/providers/openai/provider.ts +++ b/packages/ai/src/providers/openai/provider.ts @@ -19,40 +19,47 @@ import { convertResponsesTools, processResponsesStream, } from "@earendil-works/pi-ai/api/openai-responses-shared"; +import { createGrammarToolInputProperties } from "@earendil-works/pi-ai/api/constrained-sampling"; import { clampOpenAIPromptCacheKey } from "@earendil-works/pi-ai/api/openai-prompt-cache"; import { buildBaseOptions } from "@earendil-works/pi-ai/api/simple-options"; import type { CuaIncomingToolPlan } from "../../tool-catalog"; -import { - type CuaSimpleStreamOptions, - type ResponsesThreadingOptions, - threadResponsesRequest, -} from "../common"; - -export const OPENAI_CUA_RESPONSES_API = "openai-cua-responses"; +import type { CuaSimpleStreamOptions } from "../common"; -export interface OpenAIResponsesOptions extends PiOpenAIResponsesOptions, ResponsesThreadingOptions { +export interface OpenAIResponsesOptions extends PiOpenAIResponsesOptions { /** @internal Identity-addressed native dispatch compiled from selected tools. */ cuaIncomingToolPlan?: CuaIncomingToolPlan; } -export function threadRequest(context: Context, options: ResponsesThreadingOptions | undefined) { - const { context: threadedContext, onPayload } = threadResponsesRequest(context, OPENAI_CUA_RESPONSES_API, options); - return { context: threadedContext, onPayload }; +/** + * Whether a request needs cua-ai's OpenAI Responses adapter instead of pi-ai's + * builtin `openai-responses` transport: OpenAI's native computer tool is + * selected, or the transcript carries state (a deferred tool-search addition, + * or a replayed function-call namespace) that only this adapter round-trips. + */ +export function requiresCuaOpenAIAdapter(context: Context, options?: unknown): boolean { + // pi's Provider.stream signature hides cua's added option, so the plan is + // read structurally here rather than at both call sites. + const cuaIncomingToolPlan = (options as { cuaIncomingToolPlan?: CuaIncomingToolPlan } | undefined)?.cuaIncomingToolPlan; + if (cuaIncomingToolPlan?.openaiComputerName) return true; + for (const message of context.messages) { + if (message.role === "toolResult" && (message.addedToolNames?.length ?? 0) > 0) return true; + if (message.role === "assistant" && message.content.some((part) => part.type === "toolCall" && toolCallNamespace(part))) return true; + } + return false; } -export const streamOpenAIResponses: StreamFunction = (model, context, options) => { +export const streamOpenAIResponses: StreamFunction<"openai-responses", OpenAIResponsesOptions> = (model, context, options) => { if (options?.cuaIncomingToolPlan?.openaiComputerName) return streamOpenAINativeComputer(model, context, options); return streamOpenAIFunctionTools(model, context, options); }; -export const streamSimpleOpenAIResponses: StreamFunction = (model, context, options) => { +export const streamSimpleOpenAIResponses: StreamFunction<"openai-responses", CuaSimpleStreamOptions> = (model, context, options) => { if (options?.cuaIncomingToolPlan?.openaiComputerName) return streamOpenAINativeComputer(model, context, options); const base = buildBaseOptions(model, context, options, options?.apiKey); const clampedReasoning = options?.reasoning ? clampThinkingLevel(model, options.reasoning) : undefined; return streamOpenAIFunctionTools(model, context, { ...base, reasoningEffort: clampedReasoning === "off" ? undefined : clampedReasoning, - disableResponseThreading: options?.disableResponseThreading, cuaIncomingToolPlan: options?.cuaIncomingToolPlan, }); }; @@ -60,10 +67,10 @@ export const streamSimpleOpenAIResponses: StreamFunction, + model: Model<"openai-responses">, context: Context, options: OpenAIResponsesOptions | undefined, ) { @@ -72,11 +79,10 @@ function streamOpenAIFunctionTools( void (async () => { try { const apiKey = openAIApiKey(options); - const threaded = threadRequest(context, options); - const compat = model.compat as { supportsToolSearch?: boolean } | undefined; - const placement = splitDeferredTools(threaded.context, compat?.supportsToolSearch === true); - let payload = buildFunctionPayload(model, threaded.context, options, placement); - payload = (await threaded.onPayload(payload, model)) as Record; + const placement = splitDeferredTools(context, model.compat?.supportsToolSearch === true); + const grammarToolInputProperties = createGrammarToolInputProperties(context.tools, model.compat?.supportsOpenAIGrammarTools === true); + let payload = buildFunctionPayload(model, context, options, placement, grammarToolInputProperties); + payload = ((await options?.onPayload?.(payload, model)) ?? payload) as Record; const client = createOpenAIClient(model, options, apiKey); const request = client.responses.create(payload as never, { ...(options?.signal ? { signal: options.signal } : {}), @@ -102,6 +108,7 @@ function streamOpenAIFunctionTools( { serviceTier: options?.serviceTier, applyServiceTierPricing: (usage, tier) => applyServiceTierPricing(usage, tier, model), + grammarToolInputProperties, }, ); applyToolCallNamespaces(output, namespaces); @@ -121,27 +128,34 @@ function streamOpenAIFunctionTools( } function buildFunctionPayload( - model: Model, + model: Model<"openai-responses">, context: Context, options: OpenAIResponsesOptions | undefined, placement: { immediate: Tool[]; deferred: Map }, + grammarToolInputProperties: ReadonlyMap, ): Record { - const input = convertResponsesMessages(model, context, new Set(["openai"]), { deferredTools: placement.deferred }); + const compat = model.compat; + const toolOptions = { supportsStrictMode: compat?.supportsStrictMode, supportsOpenAIGrammarTools: compat?.supportsOpenAIGrammarTools }; + const input = convertResponsesMessages(model, context, new Set(["openai"]), { + deferredTools: placement.deferred, + toolOptions, + grammarToolInputProperties, + }); applyTranscriptNamespaces(input as unknown as Array>, context.messages); const retention = cacheRetention(options); - const compat = model.compat as { supportsLongCacheRetention?: boolean } | undefined; const payload: Record = { model: model.id, input, stream: true, prompt_cache_key: retention === "none" ? undefined : clampOpenAIPromptCacheKey(options?.sessionId), prompt_cache_retention: retention === "long" && compat?.supportsLongCacheRetention !== false ? "24h" : undefined, + prompt_cache_options: retention === "none" && compat?.supportsExplicitPromptCacheMode ? { mode: "explicit" } : undefined, store: false, }; if (options?.maxTokens) payload.max_output_tokens = Math.max(options.maxTokens, 16); if (options?.temperature !== undefined) payload.temperature = options.temperature; if (options?.serviceTier !== undefined) payload.service_tier = options.serviceTier; - if (placement.immediate.length > 0) payload.tools = convertResponsesTools(placement.immediate); + if (placement.immediate.length > 0) payload.tools = convertResponsesTools(placement.immediate, toolOptions); if (options?.toolChoice !== undefined) payload.tool_choice = options.toolChoice; if (model.reasoning) { if (options?.reasoningEffort || options?.reasoningSummary) { @@ -158,7 +172,7 @@ function buildFunctionPayload( } function createOpenAIClient( - model: Model, + model: Model<"openai-responses">, options: OpenAIResponsesOptions | undefined, apiKey: string, ): OpenAI { @@ -190,13 +204,20 @@ function cacheRetention(options: OpenAIResponsesOptions | undefined): "none" | " } function applyTranscriptNamespaces(input: Array>, messages: readonly Context["messages"][number][]): void { - const calls = messages.flatMap((message) => message.role === "assistant" - ? message.content.filter((part): part is ToolCall => part.type === "toolCall") - : []); - let callIndex = 0; + const namespaces = new Map(); + for (const message of messages) { + if (message.role !== "assistant") continue; + for (const part of message.content) { + if (part.type !== "toolCall") continue; + const namespace = toolCallNamespace(part); + if (!namespace) continue; + const callId = part.id.split("|", 1)[0]!; + namespaces.set(callId, namespace); + } + } for (const item of input) { - if (item.type !== "function_call") continue; - const namespace = toolCallNamespace(calls[callIndex++]); + if (item.type !== "function_call" || typeof item.call_id !== "string") continue; + const namespace = namespaces.get(item.call_id); if (namespace) item.namespace = namespace; } } @@ -239,7 +260,7 @@ function toolCallNamespace(call: ToolCall | undefined): string | undefined { function applyServiceTierPricing( usage: AssistantMessage["usage"], serviceTier: OpenAIResponsesOptions["serviceTier"], - model: Model, + model: Model<"openai-responses">, ): void { const multiplier = serviceTier === "flex" ? 0.5 : serviceTier === "priority" ? (model.id === "gpt-5.5" ? 2.5 : 2) : 1; if (multiplier === 1) return; @@ -252,7 +273,7 @@ function applyServiceTierPricing( /** Responses adapter used only when the selected catalog contains OpenAI's native computer tool. */ function streamOpenAINativeComputer( - model: Model, + model: Model<"openai-responses">, context: Context, options: OpenAIResponsesOptions | CuaSimpleStreamOptions | undefined, ) { @@ -265,15 +286,15 @@ function streamOpenAINativeComputer( const nativeName = options?.cuaIncomingToolPlan?.openaiComputerName; if (!nativeName) throw new Error("OpenAI native computer incoming plan is missing"); const placement = splitDeferredTools(context); - const threaded = threadRequest(context, options); let payload: Record = { model: model.id, - instructions: threaded.context.systemPrompt, - input: convertMessages(threaded.context.messages, nativeName, placement.deferred), + instructions: context.systemPrompt, + input: convertMessages(context.messages, nativeName, placement.deferred), tools: convertTools(placement.immediate), max_output_tokens: options?.maxTokens ?? model.maxTokens, + store: false, }; - payload = (await threaded.onPayload(payload, model)) as Record; + payload = ((await options?.onPayload?.(payload, model)) ?? payload) as Record; const client = new OpenAI({ apiKey, baseURL: model.baseUrl || "https://api.openai.com/v1", diff --git a/packages/ai/test/models.test.ts b/packages/ai/test/models.test.ts index ddea4ba5..dfd1de0c 100644 --- a/packages/ai/test/models.test.ts +++ b/packages/ai/test/models.test.ts @@ -10,7 +10,6 @@ import { GOOGLE_CUA_INTERACTIONS_API, listCuaModels, META_RESPONSES_API, - OPENAI_CUA_RESPONSES_API, parseCuaModelRef, XAI_CUA_RESPONSES_API, } from "../src/index"; @@ -134,12 +133,13 @@ describe("CUA model refs", () => { expect(getCuaModel("yutori:n1.5-latest").api).toBe("yutori-chat-completions"); }); - it("routes Responses models to provider-specific threading APIs", () => { - // Registry models carry pi-ai's builtin "openai-responses" api and must - // be routed to provider-specific previous_response_id transports. - expect(getCuaModel("openai:gpt-5.6-sol").api).toBe(OPENAI_CUA_RESPONSES_API); - expect(getCuaModel("openai:gpt-5.5").api).toBe(OPENAI_CUA_RESPONSES_API); - expect(getCuaModel("openai:gpt-5.4-mini").api).toBe(OPENAI_CUA_RESPONSES_API); + it("routes Responses models to provider-specific transports, and keeps OpenAI on pi's builtin one", () => { + // OpenAI has a fully equivalent builtin transport (automatic prompt + // caching, no previous_response_id needed), so it keeps pi-ai's + // registry api id instead of being routed to a CUA-owned one. + expect(getCuaModel("openai:gpt-5.6-sol").api).toBe("openai-responses"); + expect(getCuaModel("openai:gpt-5.5").api).toBe("openai-responses"); + expect(getCuaModel("openai:gpt-5.4-mini").api).toBe("openai-responses"); expect(getCuaModel("google:gemini-3.6-flash").api).toBe(GOOGLE_CUA_INTERACTIONS_API); expect(getCuaModel("meta:muse-spark-1.1").api).toBe(META_RESPONSES_API); expect(getCuaModel("xai:grok-4.5").api).toBe(XAI_CUA_RESPONSES_API); diff --git a/packages/ai/test/openai-adapter-routing.test.ts b/packages/ai/test/openai-adapter-routing.test.ts new file mode 100644 index 00000000..74dcc56e --- /dev/null +++ b/packages/ai/test/openai-adapter-routing.test.ts @@ -0,0 +1,187 @@ +import { describe, expect, it, vi } from "vitest"; +import type { ToolCall } from "@earendil-works/pi-ai"; +import { createCuaModels, getCuaModel } from "../src/index"; + +const { responsesCreate } = vi.hoisted(() => ({ responsesCreate: vi.fn() })); + +vi.mock("openai", () => ({ + default: class { + responses = { + create: (...args: unknown[]) => ({ + withResponse: async () => ({ data: responsePayload(responsesCreate(...args)), response: { status: 200, headers: new Headers() } }), + }), + }; + }, +})); + +async function* responseEvents(response: Record) { + yield { type: "response.created", response: { id: response.id } }; + for (const [outputIndex, item] of ((response.output as unknown[]) ?? []).entries()) { + yield { type: "response.output_item.added", output_index: outputIndex, item }; + yield { type: "response.output_item.done", output_index: outputIndex, item }; + } + yield { type: "response.completed", response }; +} + +// pi's builtin transport iterates an event stream; the CUA native-computer +// adapter reads the raw response object's fields directly. Serve both from +// one mock so a single test file can exercise either dispatch path. +function responsePayload(response: Record): AsyncIterable & Record { + return Object.assign(responseEvents(response), response); +} + +const model = getCuaModel("openai:gpt-5.5"); +const tools = [{ name: "lookup", description: "lookup", parameters: { type: "object" } as never }]; + +describe("OpenAI adapter routing", () => { + it("streams a plain function-tool context through pi's builtin transport", async () => { + responsesCreate.mockReturnValueOnce({ + id: "resp_1", + status: "completed", + usage: { input_tokens: 1, output_tokens: 1 }, + output: [{ + type: "function_call", + id: "fc_1", + call_id: "call_1", + name: "lookup", + namespace: "deferred_tools", + arguments: "{}", + status: "completed", + }], + }); + const message = await createCuaModels().streamSimple(model, { + messages: [{ role: "user", content: "look it up", timestamp: 1 }], + tools, + }, { apiKey: "test", sessionId: "session_1" }).result(); + + // pi-ai 0.83.0's builtin Responses path does not parse the namespace + // field on function_call items, unlike the CUA adapter. + const call = message.content.find((part): part is ToolCall => part.type === "toolCall") as (ToolCall & { namespace?: string }) | undefined; + expect(call?.name).toBe("lookup"); + expect(call?.namespace).toBeUndefined(); + + // The measurement A/B depends on the default path relying on prompt + // caching instead of `previous_response_id` threading. + const payload = responsesCreate.mock.calls[0]![0] as Record; + expect(payload.store).toBe(false); + expect(payload.prompt_cache_key).toBe("session_1"); + expect(payload.previous_response_id).toBeUndefined(); + }); + + it("reaches the CUA adapter when the incoming plan selects OpenAI's native computer tool", async () => { + responsesCreate.mockReturnValueOnce({ + id: "resp_2", + output: [{ + type: "computer_call", + call_id: "computer_1", + action: { type: "click", x: 10, y: 20 }, + }], + }); + const message = await createCuaModels().streamSimple(model, { + messages: [{ role: "user", content: "click it", timestamp: 1 }], + tools: [{ name: "computer", description: "placeholder", parameters: { type: "object" } as never }], + }, { + apiKey: "test", + cuaIncomingToolPlan: { openaiComputerName: "computer", yutoriNames: {}, googleNames: {}, googleExcludedNames: [], nativeToolNames: ["computer"] }, + } as never).result(); + + // Only the CUA native-computer adapter understands computer_call items; + // pi's builtin transport has no case for them and would emit nothing. + const call = message.content.find((part): part is ToolCall => part.type === "toolCall"); + expect(call).toMatchObject({ name: "computer", arguments: { action: { type: "click", x: 10, y: 20 } } }); + }); + + it("reaches the CUA adapter when the transcript carries a deferred tool-search addition", async () => { + responsesCreate.mockReturnValueOnce({ + id: "resp_3", + status: "completed", + usage: { input_tokens: 1, output_tokens: 1 }, + output: [{ + type: "function_call", + id: "fc_3", + call_id: "call_3", + name: "lookup", + namespace: "deferred_tools", + arguments: "{}", + status: "completed", + }], + }); + const message = await createCuaModels().streamSimple(model, { + messages: [ + { role: "user", content: "load and look it up", timestamp: 1 }, + { + role: "toolResult", + toolCallId: "call_loader", + toolName: "loader", + content: [{ type: "text", text: "loaded" }], + isError: false, + addedToolNames: ["lookup"], + timestamp: 2, + }, + ], + tools, + }, { apiKey: "test" } as never).result(); + + // The CUA adapter round-trips the namespace pi's builtin drops. + const call = message.content.find((part): part is ToolCall => part.type === "toolCall") as (ToolCall & { namespace?: string }) | undefined; + expect(call?.namespace).toBe("deferred_tools"); + }); + + it("pairs replayed namespaces by call id, not ordinal, across an aborted assistant turn", async () => { + responsesCreate.mockReturnValueOnce({ id: "resp_4", usage: {}, output: [] }); + await createCuaModels().streamSimple(model, { + messages: [ + { role: "user", content: "load and look it up", timestamp: 1 }, + { + role: "assistant", + content: [{ type: "toolCall", id: "call_loader|fc_loader", name: "loader", arguments: {} }], + api: model.api, + provider: model.provider, + model: model.id, + usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + stopReason: "aborted", + timestamp: 2, + }, + { + role: "toolResult", + toolCallId: "call_loader|fc_loader", + toolName: "loader", + content: [{ type: "text", text: "loaded" }], + isError: false, + addedToolNames: ["lookup"], + timestamp: 3, + }, + { + role: "assistant", + content: [{ type: "toolCall", id: "call_lookup|fc_lookup", name: "lookup", arguments: {}, namespace: "deferred_tools" } as ToolCall], + api: model.api, + provider: model.provider, + model: model.id, + usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + stopReason: "toolUse", + timestamp: 4, + }, + { + role: "toolResult", + toolCallId: "call_lookup|fc_lookup", + toolName: "lookup", + content: [{ type: "text", text: "ok" }], + isError: false, + timestamp: 5, + }, + ], + tools, + }, { apiKey: "test" } as never).result(); + + // pi drops the aborted "loader" assistant message from the replayed + // transcript entirely, so pairing by ordinal would shift "lookup"'s + // namespace onto the missing "loader" call instead. + const payload = responsesCreate.mock.calls.at(-1)?.[0] as { input: Array> }; + expect(payload.input).toContainEqual(expect.objectContaining({ + type: "function_call", + call_id: "call_lookup", + name: "lookup", + namespace: "deferred_tools", + })); + }); +}); diff --git a/packages/ai/test/openai-native-provider.test.ts b/packages/ai/test/openai-native-provider.test.ts index 48c60c0a..7d110799 100644 --- a/packages/ai/test/openai-native-provider.test.ts +++ b/packages/ai/test/openai-native-provider.test.ts @@ -15,7 +15,7 @@ vi.mock("openai", () => ({ }, })); -const model = getCuaModel("openai:gpt-5.5") as Model; +const model = getCuaModel("openai:gpt-5.5") as Model<"openai-responses">; const incoming = { openaiComputerName: "computer", yutoriNames: {}, googleNames: {}, googleExcludedNames: [], nativeToolNames: ["computer"] }; describe("OpenAI native computer Responses adapter", () => { @@ -47,6 +47,10 @@ describe("OpenAI native computer Responses adapter", () => { pending_safety_checks: [{ id: "check_1", code: "malicious_instructions" }], }, }); + + const payload = responsesCreate.mock.calls.at(-1)?.[0] as Record; + expect(payload.store).toBe(false); + expect(payload.previous_response_id).toBeUndefined(); }); it("round-trips function-call namespaces beside the native computer adapter", async () => { @@ -91,15 +95,17 @@ describe("OpenAI native computer Responses adapter", () => { { name: "computer", description: "placeholder", parameters: { type: "object" } as never }, { name: "lookup", description: "lookup", parameters: { type: "object" } as never }, ], - }, { apiKey: "test", cuaIncomingToolPlan: incoming, disableResponseThreading: true }).result(); + }, { apiKey: "test", cuaIncomingToolPlan: incoming }).result(); - const payload = responsesCreate.mock.calls.at(-1)?.[0] as { input: Array> }; + const payload = responsesCreate.mock.calls.at(-1)?.[0] as { input: Array>; store?: unknown; previous_response_id?: unknown }; expect(payload.input).toContainEqual(expect.objectContaining({ type: "function_call", call_id: "call_lookup", name: "lookup", namespace: "deferred_tools", })); + expect(payload.store).toBe(false); + expect(payload.previous_response_id).toBeUndefined(); }); it("serializes native results as computer_call_output and ordinary results as function output", async () => { @@ -133,11 +139,10 @@ describe("OpenAI native computer Responses adapter", () => { }, { apiKey: "test", cuaIncomingToolPlan: incoming, - disableResponseThreading: true, onPayload: (payload) => ({ ...(payload as Record), tools: [{ type: "computer" }] }), }).result(); - const payload = responsesCreate.mock.calls.at(-1)?.[0] as { input: Array> }; + const payload = responsesCreate.mock.calls.at(-1)?.[0] as { input: Array>; store?: unknown; previous_response_id?: unknown }; expect(payload.input).toEqual(expect.arrayContaining([ expect.objectContaining({ type: "computer_call", call_id: "computer_1" }), expect.objectContaining({ @@ -147,5 +152,7 @@ describe("OpenAI native computer Responses adapter", () => { output: expect.objectContaining({ type: "computer_screenshot", image_url: expect.stringContaining("data:image/png;base64,") }), }), ])); + expect(payload.store).toBe(false); + expect(payload.previous_response_id).toBeUndefined(); }); }); diff --git a/packages/ai/test/openai-threading.test.ts b/packages/ai/test/openai-threading.test.ts deleted file mode 100644 index d41cbd07..00000000 --- a/packages/ai/test/openai-threading.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { describe, expect, it } from "vitest"; -import type { Context, Message, Model } from "@earendil-works/pi-ai"; -import { OPENAI_CUA_RESPONSES_API, threadRequest } from "../src/providers/openai/provider"; - -const TURNS = 6; -const model = {} as Model; - -/** Multi-turn context where each assistant turn carries a distinct responseId followed by a screenshot tool result. */ -function multiTurnContext(): Context { - const messages: Message[] = [{ role: "user", content: "book a flight", timestamp: 0 }]; - for (let turn = 0; turn < TURNS; turn += 1) { - messages.push({ - role: "assistant", - content: [{ type: "toolCall", id: `call_${turn}`, name: "click", arguments: { x: turn, y: turn } }], - api: OPENAI_CUA_RESPONSES_API, - provider: "openai", - model: "gpt-5.5", - responseId: `resp_${turn}`, - usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, - stopReason: "toolUse", - timestamp: 0, - }); - messages.push({ - role: "toolResult", - toolCallId: `call_${turn}`, - toolName: "click", - content: [{ type: "image", mimeType: "image/png", data: `screenshot-${turn}` }], - isError: false, - timestamp: 0, - }); - } - return { messages, tools: [], systemPrompt: "control the browser" }; -} - -describe("openai threadRequest", () => { - it("prunes to the delta and injects store + previous_response_id when threading (default)", async () => { - const { context, onPayload } = threadRequest(multiTurnContext(), undefined); - // Only the latest tool result (after the last assistant turn) is sent; the rest lives server-side. - expect(context.messages).toHaveLength(1); - expect((context.messages[0] as { toolCallId?: string }).toolCallId).toBe(`call_${TURNS - 1}`); - expect(await onPayload({ input: [] }, model)).toEqual({ input: [], store: true, previous_response_id: `resp_${TURNS - 1}` }); - }); - - it("sends full history with store but no previous_response_id when disabled by option", async () => { - const ctx = multiTurnContext(); - const { context, onPayload } = threadRequest(ctx, { disableResponseThreading: true }); - expect(context).toBe(ctx); - expect(await onPayload({}, model)).toEqual({ store: true }); - }); - - it("falls back to full history when the latest assistant turn lacks a responseId", async () => { - const ctx = multiTurnContext(); - ctx.messages.push({ - role: "assistant", - content: [{ type: "text", text: "request failed" }], - api: OPENAI_CUA_RESPONSES_API, - provider: "openai", - model: "gpt-5.5", - usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, - stopReason: "error", - timestamp: 0, - }); - const { context, onPayload } = threadRequest(ctx, undefined); - expect(context).toBe(ctx); - expect(((await onPayload({}, model)) as Record).previous_response_id).toBeUndefined(); - }); - - it("ignores a responseId from an errored turn so it never anchors previous_response_id", async () => { - const ctx = multiTurnContext(); - // An error after response.created can capture a responseId for a response the server never stored. - ctx.messages.push({ - role: "assistant", - content: [{ type: "text", text: "request failed" }], - api: OPENAI_CUA_RESPONSES_API, - provider: "openai", - model: "gpt-5.5", - responseId: "resp_failed", - usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, - stopReason: "error", - timestamp: 0, - }); - const { context, onPayload } = threadRequest(ctx, undefined); - expect(context).toBe(ctx); - expect(((await onPayload({}, model)) as Record).previous_response_id).toBeUndefined(); - }); - - it("never anchors previous_response_id on an assistant turn from a different api", async () => { - const ctx = multiTurnContext(); - // A mid-session -m provider switch leaves the prior provider's turn (and its foreign id) as the anchor. - ctx.messages.push({ - role: "assistant", - content: [{ type: "text", text: "done" }], - api: "anthropic-messages", - provider: "anthropic", - model: "claude-opus-4-8", - responseId: "msg_anthropic", - usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, - stopReason: "stop", - timestamp: 0, - }); - const { context, onPayload } = threadRequest(ctx, undefined); - expect(context).toBe(ctx); - expect(((await onPayload({}, model)) as Record).previous_response_id).toBeUndefined(); - }); - - it("composes a caller onPayload on top of the threaded payload", async () => { - const { onPayload } = threadRequest(multiTurnContext(), { - onPayload: (payload) => ({ wrapped: payload }), - }); - expect(await onPayload({ input: [] }, model)).toEqual({ - wrapped: { input: [], store: true, previous_response_id: `resp_${TURNS - 1}` }, - }); - }); -}); diff --git a/packages/ai/test/providers.test.ts b/packages/ai/test/providers.test.ts index 2faf9e71..a23119d3 100644 --- a/packages/ai/test/providers.test.ts +++ b/packages/ai/test/providers.test.ts @@ -2,7 +2,6 @@ import { describe, expect, it } from "vitest"; import { createCuaModels, cuaModels, - OPENAI_CUA_RESPONSES_API, TZAFON_RESPONSES_API, YUTORI_CHAT_COMPLETIONS_API, } from "../src/index"; @@ -41,9 +40,10 @@ describe("createCuaModels", () => { const models = createCuaModels(); const openaiIds = models.getModels("openai").map((m) => m.id); expect(openaiIds).toContain("gpt-5.4"); - // The catalog keeps pi's api ids; getCuaModel() routes to - // openai-cua-responses, which the wrapped provider dispatches. - expect(models.getModel("openai", "gpt-5.4")?.api).not.toBe(OPENAI_CUA_RESPONSES_API); + // OpenAI models keep pi's builtin "openai-responses" api id on both the + // collection and getCuaModel(); the wrapped provider only intercepts + // requests that need the CUA adapter (see requiresCuaOpenAIAdapter). + expect(models.getModel("openai", "gpt-5.4")?.api).toBe("openai-responses"); const xaiIds = models.getModels("xai").map((m) => m.id); expect(xaiIds).toContain("grok-4.3"); diff --git a/packages/ai/vitest.config.ts b/packages/ai/vitest.config.ts index 219048aa..25582a14 100644 --- a/packages/ai/vitest.config.ts +++ b/packages/ai/vitest.config.ts @@ -11,5 +11,9 @@ export default defineConfig({ // Unit runs cover every test file except the opt-in suites; use // vitest.integration.config.ts to run those. exclude: [...configDefaults.exclude, "**/*.integration.test.ts", "**/*.live.test.ts"], + // pi-ai ships real ESM and imports "openai" itself; both need to run + // through Vitest's module graph (not Node's native loader) for + // vi.mock("openai") to intercept requests pi's builtin transport makes. + server: { deps: { inline: [/@earendil-works\/pi-ai/, /^openai$/] } }, }, }); diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index 94b8418f..873ed864 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 0.10.0 - 2026-08-13 + +- `--print -o jsonl` schema bumps to version 2: every assistant message now + also emits an `assistant_usage` event (`turn`, `model`, `api`, `input`, + `output`, `cache_read`, `cache_write`, `reasoning`, `total_tokens`, and a + derived `cache_hit_ratio`), including tool-only turns with no text. + ## 0.9.0 - 2026-08-04 Breaking: upgrade the pi stack to 0.83.0 (`pi-ai`, `pi-agent-core`, diff --git a/packages/cli/README.md b/packages/cli/README.md index 134727b0..e08e49ad 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -204,10 +204,18 @@ Add `--jsonl-include-deltas` for assistant-token deltas and The first event of every `--print -o jsonl` run is `session_created` with a `schema_version` field. The current schema -version is `1`. The `model` field carries a provider-qualified ref +version is `2`. The `model` field carries a provider-qualified ref (e.g. `openai:gpt-5.6-sol`); use `parseCuaModelRef` from `@onkernel/cua-ai` if you only need the bare model id. +Every assistant message also emits an `assistant_usage` event (including +tool-only turns with no text): `turn`, `model`, `api`, `input`, `output`, +`cache_read`, `cache_write`, `reasoning`, `total_tokens`, and +`cache_hit_ratio`. OpenAI's billed prompt tokens are `input + cache_read + +cache_write` (the provider already subtracts cached and cache-write tokens +out of `input`), so `cache_hit_ratio` is `cache_read` over that total, +reported as `0` when the total is `0`. + ## Sessions and transcripts `--print`, the interactive TUI, and any `-s ` invocation persist diff --git a/packages/cli/src/output/harness-jsonl.ts b/packages/cli/src/output/harness-jsonl.ts index c3bf79a2..f4ab623c 100644 --- a/packages/cli/src/output/harness-jsonl.ts +++ b/packages/cli/src/output/harness-jsonl.ts @@ -2,13 +2,14 @@ import type { AgentHarnessEvent, KernelBrowser, } from "@onkernel/cua-agent"; +import type { Usage } from "@onkernel/cua-ai"; import type { CuaCliHarness } from "../harness"; /** * Schema version stamped on every `session_created` event. Bump when the * jsonl shape changes in a way external consumers need to detect. */ -export const CUA_JSONL_SCHEMA_VERSION = 1; +export const CUA_JSONL_SCHEMA_VERSION = 2; export interface JsonlSinkOptions { harness: CuaCliHarness; @@ -90,6 +91,7 @@ export function attachHarnessJsonlSink(opts: JsonlSinkOptions): () => void { } else if (msg.role === "assistant") { const text = textOf(msg.content); if (text) emit({ type: "assistant_text_done", text, ts: Date.now() }); + emit({ type: "assistant_usage", turn, model: msg.model, api: msg.api, ...usageFields(msg.usage), ts: Date.now() }); } return; } @@ -155,6 +157,24 @@ export function attachHarnessJsonlSink(opts: JsonlSinkOptions): () => void { }); } +/** + * OpenAI's `input_tokens` (and pi-ai's `Usage.input`) already excludes cached + * and cache-write tokens, so the billed prompt is `input + cacheRead + + * cacheWrite` and the cache hit ratio is `cacheRead` over that total. + */ +function usageFields(usage: Usage): Record { + const billedPrompt = usage.input + usage.cacheRead + usage.cacheWrite; + return { + input: usage.input, + output: usage.output, + cache_read: usage.cacheRead, + cache_write: usage.cacheWrite, + reasoning: usage.reasoning, + total_tokens: usage.totalTokens, + cache_hit_ratio: billedPrompt > 0 ? usage.cacheRead / billedPrompt : 0, + }; +} + function textOf(content: unknown): string { if (typeof content === "string") return content; if (!Array.isArray(content)) return ""; diff --git a/packages/cli/test/fixtures/scripted-provider.ts b/packages/cli/test/fixtures/scripted-provider.ts index d6763fa1..6c2ff2fe 100644 --- a/packages/cli/test/fixtures/scripted-provider.ts +++ b/packages/cli/test/fixtures/scripted-provider.ts @@ -7,6 +7,7 @@ import { type CuaSimpleStreamOptions, type Model, type MutableModels, + type Usage, } from "@onkernel/cua-ai"; /** One scripted step replayed when the harness asks the provider for a turn. */ @@ -23,6 +24,8 @@ export interface ScriptedTurn { * calls and to "toolUse" otherwise. */ stopReason?: "stop" | "toolUse" | "length"; + /** Overrides the zeroed default usage on the resulting assistant message. */ + usage?: Partial; } export interface ScriptedProviderHandle { @@ -88,7 +91,7 @@ export function createScriptedCuaModels(providerId: string, turns: ScriptedTurn[ function buildStream(model: Model, turn: ScriptedTurn | undefined, signal?: AbortSignal) { const stream = createAssistantMessageEventStream(); void (async () => { - const message = baseAssistantMessage(model); + const message = baseAssistantMessage(model, turn?.usage); if (!turn) { message.stopReason = "stop"; stream.push({ type: "start", partial: message }); @@ -222,7 +225,7 @@ async function waitForAbort(signal?: AbortSignal): Promise { }); } -function baseAssistantMessage(model: Model): AssistantMessage { +function baseAssistantMessage(model: Model, usage?: Partial): AssistantMessage { return { role: "assistant", content: [], @@ -236,6 +239,7 @@ function baseAssistantMessage(model: Model): AssistantMessage { cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + ...usage, }, stopReason: "stop", timestamp: Date.now(), diff --git a/packages/cli/test/print.test.ts b/packages/cli/test/print.test.ts index f022a691..01f1e8f2 100644 --- a/packages/cli/test/print.test.ts +++ b/packages/cli/test/print.test.ts @@ -34,7 +34,30 @@ describe("runPrint", () => { expect(types).toContain("assistant_text_done"); expect(types).toContain("turn_done"); expect(types).toContain("run_complete"); - expect((events[0] as { schema_version: number }).schema_version).toBe(1); + expect((events[0] as { schema_version: number }).schema_version).toBe(2); + }); + + it("emits assistant_usage with the billed-prompt cache hit ratio", async () => { + fixture = await buildTestHarness({ + turns: [ + { + steps: [{ type: "text", text: "ok" }], + usage: { input: 100, output: 20, cacheRead: 300, cacheWrite: 50, totalTokens: 420 }, + }, + ], + }); + const events = await runPrintAsJsonl(fixture, "go"); + const usage = events.find((e) => e.type === "assistant_usage") as Record; + expect(usage).toMatchObject({ + turn: 1, + input: 100, + output: 20, + cache_read: 300, + cache_write: 50, + total_tokens: 420, + }); + // billed prompt = input + cache_read + cache_write = 450; ratio = cache_read / billed prompt. + expect(usage.cache_hit_ratio).toBeCloseTo(300 / 450); }); it("returns exit code 1 when the provider emits an error", async () => { From d3166bfffde55e43a4c9da8bdbce891698ca06f5 Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 13 Aug 2026 15:31:31 +0000 Subject: [PATCH 2/3] Send prompt-cache fields on the OpenAI native computer path The native computer adapter relied on stored response state for context reuse, so it never sent a cache key of its own. With threading gone it replayed the whole transcript statelessly and uncached. Extract the prompt-cache request fields into one helper shared with the function-tool path, and build the native path's client through createOpenAIClient so session-affinity headers match too. --- packages/ai/CHANGELOG.md | 4 +++ packages/ai/src/providers/openai/provider.ts | 35 +++++++++++-------- .../ai/test/openai-native-provider.test.ts | 12 +++++++ 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 43acec14..f70dced8 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -15,6 +15,10 @@ Breaking: OpenAI models no longer carry a CUA-owned api id. OpenAI's native computer tool, or whose transcript carries a deferred tool-search addition or a replayed function-call namespace (pi-ai 0.83.0's builtin transport does not round-trip either). +- OpenAI's native computer adapter now sends the same `prompt_cache_key`, + `prompt_cache_retention`, `prompt_cache_options`, and session-affinity + headers as the function-tool path. It previously relied on stored response + state for context reuse and sent no cache key of its own. - Google, Meta, xAI, and Tzafon are unaffected: their `previous_response_id` threading helpers in `providers/common.ts` are unchanged. diff --git a/packages/ai/src/providers/openai/provider.ts b/packages/ai/src/providers/openai/provider.ts index 1fb0e85d..78c6ec6e 100644 --- a/packages/ai/src/providers/openai/provider.ts +++ b/packages/ai/src/providers/openai/provider.ts @@ -30,6 +30,9 @@ export interface OpenAIResponsesOptions extends PiOpenAIResponsesOptions { cuaIncomingToolPlan?: CuaIncomingToolPlan; } +/** The request fields both the function-tool and native-computer paths read, whichever option shape the caller passed. */ +type OpenAIRequestOptions = Pick; + /** * Whether a request needs cua-ai's OpenAI Responses adapter instead of pi-ai's * builtin `openai-responses` transport: OpenAI's native computer tool is @@ -142,14 +145,11 @@ function buildFunctionPayload( grammarToolInputProperties, }); applyTranscriptNamespaces(input as unknown as Array>, context.messages); - const retention = cacheRetention(options); const payload: Record = { model: model.id, input, stream: true, - prompt_cache_key: retention === "none" ? undefined : clampOpenAIPromptCacheKey(options?.sessionId), - prompt_cache_retention: retention === "long" && compat?.supportsLongCacheRetention !== false ? "24h" : undefined, - prompt_cache_options: retention === "none" && compat?.supportsExplicitPromptCacheMode ? { mode: "explicit" } : undefined, + ...promptCacheFields(model, options), store: false, }; if (options?.maxTokens) payload.max_output_tokens = Math.max(options.maxTokens, 16); @@ -171,9 +171,20 @@ function buildFunctionPayload( return payload; } +/** Prompt-cache request fields, matching what pi's builtin Responses transport sends. */ +function promptCacheFields(model: Model<"openai-responses">, options: OpenAIRequestOptions | undefined): Record { + const compat = model.compat; + const retention = cacheRetention(options); + return { + prompt_cache_key: retention === "none" ? undefined : clampOpenAIPromptCacheKey(options?.sessionId), + prompt_cache_retention: retention === "long" && compat?.supportsLongCacheRetention !== false ? "24h" : undefined, + prompt_cache_options: retention === "none" && compat?.supportsExplicitPromptCacheMode ? { mode: "explicit" } : undefined, + }; +} + function createOpenAIClient( model: Model<"openai-responses">, - options: OpenAIResponsesOptions | undefined, + options: OpenAIRequestOptions | undefined, apiKey: string, ): OpenAI { const headers: Record = { ...model.headers, ...options?.headers }; @@ -188,7 +199,7 @@ function createOpenAIClient( }); } -function openAIApiKey(options: OpenAIResponsesOptions | undefined): string { +function openAIApiKey(options: OpenAIRequestOptions | undefined): string { const apiKey = options?.apiKey || options?.env?.OPENAI_API_KEY || process.env.OPENAI_API_KEY; if (apiKey) return apiKey; const hasAuthorization = Object.entries(options?.headers ?? {}).some(([name, value]) => @@ -198,7 +209,7 @@ function openAIApiKey(options: OpenAIResponsesOptions | undefined): string { throw new Error("No API key for provider: openai"); } -function cacheRetention(options: OpenAIResponsesOptions | undefined): "none" | "short" | "long" { +function cacheRetention(options: OpenAIRequestOptions | undefined): "none" | "short" | "long" { if (options?.cacheRetention) return options.cacheRetention; return (options?.env?.PI_CACHE_RETENTION ?? process.env.PI_CACHE_RETENTION) === "long" ? "long" : "short"; } @@ -281,8 +292,7 @@ function streamOpenAINativeComputer( const output = initialAssistantMessage(model); void (async () => { try { - const apiKey = options?.apiKey || process.env.OPENAI_API_KEY; - if (!apiKey) throw new Error("No API key for provider: openai"); + const apiKey = openAIApiKey(options); const nativeName = options?.cuaIncomingToolPlan?.openaiComputerName; if (!nativeName) throw new Error("OpenAI native computer incoming plan is missing"); const placement = splitDeferredTools(context); @@ -292,14 +302,11 @@ function streamOpenAINativeComputer( input: convertMessages(context.messages, nativeName, placement.deferred), tools: convertTools(placement.immediate), max_output_tokens: options?.maxTokens ?? model.maxTokens, + ...promptCacheFields(model, options), store: false, }; payload = ((await options?.onPayload?.(payload, model)) ?? payload) as Record; - const client = new OpenAI({ - apiKey, - baseURL: model.baseUrl || "https://api.openai.com/v1", - defaultHeaders: { ...model.headers, ...options?.headers }, - }); + const client = createOpenAIClient(model, options, apiKey); const request = client.responses.create(payload as never, { signal: options?.signal }); const { data: response, response: rawResponse } = await request.withResponse(); await options?.onResponse?.({ status: rawResponse.status, headers: headersToRecord(rawResponse.headers) }, model); diff --git a/packages/ai/test/openai-native-provider.test.ts b/packages/ai/test/openai-native-provider.test.ts index 7d110799..d4cd523e 100644 --- a/packages/ai/test/openai-native-provider.test.ts +++ b/packages/ai/test/openai-native-provider.test.ts @@ -53,6 +53,18 @@ describe("OpenAI native computer Responses adapter", () => { expect(payload.previous_response_id).toBeUndefined(); }); + it("sends the same prompt-cache fields as the function-tool path", async () => { + responsesCreate.mockReturnValueOnce({ id: "resp_cache", usage: {}, output: [] }); + await openai.streamOpenAIResponses(model, { + messages: [{ role: "user", content: "go", timestamp: 1 }], + tools: [{ name: "computer", description: "placeholder", parameters: { type: "object" } as never }], + }, { apiKey: "test", sessionId: "session_native", cuaIncomingToolPlan: incoming }).result(); + + const payload = responsesCreate.mock.calls.at(-1)?.[0] as Record; + expect(payload.prompt_cache_key).toBe("session_native"); + expect(payload.store).toBe(false); + }); + it("round-trips function-call namespaces beside the native computer adapter", async () => { responsesCreate.mockReturnValueOnce({ id: "resp_namespace", From 3f8238c0ead9da65e74a626116cb55f52bf7df1d Mon Sep 17 00:00:00 2001 From: rgarcia <72655+rgarcia@users.noreply.github.com> Date: Thu, 13 Aug 2026 17:44:47 +0000 Subject: [PATCH 3/3] Delete the xAI and Meta Responses forks Both forks existed only to thread `previous_response_id` and to set `parallel_tool_calls: false`. The tool catalog already emits that field for every provider whose capabilities mark it as serializing state mutations, which includes xAI and Meta, so with threading gone neither fork contributes anything pi's builtin Responses transport does not already do. pi's registry gives grok-4.5 the `openai-responses` api and xAI's base URL directly, so xAI is now pi's builtin provider untouched. Meta is not in pi's registry, so it keeps its provider registration but streams through pi's builtin Responses transport against its own base URL and credentials. Meta also regains pi's stateless encrypted-reasoning replay, which the fork deleted because it relied on stored response state. Verified live: both models complete a multi-turn browser task through pi's builtin transport with no errors and 93-99% prompt cache hit rates on later turns. --- packages/agent/CHANGELOG.md | 4 +- packages/agent/src/agent.ts | 4 +- packages/ai/CHANGELOG.md | 15 +++- packages/ai/src/index.ts | 6 -- packages/ai/src/models.ts | 8 +- packages/ai/src/providers.ts | 31 ++----- packages/ai/src/providers/meta/provider.ts | 62 -------------- packages/ai/src/providers/xai/provider.ts | 57 ------------- packages/ai/test/meta-threading.test.ts | 97 ---------------------- packages/ai/test/models.test.ts | 18 ++-- packages/ai/test/providers.test.ts | 2 +- packages/ai/test/xai-threading.test.ts | 87 ------------------- 12 files changed, 36 insertions(+), 355 deletions(-) delete mode 100644 packages/ai/src/providers/meta/provider.ts delete mode 100644 packages/ai/src/providers/xai/provider.ts delete mode 100644 packages/ai/test/meta-threading.test.ts delete mode 100644 packages/ai/test/xai-threading.test.ts diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index 134d2d13..54f3fda5 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -5,8 +5,8 @@ - `responseThreading` (`CuaAgentOptions`/`CuaAgentHarnessOptions`) no longer affects OpenAI models: OpenAI now streams through pi-ai's builtin Responses transport and its automatic prompt caching regardless of this flag. The - option still governs Google, Meta, xAI, and Tzafon's - `previous_response_id`-style continuation. + option still governs Google and Tzafon's `previous_response_id`-style + continuation. - Exempt OpenAI's native computer tool from the tool-result image replay limit, alongside Tzafon: its `computer_call_output` items must each carry a screenshot, and stateless replay no longer leaves them in provider-stored diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index d3317fdd..89886cea 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -92,7 +92,7 @@ export type CuaAgentOptions = Omit & streamFn?: AgentOptions["streamFn"]; emptyResponseRecovery?: CuaEmptyResponseRecoveryOptions; toolResultImageReplayLimit?: ToolResultImageReplayLimit; - /** Governs Google, Meta, xAI, and Tzafon's `previous_response_id`-style continuation. OpenAI streams through pi's automatic prompt caching regardless of this flag. Defaults to `true`. */ + /** Governs Google and Tzafon's `previous_response_id`-style continuation. Every other provider streams through pi's transports and their automatic prompt caching regardless of this flag. Defaults to `true`. */ responseThreading?: boolean; retry?: CuaRetryOptions; }; @@ -113,7 +113,7 @@ type CuaAgentHarnessOptionsBase< onPayload?: SimpleStreamOptions["onPayload"]; emptyResponseRecovery?: CuaEmptyResponseRecoveryOptions; toolResultImageReplayLimit?: ToolResultImageReplayLimit; - /** Governs Google, Meta, xAI, and Tzafon's `previous_response_id`-style continuation. OpenAI streams through pi's automatic prompt caching regardless of this flag. Defaults to `true`. */ + /** Governs Google and Tzafon's `previous_response_id`-style continuation. Every other provider streams through pi's transports and their automatic prompt caching regardless of this flag. Defaults to `true`. */ responseThreading?: boolean; retry?: CuaRetryOptions; }; diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index f70dced8..aa0487c5 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -19,8 +19,19 @@ Breaking: OpenAI models no longer carry a CUA-owned api id. `prompt_cache_retention`, `prompt_cache_options`, and session-affinity headers as the function-tool path. It previously relied on stored response state for context reuse and sent no cache key of its own. -- Google, Meta, xAI, and Tzafon are unaffected: their `previous_response_id` - threading helpers in `providers/common.ts` are unchanged. +- Remove the xAI and Meta Responses forks. Both existed only to thread + `previous_response_id` and to set `parallel_tool_calls: false`, which the tool + catalog already emits for those providers. `xai-cua-responses` and + `meta-responses` are gone: Grok streams through pi's builtin xAI provider, and + Meta registers pi's builtin Responses transport against its own base URL and + credentials. `XAI_CUA_RESPONSES_API`, `META_RESPONSES_API`, + `streamXaiResponses`, `streamSimpleXaiResponses`, `streamMetaResponses`, and + `streamSimpleMetaResponses` are no longer exported. Meta also regains pi's + stateless encrypted-reasoning replay, which the fork deleted because it relied + on stored response state. +- Google and Tzafon keep their continuation protocols: each threads a + provider-specific field with no builtin equivalent, and the shared helpers in + `providers/common.ts` are unchanged for them. ## 0.10.0 - 2026-08-04 diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index cd4e8c04..fca2e3c2 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -4,21 +4,15 @@ export { createCuaModels, cuaModels, GOOGLE_CUA_INTERACTIONS_API, - META_RESPONSES_API, streamGoogleInteractions, - streamMetaResponses, streamOpenAIResponses, streamSimpleGoogleInteractions, - streamSimpleMetaResponses, streamSimpleOpenAIResponses, streamSimpleTzafonResponses, - streamSimpleXaiResponses, streamSimpleYutori, streamTzafonResponses, - streamXaiResponses, streamYutori, TZAFON_RESPONSES_API, - XAI_CUA_RESPONSES_API, YUTORI_CHAT_COMPLETIONS_API, } from "./providers"; export * from "./models"; diff --git a/packages/ai/src/models.ts b/packages/ai/src/models.ts index 34fffca2..cf167abd 100644 --- a/packages/ai/src/models.ts +++ b/packages/ai/src/models.ts @@ -1,8 +1,6 @@ import type { Api, Model } from "@earendil-works/pi-ai"; import { getBuiltinModel, getBuiltinModels } from "@earendil-works/pi-ai/providers/all"; import { GOOGLE_CUA_INTERACTIONS_API } from "./providers/google/provider"; -import { META_RESPONSES_API } from "./providers/meta/provider"; -import { XAI_CUA_RESPONSES_API } from "./providers/xai/provider"; /** Providers with curated computer-use model support. */ export type CuaProvider = "openai" | "anthropic" | "google" | "meta" | "xai" | "moonshotai" | "openrouter" | "tzafon" | "yutori"; @@ -237,13 +235,9 @@ export function routeCuaApi(model: Model): Model { if (model.provider === "google" && model.api !== GOOGLE_CUA_INTERACTIONS_API) { return { ...model, api: GOOGLE_CUA_INTERACTIONS_API }; } - if (model.provider === "meta" && model.api !== META_RESPONSES_API) { - return { ...model, api: META_RESPONSES_API }; - } if (model.provider === "xai" && model.id === "grok-4.5") { return { ...model, - api: XAI_CUA_RESPONSES_API, thinkingLevelMap: { off: "low", minimal: "low", xhigh: "high" }, cost: { ...model.cost, @@ -327,7 +321,7 @@ function cuaModel(provider: "meta" | "tzafon" | "yutori", id: string, name: stri // computer-use cookbook configures 128,000 maximum output tokens. return { ...base, - api: META_RESPONSES_API, + api: "openai-responses", baseUrl: "https://api.meta.ai/v1", thinkingLevelMap: { off: null, xhigh: "xhigh" }, cost: { input: 1.25, output: 4.25, cacheRead: 0.15, cacheWrite: 0 }, diff --git a/packages/ai/src/providers.ts b/packages/ai/src/providers.ts index 37835409..fd6de719 100644 --- a/packages/ai/src/providers.ts +++ b/packages/ai/src/providers.ts @@ -10,15 +10,17 @@ import { type SimpleStreamOptions, type StreamOptions, } from "@earendil-works/pi-ai"; +import { + stream as piStreamOpenAIResponses, + streamSimple as piStreamSimpleOpenAIResponses, +} from "@earendil-works/pi-ai/api/openai-responses"; import { builtinModels } from "@earendil-works/pi-ai/providers/all"; import { cuaApiKeyEnvVarsForProvider } from "./api-keys"; import { cuaOverrideModels } from "./models"; import { withAnthropicBrowserFallback } from "./providers/anthropic/browser-fallback"; import { GOOGLE_CUA_INTERACTIONS_API, streamGoogleInteractions, streamSimpleGoogleInteractions } from "./providers/google/provider"; -import { META_RESPONSES_API, streamMetaResponses, streamSimpleMetaResponses } from "./providers/meta/provider"; import { requiresCuaOpenAIAdapter, streamOpenAIResponses, streamSimpleOpenAIResponses } from "./providers/openai/provider"; import { streamSimpleTzafonResponses, streamTzafonResponses, TZAFON_RESPONSES_API } from "./providers/tzafon/provider"; -import { streamSimpleXaiResponses, streamXaiResponses, XAI_CUA_RESPONSES_API } from "./providers/xai/provider"; import { streamSimpleYutori, streamYutori, YUTORI_CHAT_COMPLETIONS_API } from "./providers/yutori/provider"; /** @@ -34,11 +36,13 @@ import { streamSimpleYutori, streamYutori, YUTORI_CHAT_COMPLETIONS_API } from ". * namespace (see {@link requiresCuaOpenAIAdapter}). * - `google` intercepts `google-cua-interactions` for current native computer * use and resolves API keys from `GOOGLE_API_KEY` or `GEMINI_API_KEY`. - * - `xai` intercepts `xai-cua-responses` so Grok can use stateful Responses - * tool loops while preserving pi's builtin xAI auth and catalog. + * - `xai` is pi's builtin provider untouched: Grok streams through pi's + * Responses transport, and the catalog supplies its serial-tool-call field. * - `moonshotai` is pi's builtin provider untouched: Kimi streams through the * plain OpenAI-compatible chat completions transport with `MOONSHOT_API_KEY`. * - `meta`, `tzafon`, and `yutori` are CUA-only providers pi does not ship. + * `meta` speaks the OpenAI Responses wire protocol, so it registers pi's + * builtin transport against Meta's base URL and credentials. * * Each call returns an independent collection; register additional providers * or credentials on it freely. Use {@link cuaModels} for the shared default. @@ -51,8 +55,6 @@ export function createCuaModels(options?: CreateModelsOptions): MutableModels { if (openai) models.setProvider(withOpenAICuaComputerAdapter(openai)); const google = models.getProvider("google"); if (google) models.setProvider(withGoogleCuaInteractions(google)); - const xai = models.getProvider("xai"); - if (xai) models.setProvider(withXaiCuaResponses(xai)); models.setProvider(metaProvider()); models.setProvider(tzafonProvider()); models.setProvider(yutoriProvider()); @@ -105,19 +107,6 @@ function withGoogleCuaInteractions(base: Provider): Provider { }; } -function withXaiCuaResponses(base: Provider): Provider { - return { - ...base, - stream: (model: Model, context: Context, options?: StreamOptions) => - model.api === XAI_CUA_RESPONSES_API - ? streamXaiResponses(model as never, context, options) - : base.stream(model, context, options), - streamSimple: (model: Model, context: Context, options?: SimpleStreamOptions) => - model.api === XAI_CUA_RESPONSES_API - ? streamSimpleXaiResponses(model as never, context, options) - : base.streamSimple(model, context, options), - }; -} function metaProvider(): Provider { return createProvider({ @@ -126,7 +115,7 @@ function metaProvider(): Provider { baseUrl: "https://api.meta.ai/v1", auth: { apiKey: envApiKeyAuth("Meta Model API key", cuaApiKeyEnvVarsForProvider("meta")) }, models: cuaOverrideModels("meta"), - api: { stream: streamMetaResponses, streamSimple: streamSimpleMetaResponses }, + api: { stream: piStreamOpenAIResponses, streamSimple: piStreamSimpleOpenAIResponses }, }); } @@ -153,8 +142,6 @@ function yutoriProvider(): Provider { } export { GOOGLE_CUA_INTERACTIONS_API, streamGoogleInteractions, streamSimpleGoogleInteractions }; -export { META_RESPONSES_API, streamMetaResponses, streamSimpleMetaResponses }; export { streamOpenAIResponses, streamSimpleOpenAIResponses }; export { TZAFON_RESPONSES_API, streamSimpleTzafonResponses, streamTzafonResponses }; -export { XAI_CUA_RESPONSES_API, streamSimpleXaiResponses, streamXaiResponses }; export { YUTORI_CHAT_COMPLETIONS_API, streamSimpleYutori, streamYutori }; diff --git a/packages/ai/src/providers/meta/provider.ts b/packages/ai/src/providers/meta/provider.ts deleted file mode 100644 index dffa9e15..00000000 --- a/packages/ai/src/providers/meta/provider.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { - type Context, - type OpenAIResponsesOptions as PiOpenAIResponsesOptions, - type StreamFunction, -} from "@earendil-works/pi-ai"; -import { - stream as piStreamOpenAIResponses, - streamSimple as piStreamSimpleOpenAIResponses, -} from "@earendil-works/pi-ai/api/openai-responses"; -import { - type CuaSimpleStreamOptions, - type ResponsesThreadingOptions, - threadResponsesRequest, -} from "../common"; - -export const META_RESPONSES_API = "meta-responses"; - -/** - * Stream options for Meta's OpenAI-compatible Responses API. - */ -export interface MetaResponsesOptions extends PiOpenAIResponsesOptions, ResponsesThreadingOptions {} - -/** - * Apply Meta's computer-use payload constraints on top of shared Responses threading. - */ -export function threadMetaRequest(context: Context, options: ResponsesThreadingOptions | undefined) { - const callerOnPayload = options?.onPayload; - const threaded = threadResponsesRequest(context, META_RESPONSES_API, { - ...options, - onPayload: async (payload, model) => { - const constrained: Record = { ...(payload as Record), parallel_tool_calls: false }; - delete constrained.include; - return callerOnPayload ? ((await callerOnPayload(constrained, model)) ?? constrained) : constrained; - }, - }); - const onPayload: typeof threaded.onPayload = async (payload, model) => { - const prepared = await threaded.onPayload(payload, model); - const sanitized: Record = { - ...(prepared as Record), - store: true, - parallel_tool_calls: false, - }; - delete sanitized.previous_response_id; - if (threaded.previousResponseId) sanitized.previous_response_id = threaded.previousResponseId; - // CUA uses stored response state instead of stateless encrypted reasoning replay. - delete sanitized.include; - return sanitized; - }; - return { context: threaded.context, onPayload }; -} - -// Meta implements the OpenAI Responses wire protocol. The model carries a -// distinct api id so response ids can never be threaded across providers. -export const streamMetaResponses: StreamFunction = (model, context, options) => { - const threaded = threadMetaRequest(context, options); - return piStreamOpenAIResponses(model as never, threaded.context, { ...options, onPayload: threaded.onPayload }); -}; - -export const streamSimpleMetaResponses: StreamFunction = (model, context, options) => { - const threaded = threadMetaRequest(context, options); - return piStreamSimpleOpenAIResponses(model as never, threaded.context, { ...options, onPayload: threaded.onPayload }); -}; diff --git a/packages/ai/src/providers/xai/provider.ts b/packages/ai/src/providers/xai/provider.ts deleted file mode 100644 index 24b64828..00000000 --- a/packages/ai/src/providers/xai/provider.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { - type Context, - type OpenAIResponsesOptions as PiOpenAIResponsesOptions, - type StreamFunction, -} from "@earendil-works/pi-ai"; -import { - stream as piStreamOpenAIResponses, - streamSimple as piStreamSimpleOpenAIResponses, -} from "@earendil-works/pi-ai/api/openai-responses"; -import { - type CuaSimpleStreamOptions, - type ResponsesThreadingOptions, - threadResponsesRequest, -} from "../common"; - -export const XAI_CUA_RESPONSES_API = "xai-cua-responses"; - -/** - * Stream options for xAI's OpenAI-compatible Responses API. - */ -export interface XaiResponsesOptions extends PiOpenAIResponsesOptions, ResponsesThreadingOptions {} - -/** - * Apply xAI's serial computer-use constraints on top of shared Responses threading. - */ -export function threadXaiRequest(context: Context, options: ResponsesThreadingOptions | undefined) { - const callerOnPayload = options?.onPayload; - const threaded = threadResponsesRequest(context, XAI_CUA_RESPONSES_API, { - ...options, - onPayload: async (payload, model) => { - const constrained = { ...(payload as Record), parallel_tool_calls: false }; - return callerOnPayload ? ((await callerOnPayload(constrained, model)) ?? constrained) : constrained; - }, - }); - const onPayload: typeof threaded.onPayload = async (payload, model) => { - const prepared = await threaded.onPayload(payload, model); - const constrained: Record = { - ...(prepared as Record), - store: true, - parallel_tool_calls: false, - }; - delete constrained.previous_response_id; - if (threaded.previousResponseId) constrained.previous_response_id = threaded.previousResponseId; - return constrained; - }; - return { context: threaded.context, onPayload }; -} - -export const streamXaiResponses: StreamFunction = (model, context, options) => { - const threaded = threadXaiRequest(context, options); - return piStreamOpenAIResponses(model as never, threaded.context, { ...options, onPayload: threaded.onPayload }); -}; - -export const streamSimpleXaiResponses: StreamFunction = (model, context, options) => { - const threaded = threadXaiRequest(context, options); - return piStreamSimpleOpenAIResponses(model as never, threaded.context, { ...options, onPayload: threaded.onPayload }); -}; diff --git a/packages/ai/test/meta-threading.test.ts b/packages/ai/test/meta-threading.test.ts deleted file mode 100644 index 43f18a16..00000000 --- a/packages/ai/test/meta-threading.test.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { describe, expect, it } from "vitest"; -import type { Context, Message, Model } from "@earendil-works/pi-ai"; -import { META_RESPONSES_API, threadMetaRequest } from "../src/providers/meta/provider"; - -const model = {} as Model; - -function multiTurnContext(): Context { - const messages: Message[] = [ - { role: "user", content: "inspect the browser", timestamp: 0 }, - { - role: "assistant", - content: [{ type: "toolCall", id: "call_1|fc_1", name: "screenshot", arguments: {} }], - api: META_RESPONSES_API, - provider: "meta", - model: "muse-spark-1.1", - responseId: "resp_meta_1", - usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, - stopReason: "toolUse", - timestamp: 0, - }, - { - role: "toolResult", - toolCallId: "call_1|fc_1", - toolName: "screenshot", - content: [{ type: "image", mimeType: "image/png", data: "screenshot" }], - isError: false, - timestamp: 0, - }, - ]; - return { messages, tools: [], systemPrompt: "control the browser" }; -} - -describe("Meta Responses threading", () => { - it("threads the latest response and applies serial computer-use defaults", async () => { - const { context, onPayload } = threadMetaRequest(multiTurnContext(), undefined); - expect(context.messages).toHaveLength(1); - expect(await onPayload({ input: [] }, model)).toEqual({ - input: [], - store: true, - parallel_tool_calls: false, - previous_response_id: "resp_meta_1", - }); - }); - - it("omits encrypted reasoning replay on the initial request", async () => { - const context: Context = { messages: [{ role: "user", content: "hello", timestamp: 0 }] }; - const { onPayload } = threadMetaRequest(context, undefined); - expect(await onPayload({ input: [], include: ["reasoning.encrypted_content"] }, model)).toEqual({ - input: [], - store: true, - parallel_tool_calls: false, - }); - }); - - it("sends full history without previous_response_id when threading is disabled", async () => { - const context = multiTurnContext(); - const prepared = threadMetaRequest(context, { - disableResponseThreading: true, - onPayload: (payload) => ({ ...(payload as object), previous_response_id: "foreign-response" }), - }); - expect(prepared.context).toBe(context); - expect(await prepared.onPayload({ input: [] }, model)).toEqual({ - input: [], - store: true, - parallel_tool_calls: false, - }); - }); - - it("passes the prepared request to a caller payload hook", async () => { - const { onPayload } = threadMetaRequest(multiTurnContext(), { - onPayload: (payload) => ({ ...(payload as object), caller_field: true }), - }); - expect(await onPayload({}, model)).toEqual({ - caller_field: true, - store: true, - parallel_tool_calls: false, - previous_response_id: "resp_meta_1", - }); - }); - - it("reapplies Meta constraints after caller payload hooks", async () => { - const { onPayload } = threadMetaRequest(multiTurnContext(), { - onPayload: (payload) => ({ - ...(payload as object), - store: false, - parallel_tool_calls: true, - previous_response_id: "wrong-response", - include: ["reasoning.encrypted_content"], - }), - }); - expect(await onPayload({}, model)).toEqual({ - store: true, - parallel_tool_calls: false, - previous_response_id: "resp_meta_1", - }); - }); -}); diff --git a/packages/ai/test/models.test.ts b/packages/ai/test/models.test.ts index dfd1de0c..6f5628f0 100644 --- a/packages/ai/test/models.test.ts +++ b/packages/ai/test/models.test.ts @@ -9,9 +9,7 @@ import { getCuaModel, GOOGLE_CUA_INTERACTIONS_API, listCuaModels, - META_RESPONSES_API, parseCuaModelRef, - XAI_CUA_RESPONSES_API, } from "../src/index"; describe("CUA model refs", () => { @@ -78,7 +76,7 @@ describe("CUA model refs", () => { const muse = getCuaModel("meta:muse-spark-1.1"); expect(muse.provider).toBe("meta"); - expect(muse.api).toBe(META_RESPONSES_API); + expect(muse.api).toBe("openai-responses"); expect(muse.baseUrl).toBe("https://api.meta.ai/v1"); expect(muse.contextWindow).toBe(1_048_576); expect(muse.maxTokens).toBe(128_000); @@ -89,7 +87,7 @@ describe("CUA model refs", () => { expect(cuaOverrideModels("xai")).toEqual([]); const grok = getCuaModel("xai:grok-4.5"); expect(grok.provider).toBe("xai"); - expect(grok.api).toBe(XAI_CUA_RESPONSES_API); + expect(grok.api).toBe("openai-responses"); expect(grok.baseUrl).toBe("https://api.x.ai/v1"); expect(grok.contextWindow).toBe(500_000); expect(grok.maxTokens).toBe(500_000); @@ -133,16 +131,16 @@ describe("CUA model refs", () => { expect(getCuaModel("yutori:n1.5-latest").api).toBe("yutori-chat-completions"); }); - it("routes Responses models to provider-specific transports, and keeps OpenAI on pi's builtin one", () => { - // OpenAI has a fully equivalent builtin transport (automatic prompt - // caching, no previous_response_id needed), so it keeps pi-ai's - // registry api id instead of being routed to a CUA-owned one. + it("keeps every Responses model on pi's builtin transport except Google's Interactions API", () => { + // A CUA-owned api id now exists only where pi ships no equivalent + // transport. OpenAI, Meta, and xAI all speak the Responses protocol pi + // already implements, including its automatic prompt caching. expect(getCuaModel("openai:gpt-5.6-sol").api).toBe("openai-responses"); expect(getCuaModel("openai:gpt-5.5").api).toBe("openai-responses"); expect(getCuaModel("openai:gpt-5.4-mini").api).toBe("openai-responses"); expect(getCuaModel("google:gemini-3.6-flash").api).toBe(GOOGLE_CUA_INTERACTIONS_API); - expect(getCuaModel("meta:muse-spark-1.1").api).toBe(META_RESPONSES_API); - expect(getCuaModel("xai:grok-4.5").api).toBe(XAI_CUA_RESPONSES_API); + expect(getCuaModel("meta:muse-spark-1.1").api).toBe("openai-responses"); + expect(getCuaModel("xai:grok-4.5").api).toBe("openai-responses"); }); it("rejects supported model IDs that are not in pi-ai or overrides", () => { diff --git a/packages/ai/test/providers.test.ts b/packages/ai/test/providers.test.ts index a23119d3..ea651aba 100644 --- a/packages/ai/test/providers.test.ts +++ b/packages/ai/test/providers.test.ts @@ -19,7 +19,7 @@ describe("createCuaModels", () => { it("lists CUA provider catalogs", () => { const models = createCuaModels(); - expect(models.getModel("meta", "muse-spark-1.1")?.api).toBe("meta-responses"); + expect(models.getModel("meta", "muse-spark-1.1")?.api).toBe("openai-responses"); expect(models.getModel("xai", "grok-4.5")?.api).toBe("openai-responses"); expect(models.getModel("moonshotai", "kimi-k3")?.api).toBe("openai-completions"); expect(models.getModel("openrouter", "moonshotai/kimi-k3")?.api).toBe("openai-completions"); diff --git a/packages/ai/test/xai-threading.test.ts b/packages/ai/test/xai-threading.test.ts deleted file mode 100644 index 1907aca6..00000000 --- a/packages/ai/test/xai-threading.test.ts +++ /dev/null @@ -1,87 +0,0 @@ -import type { Context, Message, Model } from "@earendil-works/pi-ai"; -import { describe, expect, it } from "vitest"; -import { threadXaiRequest, XAI_CUA_RESPONSES_API } from "../src/providers/xai/provider"; - -const model = {} as Model; - -function multiTurnContext(): Context { - const messages: Message[] = [ - { role: "user", content: "inspect the browser", timestamp: 0 }, - { - role: "assistant", - content: [{ type: "toolCall", id: "call_1|fc_1", name: "screenshot", arguments: {} }], - api: XAI_CUA_RESPONSES_API, - provider: "xai", - model: "grok-4.5", - responseId: "response_xai_1", - usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, - stopReason: "toolUse", - timestamp: 0, - }, - { - role: "toolResult", - toolCallId: "call_1|fc_1", - toolName: "screenshot", - content: [{ type: "image", mimeType: "image/png", data: "screenshot" }], - isError: false, - timestamp: 0, - }, - ]; - return { messages, tools: [], systemPrompt: "control the browser" }; -} - -describe("xAI Responses threading", () => { - it("threads the latest response and applies serial computer-use defaults", async () => { - const { context, onPayload } = threadXaiRequest(multiTurnContext(), undefined); - expect(context.messages).toHaveLength(1); - expect(await onPayload({ input: [] }, model)).toEqual({ - input: [], - store: true, - parallel_tool_calls: false, - previous_response_id: "response_xai_1", - }); - }); - - it("preserves encrypted reasoning replay", async () => { - const context: Context = { messages: [{ role: "user", content: "hello", timestamp: 0 }] }; - const { onPayload } = threadXaiRequest(context, undefined); - expect(await onPayload({ input: [], include: ["reasoning.encrypted_content"] }, model)).toEqual({ - input: [], - include: ["reasoning.encrypted_content"], - store: true, - parallel_tool_calls: false, - }); - }); - - it("sends full history without previous_response_id when threading is disabled", async () => { - const context = multiTurnContext(); - const prepared = threadXaiRequest(context, { - disableResponseThreading: true, - onPayload: (payload) => ({ ...(payload as object), previous_response_id: "foreign-response" }), - }); - expect(prepared.context).toBe(context); - expect(await prepared.onPayload({ input: [] }, model)).toEqual({ - input: [], - store: true, - parallel_tool_calls: false, - }); - }); - - it("reapplies xAI constraints after caller payload hooks", async () => { - const { onPayload } = threadXaiRequest(multiTurnContext(), { - onPayload: (payload) => ({ - ...(payload as object), - store: false, - parallel_tool_calls: true, - previous_response_id: "wrong-response", - include: ["reasoning.encrypted_content"], - }), - }); - expect(await onPayload({}, model)).toEqual({ - store: true, - parallel_tool_calls: false, - previous_response_id: "response_xai_1", - include: ["reasoning.encrypted_content"], - }); - }); -});