Add pi extension for CUA browser tools - #73
Open
rgarcia wants to merge 7 commits into
Open
Conversation
![cursor[bot]](https://avatars.githubusercontent.com/in/1210556?s=60&v=4){kind=small}
There was a problem hiding this comment.
Cursor Bugbot has reviewed your changes using high effort and found 1 potential issue.
Autofix Details
Bugbot Autofix prepared a fix for the issue found in the latest run.
- ✅ Fixed: Native display size mismatch
- Anthropic native computer specs now use a runtime-resolved browser viewport for provider payload compilation, so declared display dimensions match the controlled session.
Or push these changes by commenting:
@cursor push b533367b23
Preview (b533367b23)
diff --git a/packages/pi-extension/src/catalog.ts b/packages/pi-extension/src/catalog.ts
--- a/packages/pi-extension/src/catalog.ts
+++ b/packages/pi-extension/src/catalog.ts
@@ -48,6 +48,8 @@
coordinates: Coordinates;
}
+export const DEFAULT_VIEWPORT: Readonly<{ width: number; height: number }> = Object.freeze({ width: 1920, height: 1080 });
+
const generalTools = Object.freeze({
browser_snapshot: () => cua.tools.browser.snapshot(),
browser_text: () => cua.tools.browser.text(),
@@ -121,15 +123,15 @@
}
/** Every function tool that can be selected, with declarations for one coordinate mode. */
-export function allSelectableSpecs(coordinates: Coordinates): CuaToolSpec[] {
+export function allSelectableSpecs(coordinates: Coordinates, viewport = DEFAULT_VIEWPORT): CuaToolSpec[] {
const result = new Map<string, CuaToolSpec>();
for (const selector of CUA_SELECTORS) {
- for (const spec of expandSelection(parseSelection(selector, coordinates))) result.set(spec.name, spec);
+ for (const spec of expandSelection(parseSelection(selector, coordinates), viewport)) result.set(spec.name, spec);
}
return [...result.values()];
}
-export function expandSelection(selection: CuaSelection): CuaToolSpec[] {
+export function expandSelection(selection: CuaSelection, viewport = DEFAULT_VIEWPORT): CuaToolSpec[] {
const coordinates = selection.coordinates === "pixels" ? cua.coordinates.pixels() : cua.coordinates.normalized([0, 1000]);
const result: CuaToolSpec[] = [];
for (const selector of selection.selectors) {
@@ -157,7 +159,12 @@
break;
case "anthropic-computer":
result.push(
- cua.providers.anthropic.tools.computer({ version: "20251124", displayWidth: 1920, displayHeight: 1080, enableZoom: true }),
+ cua.providers.anthropic.tools.computer({
+ version: "20251124",
+ displayWidth: viewport.width,
+ displayHeight: viewport.height,
+ enableZoom: true,
+ }),
);
break;
default:
@@ -180,15 +187,15 @@
throw new Error(`unknown CUA tool selector "${name}"`);
}
-export function compileSpecs(model: Model<Api>, specs: readonly CuaToolSpec[], viewport = { width: 1920, height: 1080 }): CuaToolCatalog {
+export function compileSpecs(model: Model<Api>, specs: readonly CuaToolSpec[], viewport = DEFAULT_VIEWPORT): CuaToolCatalog {
return compileCuaToolCatalog({ model, requestedTools: specs, viewport });
}
export function compileSelection(
model: Model<Api>,
selection: CuaSelection,
- viewport = { width: 1920, height: 1080 },
+ viewport = DEFAULT_VIEWPORT,
): { specs: CuaToolSpec[]; catalog: CuaToolCatalog } {
- const specs = expandSelection(selection);
+ const specs = expandSelection(selection, viewport);
return { specs, catalog: compileSpecs(model, specs, viewport) };
}
diff --git a/packages/pi-extension/src/index.ts b/packages/pi-extension/src/index.ts
--- a/packages/pi-extension/src/index.ts
+++ b/packages/pi-extension/src/index.ts
@@ -1,13 +1,11 @@
import { fileURLToPath } from "node:url";
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
import { createCuaModels, type CuaToolSpec } from "@onkernel/cua-ai";
-import { allSelectableSpecs, compileSpecs, expandSelection, parseSelection, type CuaSelection } from "./catalog";
+import { allSelectableSpecs, compileSpecs, DEFAULT_VIEWPORT, expandSelection, parseSelection, type CuaSelection } from "./catalog";
import { CuaBrowserRuntime, type BrowserOptions } from "./browser-runtime";
import { CONFIG_ENTRY, restoreConfig, type PersistedConfig } from "./state";
import { statusText } from "./render";
-const VIEWPORT = { width: 1920, height: 1080 };
-
export default function cuaPiExtension(pi: ExtensionAPI): void {
pi.registerFlag("cua-tools", { type: "string", description: "Comma-separated explicit CUA tool selectors" });
pi.registerFlag("cua-coordinates", { type: "string", description: "pixels or normalized-1000", default: "pixels" });
@@ -58,12 +56,24 @@
function ensureRuntime(): CuaBrowserRuntime {
return (runtime ??= new CuaBrowserRuntime(browserOptions));
}
- function currentSpecs(): CuaToolSpec[] {
- return expandSelection(selection);
+ function currentSpecs(viewport = DEFAULT_VIEWPORT): CuaToolSpec[] {
+ return expandSelection(selection, viewport);
}
- function activeSpecs(): CuaToolSpec[] {
- return currentSpecs().filter((spec) => activeNames.has(spec.name));
+ function activeSpecs(viewport = DEFAULT_VIEWPORT): CuaToolSpec[] {
+ return currentSpecs(viewport).filter((spec) => activeNames.has(spec.name));
}
+ function hasAnthropicNativeComputer(specs: readonly CuaToolSpec[]): boolean {
+ return specs.some((spec) => spec.name === "computer" && spec.providerBinding?.kind === "anthropic-native");
+ }
+ async function compilationViewport(specs: readonly CuaToolSpec[]): Promise<{ width: number; height: number }> {
+ if (!hasAnthropicNativeComputer(specs)) return DEFAULT_VIEWPORT;
+ try {
+ const resources = await ensureRuntime().get();
+ return resources.viewport;
+ } catch {
+ return DEFAULT_VIEWPORT;
+ }
+ }
function persistCommandSelection(): void {
const state: PersistedConfig = {
version: 1,
@@ -89,7 +99,7 @@
compileSpecs(
ctx.model,
specs.filter((spec) => desired.includes(spec.name)),
- VIEWPORT,
+ DEFAULT_VIEWPORT,
);
compatibilityError = undefined;
forcedInactive = false;
@@ -152,7 +162,7 @@
pi.on("before_agent_start", (_event, ctx) => reconcile(ctx));
pi.on("before_provider_headers", (event, ctx) => {
if (!activeNames.size || compatibilityError || !ctx.model) return;
- const catalog = compileSpecs(ctx.model, activeSpecs(), VIEWPORT);
+ const catalog = compileSpecs(ctx.model, activeSpecs(), DEFAULT_VIEWPORT);
Object.assign(event.headers, catalog.headers.merge(event.headers));
});
pi.on("before_provider_request", async (event, ctx) => {
@@ -163,7 +173,8 @@
// a catalog after pi has already built a payload for the turn.
return currentSpecs().length ? withoutCuaToolSchemas(event.payload, allSpecs) : undefined;
}
- return compileSpecs(ctx.model, activeSpecs(), VIEWPORT).payload.apply(event.payload, ctx.model);
+ const viewport = await compilationViewport(activeSpecs());
+ return compileSpecs(ctx.model, activeSpecs(viewport), viewport).payload.apply(event.payload, ctx.model);
});
pi.on("tool_call", (event) => {
if (!allSpecs.has(event.toolName)) return;
diff --git a/packages/pi-extension/test/catalog.test.ts b/packages/pi-extension/test/catalog.test.ts
--- a/packages/pi-extension/test/catalog.test.ts
+++ b/packages/pi-extension/test/catalog.test.ts
@@ -59,8 +59,10 @@
});
it("compiles Anthropic native computer use only for supported Anthropic models", () => {
const selection = parseSelection("anthropic-computer", "pixels");
- const { specs, catalog } = compileSelection(getCuaModel("anthropic:claude-fable-5"), selection);
+ const viewport = { width: 1440, height: 900 };
+ const { specs, catalog } = compileSelection(getCuaModel("anthropic:claude-fable-5"), selection, viewport);
expect(specs.map((tool) => tool.name)).toEqual(["computer"]);
+ expect(catalog.entries[0]?.declaration).toMatchObject({ display_width_px: 1440, display_height_px: 900 });
expect(catalog.entries.map((entry) => entry.transport)).toEqual(["native"]);
expect(catalog.headers.requirements).toContainEqual(expect.objectContaining({ value: "computer-use-2025-11-24" }));
expect(() => compileSelection(getCuaModel("openai:gpt-5.6-sol"), selection)).toThrow("requires a anthropic model");
diff --git a/packages/pi-extension/test/extension.test.ts b/packages/pi-extension/test/extension.test.ts
--- a/packages/pi-extension/test/extension.test.ts
+++ b/packages/pi-extension/test/extension.test.ts
@@ -2,8 +2,9 @@
import type { Api, Model, Provider } from "@earendil-works/pi-ai";
import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
import { getCuaModel } from "@onkernel/cua-ai";
-import { describe, expect, it } from "vitest";
+import { describe, expect, it, vi } from "vitest";
import { allSelectableSpecs } from "../src/catalog";
+import { CuaBrowserRuntime } from "../src/browser-runtime";
import extension from "../src/index";
type Handler = (event: unknown, ctx: ExtensionContext) => unknown;
@@ -117,33 +118,40 @@
});
it("registers the CUA Anthropic provider and serializes native computer use", async () => {
- const pi = makePi({
- "cua-tools": "anthropic-computer",
- "cua-coordinates": "pixels",
- "cua-browser-timeout": "300",
- "cua-profile-save-changes": false,
- });
- extension(pi.api);
- expect(pi.providers.map((provider) => provider.id)).toContain("anthropic");
- await getHandler(pi, "session_start")({}, anthropicCtx);
- expect(pi.active).toContain("computer");
+ const getSpy = vi.spyOn(CuaBrowserRuntime.prototype, "get").mockResolvedValue({
+ viewport: { width: 1024, height: 768 },
+ } as unknown as Awaited<ReturnType<CuaBrowserRuntime["get"]>>);
+ try {
+ const pi = makePi({
+ "cua-tools": "anthropic-computer",
+ "cua-coordinates": "pixels",
+ "cua-browser-timeout": "300",
+ "cua-profile-save-changes": false,
+ });
+ extension(pi.api);
+ expect(pi.providers.map((provider) => provider.id)).toContain("anthropic");
+ await getHandler(pi, "session_start")({}, anthropicCtx);
+ expect(pi.active).toContain("computer");
- const headers: Record<string, string> = {};
- await getHandler(pi, "before_provider_headers")({ headers }, anthropicCtx);
- expect(headers["anthropic-beta"]).toContain("computer-use-2025-11-24");
+ const headers: Record<string, string> = {};
+ await getHandler(pi, "before_provider_headers")({ headers }, anthropicCtx);
+ expect(headers["anthropic-beta"]).toContain("computer-use-2025-11-24");
- const payload = { tools: [{ name: "computer", input_schema: { type: "object" } }] };
- const transformed = await getHandler(pi, "before_provider_request")({ payload }, anthropicCtx);
- expect(transformed).toEqual({
- tools: [
- expect.objectContaining({
- name: "computer",
- type: "computer_20251124",
- display_width_px: 1920,
- display_height_px: 1080,
- }),
- ],
- });
+ const payload = { tools: [{ name: "computer", input_schema: { type: "object" } }] };
+ const transformed = await getHandler(pi, "before_provider_request")({ payload }, anthropicCtx);
+ expect(transformed).toEqual({
+ tools: [
+ expect.objectContaining({
+ name: "computer",
+ type: "computer_20251124",
+ display_width_px: 1024,
+ display_height_px: 768,
+ }),
+ ],
+ });
+ } finally {
+ getSpy.mockRestore();
+ }
});
it("applies provider transforms only for the active CUA subset", async () => {You can send follow-ups to the cloud agent here.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
1 participant
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Summary
@onkernel/cua-pi-extensionpackage that exposes explicit CUA tools in pi sessionscomputer_20251124native tool through--cua-tools anthropic-computerScope
Anthropic native computer use and ordinary CUA function tools are supported. Provider-native Anthropic browser use, OpenAI, Google, Tzafon, and Yutori protocols remain outside this extension.
Tests
npm test --workspace @onkernel/cua-pi-extension(18 passed)npm test --workspace @onkernel/cua-ai(115 passed)npm test --workspace @onkernel/cua-agent(291 passed, 19 skipped)npm run typecheckclaude-fable-5agent_settled, and deleted the owned browserThe full root build reaches the unrelated ptywright native build and requires Zig, which was not installed locally.
Note
Medium Risk
New extension touches provider requests, browser provisioning, and credentials at runtime; cua-ai changes affect Anthropic native tool serialization for all consumers.
Overview
Introduces
@onkernel/cua-pi-extension, an installable pi extension that wires explicit CUA function tools and Anthropic native computer use into pi sessions via flags like--cua-tools, lazy Kernel browser create/attach,/cuaand/cua-toolscommands, provider header/request transforms, and owned-browser cleanup on shutdown.@onkernel/cua-ai/@onkernel/cua-agent0.10.1 adds Anthropic documented native computer versions20250124and20251124(display dimensions, beta headers,enable_zoomrules) while keeping early-access20260701.CI gains a
pi-extension-unitjob;release-cua-pi-extension.ymlpublishes oncua-pi-extension/v*tags with version checks and pack smoke tests. Root workspace, README, architecture, and npm release docs are updated accordingly.Reviewed by Cursor Bugbot for commit 7c22bb7. Bugbot is set up for automated code reviews on this repo. Configure here.