diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a76da2..00c353a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,23 @@ jobs: npm install "$RUNNER_TEMP"/onkernel-cua-ai-*.tgz node --input-type=module -e "import('@onkernel/cua-ai').then((m) => { if (typeof m.getCuaModel !== 'function') process.exit(1); })" + pi-extension-unit: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-node@v5 + with: + node-version: 22 + cache: npm + - run: npm ci + - run: npm run build --workspace @onkernel/cua-ai + - run: npm run build --workspace @onkernel/cua-agent + - name: Pi extension tests + run: npm test --workspace @onkernel/cua-pi-extension + - name: Pi extension typecheck + run: npm run typecheck --workspace @onkernel/cua-pi-extension + agent-unit: runs-on: ubuntu-latest timeout-minutes: 15 diff --git a/.github/workflows/release-cua-pi-extension.yml b/.github/workflows/release-cua-pi-extension.yml new file mode 100644 index 0000000..e1bb645 --- /dev/null +++ b/.github/workflows/release-cua-pi-extension.yml @@ -0,0 +1,95 @@ +name: Release CUA Pi Extension + +on: + push: + tags: + - "cua-pi-extension/v*" + +permissions: + contents: read + id-token: write + +concurrency: + group: release-cua-pi-extension-${{ github.ref_name }} + cancel-in-progress: false + +jobs: + publish: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Verify tag is on main + run: | + git fetch origin main:refs/remotes/origin/main + git merge-base --is-ancestor "$GITHUB_SHA" origin/main + + - uses: actions/setup-node@v5 + with: + node-version: 24 + registry-url: https://registry.npmjs.org + + - name: Ensure npm supports trusted publishing + run: npm install -g npm@^11.5.1 + + - run: npm ci + + - name: Verify package version matches tag + run: | + node --input-type=module <<'EOF' + import { readFileSync } from "node:fs"; + + const tag = process.env.GITHUB_REF_NAME; + const prefix = "cua-pi-extension/v"; + if (!tag?.startsWith(prefix)) { + throw new Error(`Expected tag to start with ${prefix}, got ${tag}`); + } + + const tagVersion = tag.slice(prefix.length); + const pkg = JSON.parse(readFileSync("packages/pi-extension/package.json", "utf8")); + if (pkg.version !== tagVersion) { + throw new Error(`Tag version ${tagVersion} does not match ${pkg.name} package.json version ${pkg.version}`); + } + + console.log(`${pkg.name}@${pkg.version}`); + EOF + + - run: npm run build --workspace @onkernel/cua-ai + - run: npm run build --workspace @onkernel/cua-agent + - run: npm run build --workspace @onkernel/cua-pi-extension + - run: npm run typecheck --workspace @onkernel/cua-pi-extension + + - name: Unit tests + run: npm test --workspace @onkernel/cua-pi-extension -- --exclude "**/*.live.test.ts" + + - name: Verify dependency packages are published + run: | + for package in @onkernel/cua-ai @onkernel/cua-agent @onkernel/sdk; do + version=$(node -p "require('./packages/pi-extension/package.json').dependencies['$package']") + npm view "$package@$version" version + done + + - name: Pack tarballs + run: | + mkdir -p /tmp/pack + npm pack --workspace @onkernel/cua-ai --pack-destination /tmp/pack + npm pack --workspace @onkernel/cua-pi-extension --pack-destination /tmp/pack + + - name: Packed manifest smoke test + run: | + SMOKE_DIR=$(mktemp -d) + cd "$SMOKE_DIR" + npm init -y > /dev/null + npm install --omit=dev /tmp/pack/*.tgz + node --input-type=module <<'EOF' + import { readFileSync } from "node:fs"; + const pkg = JSON.parse(readFileSync("node_modules/@onkernel/cua-pi-extension/package.json", "utf8")); + if (pkg.pi?.extensions?.[0] !== "./src/index.ts") throw new Error("missing pi extension manifest"); + console.log("packed manifest OK"); + EOF + + - name: Publish to npm + run: npm publish --workspace @onkernel/cua-pi-extension --access public diff --git a/README.md b/README.md index 6d903f1..3c5864b 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ packages/ ├── ai/ # @onkernel/cua-ai - model catalog, tool schemas, provider adapters ├── agent/ # @onkernel/cua-agent - Kernel-browser tool execution ├── cli/ # @onkernel/cua-cli - the `cua` binary +├── pi-extension/ # @onkernel/cua-pi-extension - pi function-tool extension └── ptywright/ # @onkernel/ptywright - development-only PTY/TUI test infrastructure ``` @@ -69,6 +70,7 @@ flowchart LR | [`@onkernel/cua-ai`](packages/ai) | Computer-use model catalog, tool factories/toolsets, compatibility checks, and provider adapters. | | [`@onkernel/cua-agent`](packages/agent) | Agent and harness APIs that run selected computer-use tools against a Kernel browser. | | [`@onkernel/cua-cli`](packages/cli) | The `cua` binary: argv parsing, sessions, skills, JSONL output, pi-tui front-end. | +| [`@onkernel/cua-pi-extension`](packages/pi-extension) | Explicit CUA function tools and Anthropic native computer use inside pi. | | [`@onkernel/ptywright`](packages/ptywright) | Development-only PTY/TUI test infrastructure. | --- diff --git a/docs/architecture.md b/docs/architecture.md index 1b3dfe8..64f505b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -26,6 +26,11 @@ both explicitly and may use pi's orchestration primitives directly. for each selected model, adds pi coding tools, supplies the system prompt, resolves credentials/sessions/skills, and renders text, JSONL, or TUI output. - `@onkernel/ptywright` is development-only PTY/TUI test infrastructure. +- `@onkernel/cua-pi-extension` adapts explicit CUA specs into pi extension + tools. Pi owns model selection, sessions, prompting, and RPC; cua-ai compiles + declarations and cua-agent executes against the shared browser pool. It also + registers CUA's Anthropic provider wrapper so the documented native Anthropic + computer tool retains its transport behavior. The invariant is that `packages/agent/src` contains no provider-name branches. Adding provider behavior means adding data and transforms in `cua-ai`, not a diff --git a/docs/npm-releases.md b/docs/npm-releases.md index 9c3bedb..c1f7a60 100644 --- a/docs/npm-releases.md +++ b/docs/npm-releases.md @@ -1,11 +1,12 @@ # npm releases -`@onkernel/cua-ai`, `@onkernel/cua-agent`, and `@onkernel/cua-cli` publish from -package-specific tags: +`@onkernel/cua-ai`, `@onkernel/cua-agent`, `@onkernel/cua-cli`, and +`@onkernel/cua-pi-extension` publish from package-specific tags: - `cua-ai/v0.1.0` runs `.github/workflows/release-cua-ai.yml` - `cua-agent/v0.1.0` runs `.github/workflows/release-cua-agent.yml` - `cua-cli/v0.1.0` runs `.github/workflows/release-cua-cli.yml` +- `cua-pi-extension/v0.10.1` runs `.github/workflows/release-cua-pi-extension.yml` The tag version must match the target package's `package.json` version, and the tagged commit must be contained in `main`. @@ -19,6 +20,7 @@ Configure each package on npm with a GitHub Actions trusted publisher: | `@onkernel/cua-ai` | `kernel` | `cua` | `release-cua-ai.yml` | leave blank | | `@onkernel/cua-agent` | `kernel` | `cua` | `release-cua-agent.yml` | leave blank | | `@onkernel/cua-cli` | `kernel` | `cua` | `release-cua-cli.yml` | leave blank | +| `@onkernel/cua-pi-extension` | `kernel` | `cua` | `release-cua-pi-extension.yml` | leave blank | The same configuration can be created from the npm CLI: @@ -27,6 +29,7 @@ npm install -g npm@^11.17.0 npm trust github @onkernel/cua-ai --repo kernel/cua --file release-cua-ai.yml --allow-publish npm trust github @onkernel/cua-agent --repo kernel/cua --file release-cua-agent.yml --allow-publish npm trust github @onkernel/cua-cli --repo kernel/cua --file release-cua-cli.yml --allow-publish +npm trust github @onkernel/cua-pi-extension --repo kernel/cua --file release-cua-pi-extension.yml --allow-publish ``` npm requires packages to exist before a trusted publisher can be configured. If diff --git a/package-lock.json b/package-lock.json index 0f50d15..8b55446 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,8 @@ "packages/ai", "packages/agent", "packages/ptywright", - "packages/cli" + "packages/cli", + "packages/pi-extension" ], "devDependencies": { "@types/node": "22.18.4", @@ -3514,6 +3515,10 @@ "resolved": "packages/cli", "link": true }, + "node_modules/@onkernel/cua-pi-extension": { + "resolved": "packages/pi-extension", + "link": true + }, "node_modules/@onkernel/ptywright": { "resolved": "packages/ptywright", "link": true @@ -6153,12 +6158,12 @@ }, "packages/agent": { "name": "@onkernel/cua-agent", - "version": "0.10.0", + "version": "0.10.1", "license": "MIT", "dependencies": { "@earendil-works/pi-agent-core": "0.83.0", "@earendil-works/pi-ai": "0.83.0", - "@onkernel/cua-ai": "0.10.0", + "@onkernel/cua-ai": "0.10.1", "@onkernel/sdk": "0.49.0", "sharp": "^0.35.3" }, @@ -6172,7 +6177,7 @@ }, "packages/ai": { "name": "@onkernel/cua-ai", - "version": "0.10.0", + "version": "0.10.1", "license": "MIT", "dependencies": { "@earendil-works/pi-ai": "0.83.0", @@ -6211,6 +6216,58 @@ "node": ">=22.19.0" } }, + "packages/cli/node_modules/@onkernel/cua-agent": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/@onkernel/cua-agent/-/cua-agent-0.10.0.tgz", + "integrity": "sha512-5Naa3ouUCQJt9kEWfAF9S/hpJn0mhR10p995dMRKve67ikCCwIHWfoM3B21FoVLmGhJY0dnzIAn4CI3iSZEu0Q==", + "license": "MIT", + "dependencies": { + "@earendil-works/pi-agent-core": "0.83.0", + "@earendil-works/pi-ai": "0.83.0", + "@onkernel/cua-ai": "0.10.0", + "@onkernel/sdk": "0.49.0", + "sharp": "^0.35.3" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "packages/cli/node_modules/@onkernel/cua-ai": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/@onkernel/cua-ai/-/cua-ai-0.10.0.tgz", + "integrity": "sha512-1ts3CnfartNu3UeaId8hUoXgdv3X58t+/4WKU9zVgHDxbstOy6V43sc7vU59seJ0+QZAmb+QmKtCdNk7ivSc2w==", + "license": "MIT", + "dependencies": { + "@earendil-works/pi-ai": "0.83.0", + "@tzafon/lightcone": "^0.7.0", + "openai": "^6.26.0" + }, + "engines": { + "node": ">=22.19.0" + } + }, + "packages/pi-extension": { + "name": "@onkernel/cua-pi-extension", + "version": "0.10.1", + "license": "MIT", + "dependencies": { + "@onkernel/cua-agent": "0.10.1", + "@onkernel/cua-ai": "0.10.1", + "@onkernel/sdk": "0.49.0" + }, + "devDependencies": { + "vitest": "^3.2.4" + }, + "engines": { + "node": ">=22.19.0" + }, + "peerDependencies": { + "@earendil-works/pi-agent-core": "*", + "@earendil-works/pi-ai": "*", + "@earendil-works/pi-coding-agent": "*", + "@earendil-works/pi-tui": "*" + } + }, "packages/ptywright": { "name": "@onkernel/ptywright", "version": "0.1.0", diff --git a/package.json b/package.json index d6f57a8..aa3a743 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,8 @@ "packages/ai", "packages/agent", "packages/ptywright", - "packages/cli" + "packages/cli", + "packages/pi-extension" ], "scripts": { "build": "npm run build --workspace @onkernel/cua-ai && npm run build --workspace @onkernel/cua-agent && tsc -b && npm run build --workspace @onkernel/cua-cli && npm run build:native --workspace @onkernel/ptywright --if-present", diff --git a/packages/agent/CHANGELOG.md b/packages/agent/CHANGELOG.md index 78f9f33..9385a40 100644 --- a/packages/agent/CHANGELOG.md +++ b/packages/agent/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 0.10.1 - 2026-08-06 + +- Update `@onkernel/cua-ai` to 0.10.1 for Anthropic's documented native + computer declaration versions. + ## 0.10.0 - 2026-08-04 Breaking: upgrade `@earendil-works/pi-agent-core` and `@earendil-works/pi-ai` diff --git a/packages/agent/README.md b/packages/agent/README.md index 93b9205..9bd17d6 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -132,7 +132,9 @@ Provider-native declarations compose with ordinary function tools: ```ts const tools = [ cua.providers.anthropic.tools.computer({ - version: "20260701", + version: "20251124", + displayWidth: 1440, + displayHeight: 900, enableZoom: true, }), cua.tools.browser.snapshot(), diff --git a/packages/agent/package.json b/packages/agent/package.json index 6fcccca..ba5002a 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -1,6 +1,6 @@ { "name": "@onkernel/cua-agent", - "version": "0.10.0", + "version": "0.10.1", "description": "Kernel browser computer-use Agent and AgentHarness classes built on pi-agent-core", "license": "MIT", "type": "module", @@ -45,7 +45,7 @@ "dependencies": { "@earendil-works/pi-agent-core": "0.83.0", "@earendil-works/pi-ai": "0.83.0", - "@onkernel/cua-ai": "0.10.0", + "@onkernel/cua-ai": "0.10.1", "@onkernel/sdk": "0.49.0", "sharp": "^0.35.3" }, diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 2d336ad..b3a43c9 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 0.10.1 - 2026-08-06 + +- Add Anthropic's documented `computer_20251124` and `computer_20250124` + declaration versions while retaining the existing early-access version. + ## 0.10.0 - 2026-08-04 Breaking: upgrade `@earendil-works/pi-ai` to 0.83.0. diff --git a/packages/ai/README.md b/packages/ai/README.md index 23542c5..9732e1f 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -136,7 +136,9 @@ function tools. ```ts const tools = [ cua.providers.anthropic.tools.computer({ - version: "20260701", + version: "20251124", + displayWidth: 1440, + displayHeight: 900, enableZoom: true, }), cua.tools.browser.snapshot(), @@ -149,6 +151,8 @@ Available groups: cua.providers.openai.tools.computer(); cua.providers.anthropic.source; +cua.providers.anthropic.tools.computer({ version: "20250124" }); +cua.providers.anthropic.tools.computer({ version: "20251124", enableZoom: true }); cua.providers.anthropic.tools.computer({ version: "20260701" }); cua.providers.anthropic.tools.browser({ version: "20260701" }); @@ -173,8 +177,11 @@ Moonshot accepts the ordinary browser toolset, including `browser_wait_for`, but rejects `browser_act`'s substantially larger function schema. Catalog compilation rejects that specific combination before a provider request. -Provider-native caller-visible names are fixed by protocol. Version/tool/model -mismatches fail during catalog compilation. If an Anthropic credential cannot +Provider-native caller-visible names are fixed by protocol. Anthropic computer +versions `20250124` and `20251124` emit their documented display dimensions and +beta headers; the early-access `20260701` surface remains available. Version, +tool, and model mismatches fail during catalog compilation. If an Anthropic +credential cannot access `browser_20260701`, CUA retries with an equivalent `browser` function tool and remembers that choice for the credential and process. Every `cua.providers.*` tool surface exposes its first-party `source` (or versioned diff --git a/packages/ai/package.json b/packages/ai/package.json index 31d39c1..e8f91b7 100644 --- a/packages/ai/package.json +++ b/packages/ai/package.json @@ -1,6 +1,6 @@ { "name": "@onkernel/cua-ai", - "version": "0.10.0", + "version": "0.10.1", "description": "Kernel-curated computer-use model access built on pi-ai", "license": "MIT", "type": "module", diff --git a/packages/ai/src/cua.ts b/packages/ai/src/cua.ts index a745e75..e998a8e 100644 --- a/packages/ai/src/cua.ts +++ b/packages/ai/src/cua.ts @@ -300,26 +300,43 @@ function playwright(options: CuaToolNameOptions = {}): CuaToolSpec { }); } -function anthropicNativeComputer(options: { version: "20260701"; enableZoom?: boolean; displayNumber?: number } = { version: "20260701" }): CuaToolSpec { - if (options.version !== "20260701") throw new Error(`unsupported Anthropic native computer version "${String(options.version)}"`); +type AnthropicNativeComputerOptions = { + version: "20250124" | "20251124" | "20260701"; + displayWidth?: number; + displayHeight?: number; + displayNumber?: number; + enableZoom?: boolean; +}; + +function anthropicNativeComputer(options: AnthropicNativeComputerOptions = { version: "20260701" }): CuaToolSpec { + if (options.version === "20250124" && options.enableZoom !== undefined) { + throw new Error("Anthropic computer_20250124 does not support enable_zoom"); + } + const current = options.version === "20250124" || options.version === "20251124"; const declaration = { - type: "computer_20260701", + type: `computer_${options.version}`, name: "computer", + ...(current ? { display_width_px: options.displayWidth ?? 1920, display_height_px: options.displayHeight ?? 1080 } : {}), ...(options.enableZoom !== undefined ? { enable_zoom: options.enableZoom } : {}), ...(options.displayNumber !== undefined ? { display_number: options.displayNumber } : {}), }; + const beta = options.version === "20260701" ? "computer-use-2026-07-01" : `computer-use-${formatAnthropicVersion(options.version)}`; return providerNativeSpec({ - identity: "provider.anthropic.native.computer.20260701", + identity: `provider.anthropic.native.computer.${options.version}`, name: "computer", source: providerSources.anthropic, declaration, - binding: { kind: "anthropic-native", declaration, beta: "computer-use-2026-07-01" }, + binding: { kind: "anthropic-native", declaration, beta }, toActions: (input) => mapNativeComputerInput(asNativeInput(input)), coordinates: pixels, stopTurnOnFailureMessage: "Not executed: an earlier computer action in this turn failed.", }); } +function formatAnthropicVersion(version: "20250124" | "20251124"): string { + return `${version.slice(0, 4)}-${version.slice(4, 6)}-${version.slice(6, 8)}`; +} + function anthropicNativeBrowser(options: { version: "20260701"; javascript?: boolean } = { version: "20260701" }): CuaToolSpec { if (options.version !== "20260701") throw new Error(`unsupported Anthropic native browser version "${String(options.version)}"`); const declaration = { diff --git a/packages/ai/test/anthropic-native.integration.test.ts b/packages/ai/test/anthropic-native.integration.test.ts index d187a0e..e4aa4e5 100644 --- a/packages/ai/test/anthropic-native.integration.test.ts +++ b/packages/ai/test/anthropic-native.integration.test.ts @@ -13,23 +13,33 @@ const viewport = { width: 1440, height: 900 }; const cases = [ { name: "computer", - tool: cua.providers.anthropic.tools.computer({ version: "20260701", enableZoom: true }), + model: "anthropic:claude-fable-5" as const, + tool: cua.providers.anthropic.tools.computer({ + version: "20251124", + displayWidth: viewport.width, + displayHeight: viewport.height, + enableZoom: true, + }), prompt: "Use the computer tool to take one screenshot.", expectedAction: "screenshot", + enabled: true, }, { name: "browser", + model: "anthropic:claude-opus-5" as const, tool: cua.providers.anthropic.tools.browser({ version: "20260701", javascript: true }), prompt: "Use the browser tool to navigate to example.com.", expectedAction: "navigate", + enabled: process.env.ANTHROPIC_BROWSER_20260701 === "1", }, ] as const; describe("Anthropic early-access native tools", () => { for (const current of cases) { - liveIt(`${current.name} survives catalog and pi-ai serialization`, async () => { + const run = current.enabled ? liveIt : it.skip; + run(`${current.name} survives catalog and pi-ai serialization`, async () => { const catalog = compileCuaToolCatalog({ - model: "anthropic:claude-opus-5", + model: current.model, requestedTools: [current.tool], viewport, }); diff --git a/packages/ai/test/tool-catalog.test.ts b/packages/ai/test/tool-catalog.test.ts index 2e70fb2..206a85c 100644 --- a/packages/ai/test/tool-catalog.test.ts +++ b/packages/ai/test/tool-catalog.test.ts @@ -219,6 +219,33 @@ describe("compileCuaToolCatalog", () => { expect(catalog.incoming.tzafonComputerName).toBe("computer"); }); + it("serializes Anthropic's documented native computer declaration", async () => { + const tool = cua.providers.anthropic.tools.computer({ + version: "20251124", + displayWidth: 1440, + displayHeight: 900, + enableZoom: true, + }); + const catalog = compile("anthropic:claude-fable-5", [tool]); + expect(catalog.headers.merge()).toEqual({ "anthropic-beta": "computer-use-2025-11-24" }); + const next = await catalog.payload.apply({ tools: [{ name: "computer", input_schema: {} }] }, catalog.model) as { + tools: Array>; + }; + expect(next.tools[0]).toMatchObject({ + type: "computer_20251124", + name: "computer", + display_width_px: 1440, + display_height_px: 900, + enable_zoom: true, + }); + + const legacy = compile("anthropic:claude-fable-5", [ + cua.providers.anthropic.tools.computer({ version: "20250124", displayWidth: 1280, displayHeight: 720 }), + ]); + expect(legacy.headers.merge()).toEqual({ "anthropic-beta": "computer-use-2025-01-24" }); + expect(() => cua.providers.anthropic.tools.computer({ version: "20250124", enableZoom: true })).toThrow("enable_zoom"); + }); + it("composes Anthropic native browser declarations, access fallback, and ordinary functions", async () => { const catalog = compile("anthropic:claude-opus-5", [ cua.providers.anthropic.tools.browser(), diff --git a/packages/pi-extension/CHANGELOG.md b/packages/pi-extension/CHANGELOG.md new file mode 100644 index 0000000..27bf940 --- /dev/null +++ b/packages/pi-extension/CHANGELOG.md @@ -0,0 +1,6 @@ +# Changelog + +## 0.10.1 + +- Initial pi extension with explicit CUA function tools. +- Register CUA's Anthropic provider wrapper and expose its native computer tool. diff --git a/packages/pi-extension/README.md b/packages/pi-extension/README.md new file mode 100644 index 0000000..74e0ab5 --- /dev/null +++ b/packages/pi-extension/README.md @@ -0,0 +1,75 @@ +# @onkernel/cua-pi-extension + +An installable [pi](https://pi.dev) extension that adds explicit Kernel browser +tools to pi's existing agent session. It supports CUA function tools and +Anthropic's native computer tool. It does not start `cua`, create a second model +loop, or add implicit screenshots or prompt instructions. + +## Install + +```sh +pi install ./packages/pi-extension +# or after publishing +pi install npm:@onkernel/cua-pi-extension +``` + +Function tools require `KERNEL_API_KEY` when first called. Anthropic native +computer use provisions the browser before the first provider request so its +declared display dimensions match the session viewport. `KERNEL_BASE_URL` is +honored. The extension never writes either value to session entries or output. + +## Use + +No selector means no CUA tool is active and no browser is provisioned. + +```sh +pi -p --provider openai --model gpt-5.6-sol \ + --cua-tools browser,browser-act "Open example.com and report its heading" + +pi --mode rpc --no-session --provider openai --model gpt-5.6-sol \ + --cua-tools browser + +pi -p --provider anthropic --model claude-fable-5 \ + --cua-tools anthropic-computer \ + "Open example.com and report its heading" +``` + +Use `/cua` to inspect the selected tools and browser ownership. Use +`/cua-tools browser,browser-act` to replace the session-local selection. +The command persists only selectors and browser metadata in pi's active branch. + +## Selectors + +- `browser`: `browser_snapshot`, `browser_text`, `browser_find`, + `browser_click`, `browser_hover`, `browser_drag`, `browser_fill`, + `browser_scroll_to`, `browser_scroll`, `browser_type`, `browser_key`, + `browser_navigate`, `browser_list_tabs`, `browser_new_tab`, + `browser_screenshot`, `browser_evaluate`, `browser_wait_for`. +- `computer`: `computer_click`, `computer_double_click`, + `computer_mouse_down`, `computer_mouse_up`, `computer_type`, + `computer_keypress`, `computer_scroll`, `computer_move`, `computer_drag`, + `computer_wait`, `computer_screenshot`, `computer_goto`, `computer_back`, + `computer_forward`, `computer_url`, `computer_cursor_position`. +- `mixed`: computer followed by browser. `browser-act`, `browser-batch`, + `computer-batch`, and `playwright` add one corresponding function tool. + Individual canonical function-tool names are also selectors. +- `anthropic-computer`: Anthropic's native `computer_20251124` tool, including + Claude Fable 5, Sonnet 5, Opus 5, and supported later revisions. + +Flags: `--cua-coordinates pixels|normalized-1000`, +`--cua-browser-session ID`, `--cua-profile-id ID`, `--cua-proxy-id ID`, +`--cua-browser-timeout SECONDS`, and `--cua-profile-save-changes`. +An attached session cannot be combined with profile or proxy flags. The +extension deletes only browsers it created at normal pi session shutdown. + +## Limits + +This version supports CUA function tools and Anthropic's documented native +computer-use protocol. The extension registers CUA's Anthropic provider wrapper +under the standard `anthropic` provider id; ordinary Anthropic requests continue +to delegate to pi's built-in transport. + +Provider-native Anthropic browser use, OpenAI, Google, Tzafon, and Yutori calls +remain unsupported. +Browser state can survive when attached, but element refs are process-local; +take a fresh snapshot after reload, resume, or fork. diff --git a/packages/pi-extension/package.json b/packages/pi-extension/package.json new file mode 100644 index 0000000..5065168 --- /dev/null +++ b/packages/pi-extension/package.json @@ -0,0 +1,56 @@ +{ + "name": "@onkernel/cua-pi-extension", + "version": "0.10.1", + "description": "Kernel CUA browser tools for pi", + "license": "MIT", + "type": "module", + "repository": { + "type": "git", + "url": "git+https://github.com/kernel/cua.git", + "directory": "packages/pi-extension" + }, + "homepage": "https://github.com/kernel/cua/tree/main/packages/pi-extension#readme", + "keywords": [ + "pi-package", + "pi-extension", + "computer-use", + "kernel" + ], + "pi": { + "extensions": [ + "./src/index.ts" + ] + }, + "files": [ + "src", + "README.md", + "CHANGELOG.md", + "package.json" + ], + "publishConfig": { + "access": "public" + }, + "engines": { + "node": ">=22.19.0" + }, + "scripts": { + "build": "tsc -b", + "typecheck": "tsc -b", + "clean": "tsc -b --clean && rm -rf dist-tsc", + "test": "vitest --run" + }, + "dependencies": { + "@onkernel/cua-ai": "0.10.1", + "@onkernel/cua-agent": "0.10.1", + "@onkernel/sdk": "0.49.0" + }, + "peerDependencies": { + "@earendil-works/pi-ai": "*", + "@earendil-works/pi-agent-core": "*", + "@earendil-works/pi-coding-agent": "*", + "@earendil-works/pi-tui": "*" + }, + "devDependencies": { + "vitest": "^3.2.4" + } +} diff --git a/packages/pi-extension/src/browser-runtime.ts b/packages/pi-extension/src/browser-runtime.ts new file mode 100644 index 0000000..590b0c6 --- /dev/null +++ b/packages/pi-extension/src/browser-runtime.ts @@ -0,0 +1,89 @@ +import Kernel from "@onkernel/sdk"; +import { CuaExecutionResources } from "@onkernel/cua-agent"; + +export interface BrowserOptions { + sessionId?: string; + profileId?: string; + proxyId?: string; + timeoutSeconds: number; + saveProfileChanges: boolean; +} +export interface BrowserStatus { + sessionId?: string; + owned?: boolean; + liveUrl?: string; + createdAt?: string; +} + +/** Lazily provisions one browser for one pi session. Attached sessions are never deleted. */ +export class CuaBrowserRuntime { + private pending?: Promise; + private resources?: CuaExecutionResources; + private client?: Kernel; + private status: BrowserStatus = {}; + private closed = false; + constructor( + private readonly options: BrowserOptions, + private readonly env: NodeJS.ProcessEnv = process.env, + ) {} + + getStatus(): BrowserStatus { + return { ...this.status }; + } + async get(signal?: AbortSignal): Promise { + if (signal?.aborted) throw new Error("CUA browser provisioning cancelled"); + if (this.closed) throw new Error("CUA browser runtime is closed"); + if (this.resources) return this.resources; + this.pending ??= this.provision(); + try { + const resources = await this.pending; + if (this.closed) throw new Error("CUA browser runtime is closed"); + this.resources = resources; + return resources; + } catch (error) { + this.pending = undefined; + throw error; + } + } + private async provision(): Promise { + const apiKey = this.env.KERNEL_API_KEY; + if (!apiKey) throw new Error("KERNEL_API_KEY is required when a CUA tool first executes"); + const client = new Kernel({ apiKey, ...(this.env.KERNEL_BASE_URL ? { baseURL: this.env.KERNEL_BASE_URL } : {}) }); + const attached = Boolean(this.options.sessionId); + const browser = attached + ? await client.browsers.retrieve(this.options.sessionId!) + : await client.browsers.create({ + stealth: true, + timeout_seconds: this.options.timeoutSeconds, + ...(this.options.profileId ? { profile: { id: this.options.profileId, save_changes: this.options.saveProfileChanges } } : {}), + ...(this.options.proxyId ? { proxy_id: this.options.proxyId } : {}), + }); + this.client = client; + this.status = { + sessionId: browser.session_id, + owned: !attached, + liveUrl: browser.browser_live_view_url, + createdAt: browser.created_at, + }; + return new CuaExecutionResources({ browser, client }); + } + async close(): Promise { + if (this.closed) return; + this.closed = true; + // A shutdown can race the first tool call. Wait for provisioning so an owned + // browser created after shutdown starts is still disposed and deleted. + let pendingResources: CuaExecutionResources | undefined; + try { + pendingResources = await this.pending; + } catch { + /* provisioning failure needs no cleanup */ + } + const resources = this.resources ?? pendingResources; + this.resources = undefined; + try { + await resources?.dispose(); + } finally { + if (this.status.owned && this.status.sessionId && this.client) await this.client.browsers.deleteByID(this.status.sessionId); + } + } +} diff --git a/packages/pi-extension/src/catalog.ts b/packages/pi-extension/src/catalog.ts new file mode 100644 index 0000000..f83d76c --- /dev/null +++ b/packages/pi-extension/src/catalog.ts @@ -0,0 +1,201 @@ +import type { Api, Model } from "@earendil-works/pi-ai"; +import { compileCuaToolCatalog, cua, type CuaToolCatalog, type CuaToolSpec } from "@onkernel/cua-ai"; + +export const BROWSER_BATCH_ACTIONS = [ + "snapshot", + "text", + "find", + "click", + "hover", + "drag", + "fill", + "scroll_to", + "scroll", + "type", + "key", + "navigate", + "list_tabs", + "new_tab", + "screenshot", + "evaluate", + "wait_for", +] as const; +export const COMPUTER_BATCH_ACTIONS = [ + "click", + "double_click", + "mouse_down", + "mouse_up", + "type", + "keypress", + "scroll", + "move", + "drag", + "wait", + "screenshot", + "zoom", + "goto", + "back", + "forward", + "url", + "cursor_position", +] as const; + +type Coordinates = "pixels" | "normalized-1000"; +type CoordinateSystem = ReturnType | ReturnType; + +export interface CuaSelection { + selectors: readonly string[]; + coordinates: Coordinates; +} + +export const DEFAULT_VIEWPORT: Readonly<{ width: number; height: number }> = Object.freeze({ width: 1920, height: 1080 }); + +const generalTools = Object.freeze({ + browser_snapshot: () => cua.tools.browser.snapshot(), + browser_text: () => cua.tools.browser.text(), + browser_find: () => cua.tools.browser.find(), + browser_click: () => cua.tools.browser.click(), + browser_hover: () => cua.tools.browser.hover(), + browser_drag: () => cua.tools.browser.drag(), + browser_fill: () => cua.tools.browser.fill(), + browser_scroll_to: () => cua.tools.browser.scrollTo(), + browser_scroll: () => cua.tools.browser.scroll(), + browser_type: () => cua.tools.browser.type(), + browser_key: () => cua.tools.browser.key(), + browser_navigate: () => cua.tools.browser.navigate(), + browser_list_tabs: () => cua.tools.browser.listTabs(), + browser_new_tab: () => cua.tools.browser.newTab(), + browser_screenshot: () => cua.tools.browser.screenshot(), + browser_evaluate: () => cua.tools.browser.evaluate(), + browser_wait_for: () => cua.tools.browser.waitFor(), + browser_act: () => cua.tools.browser.act(), + playwright_execute: () => cua.tools.playwright(), +}); + +const computerTools = Object.freeze({ + computer_click: (coordinates: CoordinateSystem) => cua.tools.computer.click({ coordinates }), + computer_double_click: (coordinates: CoordinateSystem) => cua.tools.computer.doubleClick({ coordinates }), + computer_mouse_down: (coordinates: CoordinateSystem) => cua.tools.computer.mouseDown({ coordinates }), + computer_mouse_up: (coordinates: CoordinateSystem) => cua.tools.computer.mouseUp({ coordinates }), + computer_type: (coordinates: CoordinateSystem) => cua.tools.computer.type({ coordinates }), + computer_keypress: (coordinates: CoordinateSystem) => cua.tools.computer.keypress({ coordinates }), + computer_scroll: (coordinates: CoordinateSystem) => cua.tools.computer.scroll({ coordinates }), + computer_move: (coordinates: CoordinateSystem) => cua.tools.computer.move({ coordinates }), + computer_drag: (coordinates: CoordinateSystem) => cua.tools.computer.drag({ coordinates }), + computer_wait: (coordinates: CoordinateSystem) => cua.tools.computer.wait({ coordinates }), + computer_screenshot: (coordinates: CoordinateSystem) => cua.tools.computer.screenshot({ coordinates }), + computer_zoom: (coordinates: CoordinateSystem) => cua.tools.computer.zoom({ coordinates }), + computer_goto: (coordinates: CoordinateSystem) => cua.tools.computer.goto({ coordinates }), + computer_back: (coordinates: CoordinateSystem) => cua.tools.computer.back({ coordinates }), + computer_forward: (coordinates: CoordinateSystem) => cua.tools.computer.forward({ coordinates }), + computer_url: (coordinates: CoordinateSystem) => cua.tools.computer.url({ coordinates }), + computer_cursor_position: (coordinates: CoordinateSystem) => cua.tools.computer.cursorPosition({ coordinates }), +}); + +export const CUA_TOOL_NAMES = Object.freeze([...Object.keys(generalTools), ...Object.keys(computerTools)]); +export const CUA_SELECTORS = Object.freeze([ + "browser", + "computer", + "mixed", + "browser-act", + "browser-batch", + "computer-batch", + "playwright", + "anthropic-computer", + ...CUA_TOOL_NAMES, +]); + +export function parseSelection(value: string | undefined, coordinates: string | undefined): CuaSelection { + const coordinateMode = coordinates ?? "pixels"; + if (coordinateMode !== "pixels" && coordinateMode !== "normalized-1000") { + throw new Error('--cua-coordinates must be "pixels" or "normalized-1000"'); + } + const selectors = + value + ?.split(",") + .map((item) => item.trim()) + .filter(Boolean) ?? []; + if (new Set(selectors).size !== selectors.length) throw new Error("--cua-tools contains duplicate selectors"); + for (const selector of selectors) { + if (!CUA_SELECTORS.includes(selector)) throw new Error(`unknown CUA tool selector "${selector}"`); + } + return Object.freeze({ selectors: Object.freeze(selectors), coordinates: coordinateMode }); +} + +/** Every function tool that can be selected, with declarations for one coordinate mode. */ +export function allSelectableSpecs(coordinates: Coordinates, viewport = DEFAULT_VIEWPORT): CuaToolSpec[] { + const result = new Map(); + for (const selector of CUA_SELECTORS) { + for (const spec of expandSelection(parseSelection(selector, coordinates), viewport)) result.set(spec.name, spec); + } + return [...result.values()]; +} + +export function expandSelection(selection: CuaSelection, viewport = DEFAULT_VIEWPORT): CuaToolSpec[] { + const coordinates = selection.coordinates === "pixels" ? cua.coordinates.pixels() : cua.coordinates.normalized([0, 1000]); + const result: CuaToolSpec[] = []; + for (const selector of selection.selectors) { + switch (selector) { + case "browser": + result.push(...cua.toolsets.browser()); + break; + case "computer": + result.push(...cua.toolsets.computer({ coordinates })); + break; + case "mixed": + result.push(...cua.toolsets.mixed({ coordinates })); + break; + case "browser-act": + result.push(cua.tools.browser.act()); + break; + case "browser-batch": + result.push(cua.tools.browser.batch({ actions: BROWSER_BATCH_ACTIONS })); + break; + case "computer-batch": + result.push(cua.tools.computer.batch({ actions: COMPUTER_BATCH_ACTIONS, coordinates })); + break; + case "playwright": + result.push(cua.tools.playwright()); + break; + case "anthropic-computer": + result.push( + cua.providers.anthropic.tools.computer({ + version: "20251124", + displayWidth: viewport.width, + displayHeight: viewport.height, + enableZoom: true, + }), + ); + break; + default: + result.push(createIndividualTool(selector, coordinates)); + } + } + const identities = new Set(); + for (const spec of result) { + if (identities.has(spec.identity)) throw new Error(`CUA selection contains duplicate tool identity "${spec.identity}"`); + identities.add(spec.identity); + } + return result; +} + +function createIndividualTool(name: string, coordinates: CoordinateSystem): CuaToolSpec { + const createComputerTool = computerTools[name as keyof typeof computerTools]; + if (createComputerTool) return createComputerTool(coordinates); + const createGeneralTool = generalTools[name as keyof typeof generalTools]; + if (createGeneralTool) return createGeneralTool(); + throw new Error(`unknown CUA tool selector "${name}"`); +} + +export function compileSpecs(model: Model, specs: readonly CuaToolSpec[], viewport = DEFAULT_VIEWPORT): CuaToolCatalog { + return compileCuaToolCatalog({ model, requestedTools: specs, viewport }); +} + +export function compileSelection( + model: Model, + selection: CuaSelection, + viewport = DEFAULT_VIEWPORT, +): { specs: CuaToolSpec[]; catalog: CuaToolCatalog } { + const specs = expandSelection(selection, viewport); + return { specs, catalog: compileSpecs(model, specs, viewport) }; +} diff --git a/packages/pi-extension/src/index.ts b/packages/pi-extension/src/index.ts new file mode 100644 index 0000000..98cd736 --- /dev/null +++ b/packages/pi-extension/src/index.ts @@ -0,0 +1,255 @@ +import { fileURLToPath } from "node:url"; +import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; +import { createCuaModels, type CuaToolSpec } from "@onkernel/cua-ai"; +import { allSelectableSpecs, compileSpecs, DEFAULT_VIEWPORT, expandSelection, parseSelection, type CuaSelection } from "./catalog"; +import { CuaBrowserRuntime, type BrowserOptions } from "./browser-runtime"; +import { CONFIG_ENTRY, restoreConfig, type PersistedConfig } from "./state"; +import { statusText } from "./render"; + +export default function cuaPiExtension(pi: ExtensionAPI): void { + pi.registerFlag("cua-tools", { type: "string", description: "Comma-separated explicit CUA tool selectors" }); + pi.registerFlag("cua-coordinates", { type: "string", description: "pixels or normalized-1000", default: "pixels" }); + pi.registerFlag("cua-browser-session", { type: "string", description: "Attach an existing Kernel browser session" }); + pi.registerFlag("cua-profile-id", { type: "string", description: "Kernel browser profile id" }); + pi.registerFlag("cua-proxy-id", { type: "string", description: "Kernel proxy id" }); + pi.registerFlag("cua-browser-timeout", { type: "string", description: "Owned browser timeout in seconds", default: "300" }); + pi.registerFlag("cua-profile-save-changes", { type: "boolean", description: "Save owned browser profile changes", default: false }); + // Parsed flag values are unavailable until after the extension factory returns, + // but session_start errors do not stop print/RPC provider calls. + validateRawCliFlags(); + + const extensionPath = fileURLToPath(import.meta.url); + let selection = parseSelection(undefined, "pixels"); + let browserOptions: BrowserOptions = defaultBrowserOptions(); + let activeNames = new Set(); + let compatibilityError: string | undefined; + let initialized = false; + let forcedInactive = false; + let sessionActive = false; + let runtime: CuaBrowserRuntime | undefined; + let allSpecs = new Map(); + + function configureDeclarations(): void { + allSpecs = new Map(allSelectableSpecs(selection.coordinates).map((spec) => [spec.name, spec])); + } + function installTools(): void { + for (const [name, spec] of allSpecs) { + const conflict = pi.getAllTools().find((tool) => tool.name === name); + if (conflict && conflict.sourceInfo.path !== extensionPath) { + throw new Error(`cannot register CUA tool "${name}": already owned by ${conflict.sourceInfo.source}`); + } + pi.registerTool({ + name: spec.name, + label: spec.name, + description: spec.declaration.description, + parameters: spec.declaration.parameters, + executionMode: "sequential", + async execute(toolCallId, input, signal) { + if (!activeNames.has(name)) throw new Error(`CUA tool "${name}" is not active`); + const selected = currentSpecs().find((candidate) => candidate.name === name); + if (!selected || compatibilityError) throw new Error(compatibilityError ?? `CUA tool "${name}" is no longer selected`); + const resources = await ensureRuntime().get(signal); + return resources.materialize(selected).execute(toolCallId, input, signal); + }, + }); + } + } + function ensureRuntime(): CuaBrowserRuntime { + if (!sessionActive) throw new Error("CUA browser runtime is unavailable outside an active pi session"); + return (runtime ??= new CuaBrowserRuntime(browserOptions)); + } + function currentSpecs(viewport = DEFAULT_VIEWPORT): CuaToolSpec[] { + return expandSelection(selection, viewport); + } + function activeSpecs(viewport = DEFAULT_VIEWPORT): CuaToolSpec[] { + return currentSpecs(viewport).filter((spec) => activeNames.has(spec.name)); + } + function persistCommandSelection(): void { + const state: PersistedConfig = { + version: 1, + origin: "command", + selectors: [...selection.selectors], + coordinates: selection.coordinates, + browser: runtime?.getStatus(), + }; + pi.appendEntry(CONFIG_ENTRY, state); + } + function reconcile(ctx: ExtensionContext, activateInitial = false): void { + const specs = currentSpecs(); + const current = pi.getActiveTools(); + const selectedNames = specs.map((spec) => spec.name); + const priorCua = current.filter((name) => allSpecs.has(name)); + // After an extension-forced incompatibility deactivation, restore the selected + // set when the next model is compatible. A user /tools deactivation remains off. + const desired = + !initialized || activateInitial || forcedInactive ? selectedNames : priorCua.filter((name) => selectedNames.includes(name)); + try { + if (desired.length && !ctx.model) throw new Error("no pi model is selected"); + if (desired.length && ctx.model) + compileSpecs( + ctx.model, + specs.filter((spec) => desired.includes(spec.name)), + DEFAULT_VIEWPORT, + ); + compatibilityError = undefined; + forcedInactive = false; + activeNames = new Set(desired); + pi.setActiveTools([...current.filter((name) => !allSpecs.has(name)), ...desired]); + } catch (error) { + compatibilityError = error instanceof Error ? error.message : String(error); + forcedInactive = true; + activeNames = new Set(); + pi.setActiveTools(current.filter((name) => !allSpecs.has(name))); + } + initialized = true; + if (ctx.mode === "tui") + ctx.ui.setStatus("cua", statusText(selection.selectors, [...activeNames], runtime?.getStatus() ?? {}, compatibilityError)); + } + + const anthropic = createCuaModels().getProvider("anthropic"); + if (!anthropic) throw new Error("CUA Anthropic provider is unavailable"); + pi.registerProvider(anthropic); + + pi.registerCommand("cua", { + description: "Show CUA tool and browser status", + handler: async (_args, ctx) => { + reconcile(ctx); + ctx.ui.notify( + statusText(selection.selectors, [...activeNames], runtime?.getStatus() ?? {}, compatibilityError), + compatibilityError ? "error" : "info", + ); + }, + }); + pi.registerCommand("cua-tools", { + description: "Replace this session's explicit CUA selectors", + handler: async (args, ctx) => { + selection = parseSelection(args, selection.coordinates); + // All selectable names were registered with this session's coordinate mode. + reconcile(ctx, true); + persistCommandSelection(); + ctx.ui.notify( + statusText(selection.selectors, [...activeNames], runtime?.getStatus() ?? {}, compatibilityError), + compatibilityError ? "error" : "info", + ); + }, + }); + + // Pi creates a fresh extension instance after the previous instance finishes session_shutdown. + pi.on("session_start", (_event, ctx) => { + const flags = readFlags(pi); + selection = flags.selection; + browserOptions = flags.browserOptions; + const saved = restoreConfig(ctx.sessionManager.getBranch()); + if (saved) selection = parseSelection(saved.selectors.join(","), saved.coordinates); + configureDeclarations(); + installTools(); + initialized = false; + forcedInactive = false; + sessionActive = true; + reconcile(ctx, true); + }); + pi.on("model_select", (_event, ctx) => reconcile(ctx)); + pi.on("before_agent_start", (_event, ctx) => reconcile(ctx)); + pi.on("before_provider_headers", (event, ctx) => { + if (!activeNames.size || compatibilityError || !ctx.model) return; + const catalog = compileSpecs(ctx.model, activeSpecs(), DEFAULT_VIEWPORT); + Object.assign(event.headers, catalog.headers.merge(event.headers)); + }); + pi.on("before_provider_request", async (event, ctx) => { + reconcile(ctx); + if (!activeNames.size || compatibilityError || !ctx.model) { + // setActiveTools() normally removes CUA declarations before serialization. + // This hook is the final pre-wire guard for a model switch that invalidates + // a catalog after pi has already built a payload for the turn. + return currentSpecs().length ? withoutCuaToolSchemas(event.payload, allSpecs) : undefined; + } + let viewport = DEFAULT_VIEWPORT; + let specs = activeSpecs(); + if (specs.some((spec) => spec.name === "computer" && spec.providerBinding?.kind === "anthropic-native")) { + viewport = (await ensureRuntime().get()).viewport; + specs = activeSpecs(viewport); + } + return compileSpecs(ctx.model, specs, viewport).payload.apply(event.payload, ctx.model); + }); + pi.on("tool_call", (event) => { + if (!allSpecs.has(event.toolName)) return; + if (!activeNames.has(event.toolName) || compatibilityError) + return { block: true, reason: compatibilityError ?? `CUA tool "${event.toolName}" is inactive` }; + }); + pi.on("session_shutdown", async () => { + sessionActive = false; + const closingRuntime = runtime; + runtime = undefined; + await closingRuntime?.close(); + }); +} + +function validateRawCliFlags(argv = process.argv.slice(2)): void { + const read = (name: string): string | undefined => { + const equals = argv.find((arg) => arg.startsWith(`--${name}=`)); + if (equals) return equals.slice(name.length + 3); + const index = argv.indexOf(`--${name}`); + return index >= 0 && !argv[index + 1]?.startsWith("--") ? argv[index + 1] : undefined; + }; + parseSelection(read("cua-tools"), read("cua-coordinates") ?? "pixels"); + const sessionId = trim(read("cua-browser-session")); + if (sessionId && (trim(read("cua-profile-id")) || trim(read("cua-proxy-id")))) + throw new Error("--cua-browser-session cannot be combined with --cua-profile-id or --cua-proxy-id"); + positiveSeconds(read("cua-browser-timeout")); +} +function readFlags(pi: ExtensionAPI): { selection: CuaSelection; browserOptions: BrowserOptions } { + const browserOptions: BrowserOptions = { + sessionId: trim(asString(pi.getFlag("cua-browser-session"))), + profileId: trim(asString(pi.getFlag("cua-profile-id"))), + proxyId: trim(asString(pi.getFlag("cua-proxy-id"))), + timeoutSeconds: positiveSeconds(asString(pi.getFlag("cua-browser-timeout"))), + saveProfileChanges: pi.getFlag("cua-profile-save-changes") === true, + }; + if (browserOptions.sessionId && (browserOptions.profileId || browserOptions.proxyId)) + throw new Error("--cua-browser-session cannot be combined with --cua-profile-id or --cua-proxy-id"); + return { selection: parseSelection(asString(pi.getFlag("cua-tools")), asString(pi.getFlag("cua-coordinates"))), browserOptions }; +} +function defaultBrowserOptions(): BrowserOptions { + return { timeoutSeconds: 300, saveProfileChanges: false }; +} +function asString(value: boolean | string | undefined): string | undefined { + return typeof value === "string" ? value : undefined; +} +function trim(value: string | undefined): string | undefined { + const result = value?.trim(); + return result || undefined; +} +function positiveSeconds(value: string | undefined): number { + const seconds = Number(value ?? "300"); + if (!Number.isSafeInteger(seconds) || seconds < 1 || seconds > 259200) + throw new Error("--cua-browser-timeout must be a whole number from 1 to 259200"); + return seconds; +} + +function withoutCuaToolSchemas(payload: unknown, cuaSpecs: ReadonlyMap): unknown { + if (!isRecord(payload) || !Array.isArray(payload.tools)) return payload; + const tools: unknown[] = []; + for (const tool of payload.tools) { + if (isRecord(tool) && Array.isArray(tool.functionDeclarations)) { + const functionDeclarations = tool.functionDeclarations.filter((declaration) => { + const name = serializedToolName(declaration); + return !name || !cuaSpecs.has(name); + }); + if (functionDeclarations.length) tools.push({ ...tool, functionDeclarations }); + continue; + } + const name = serializedToolName(tool); + if (!name || !cuaSpecs.has(name)) tools.push(tool); + } + return { ...payload, tools }; +} + +function serializedToolName(tool: unknown): string | undefined { + if (!isRecord(tool)) return undefined; + if (typeof tool.name === "string") return tool.name; + return isRecord(tool.function) && typeof tool.function.name === "string" ? tool.function.name : undefined; +} + +function isRecord(value: unknown): value is Record { + return Boolean(value && typeof value === "object" && !Array.isArray(value)); +} diff --git a/packages/pi-extension/src/render.ts b/packages/pi-extension/src/render.ts new file mode 100644 index 0000000..96bd521 --- /dev/null +++ b/packages/pi-extension/src/render.ts @@ -0,0 +1,9 @@ +import type { BrowserStatus } from "./browser-runtime"; + +export function statusText(selectors: readonly string[], active: readonly string[], browser: BrowserStatus, error?: string): string { + const tools = active.length ? active.join(", ") : "none"; + const browserText = browser.sessionId + ? `${browser.owned ? "owned" : "attached"} ${browser.sessionId}${browser.liveUrl ? ` ${browser.liveUrl}` : ""}` + : "not provisioned"; + return `cua: selected=${selectors.join(",") || "none"}; active=${tools}; browser=${browserText}${error ? `; unavailable=${error}` : ""}`; +} diff --git a/packages/pi-extension/src/state.ts b/packages/pi-extension/src/state.ts new file mode 100644 index 0000000..5689efc --- /dev/null +++ b/packages/pi-extension/src/state.ts @@ -0,0 +1,32 @@ +import type { CuaSelection } from "./catalog"; + +export const CONFIG_ENTRY = "cua-pi-config-v1"; +export interface PersistedConfig { + version: 1; + origin: "command"; + selectors: string[]; + coordinates: CuaSelection["coordinates"]; + browser?: { sessionId?: string; owned?: boolean; liveUrl?: string; createdAt?: string }; +} +export function restoreConfig(entries: readonly unknown[]): PersistedConfig | undefined { + for (const entry of [...entries].reverse()) { + const candidate = entry as { type?: unknown; customType?: unknown; data?: unknown }; + if (candidate.type !== "custom" || candidate.customType !== CONFIG_ENTRY || !candidate.data || typeof candidate.data !== "object") + continue; + const data = candidate.data as Partial; + if ( + data.version === 1 && + data.origin === "command" && + Array.isArray(data.selectors) && + data.selectors.every((selector) => typeof selector === "string") && + (data.coordinates === "pixels" || data.coordinates === "normalized-1000") + ) { + return { + version: 1, + origin: "command", + selectors: data.selectors, + coordinates: data.coordinates, + }; + } + } +} diff --git a/packages/pi-extension/test/browser-runtime.test.ts b/packages/pi-extension/test/browser-runtime.test.ts new file mode 100644 index 0000000..9e6316f --- /dev/null +++ b/packages/pi-extension/test/browser-runtime.test.ts @@ -0,0 +1,84 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const state = vi.hoisted(() => ({ create: vi.fn(), retrieve: vi.fn(), deleteByID: vi.fn(), dispose: vi.fn() })); +vi.mock("@onkernel/sdk", () => ({ + default: class Kernel { + browsers = { create: state.create, retrieve: state.retrieve, deleteByID: state.deleteByID }; + }, +})); +vi.mock("@onkernel/cua-agent", () => ({ + CuaExecutionResources: class { + dispose = state.dispose; + constructor(_options: unknown) {} + }, +})); + +import { CuaBrowserRuntime } from "../src/browser-runtime"; + +const owned = { session_id: "owned", created_at: "2026-01-01T00:00:00Z", browser_live_view_url: "https://live" }; + +beforeEach(() => { + state.create.mockReset(); + state.retrieve.mockReset(); + state.deleteByID.mockReset(); + state.dispose.mockReset(); +}); + +describe("CuaBrowserRuntime", () => { + it("creates one shared owned browser for concurrent first calls and deletes it on close", async () => { + state.create.mockResolvedValue(owned); + const runtime = new CuaBrowserRuntime({ timeoutSeconds: 60, saveProfileChanges: false }, { KERNEL_API_KEY: "test" }); + const [first, second] = await Promise.all([runtime.get(), runtime.get()]); + expect(first).toBe(second); + expect(state.create).toHaveBeenCalledTimes(1); + expect(runtime.getStatus()).toMatchObject({ sessionId: "owned", owned: true, liveUrl: "https://live" }); + await runtime.close(); + expect(state.dispose).toHaveBeenCalledTimes(1); + expect(state.deleteByID).toHaveBeenCalledWith("owned"); + }); + + it("deletes an owned browser when resource disposal fails", async () => { + state.create.mockResolvedValue(owned); + state.dispose.mockRejectedValue(new Error("dispose failed")); + const runtime = new CuaBrowserRuntime({ timeoutSeconds: 60, saveProfileChanges: false }, { KERNEL_API_KEY: "test" }); + await runtime.get(); + await expect(runtime.close()).rejects.toThrow("dispose failed"); + expect(state.deleteByID).toHaveBeenCalledWith("owned"); + }); + + it("does not delete an attached browser", async () => { + state.retrieve.mockResolvedValue({ ...owned, session_id: "attached" }); + const runtime = new CuaBrowserRuntime( + { sessionId: "attached", timeoutSeconds: 60, saveProfileChanges: false }, + { KERNEL_API_KEY: "test" }, + ); + await runtime.get(); + await runtime.close(); + expect(state.retrieve).toHaveBeenCalledWith("attached"); + expect(state.deleteByID).not.toHaveBeenCalled(); + }); + + it("waits for in-flight provisioning during close and cleans up the resulting browser", async () => { + let resolve!: (value: typeof owned) => void; + state.create.mockReturnValue( + new Promise((done) => { + resolve = done; + }), + ); + const runtime = new CuaBrowserRuntime({ timeoutSeconds: 60, saveProfileChanges: false }, { KERNEL_API_KEY: "test" }); + const pending = runtime.get().catch(() => undefined); + const closing = runtime.close(); + resolve(owned); + await Promise.all([pending, closing]); + expect(state.deleteByID).toHaveBeenCalledWith("owned"); + }); + + it("fails before provisioning when cancelled or unconfigured", async () => { + const cancelled = new AbortController(); + cancelled.abort(); + await expect( + new CuaBrowserRuntime({ timeoutSeconds: 60, saveProfileChanges: false }, { KERNEL_API_KEY: "test" }).get(cancelled.signal), + ).rejects.toThrow("cancelled"); + await expect(new CuaBrowserRuntime({ timeoutSeconds: 60, saveProfileChanges: false }, {}).get()).rejects.toThrow("KERNEL_API_KEY"); + }); +}); diff --git a/packages/pi-extension/test/catalog.test.ts b/packages/pi-extension/test/catalog.test.ts new file mode 100644 index 0000000..c4a3f25 --- /dev/null +++ b/packages/pi-extension/test/catalog.test.ts @@ -0,0 +1,78 @@ +import { getCuaModel } from "@onkernel/cua-ai"; +import { describe, expect, it } from "vitest"; +import { + BROWSER_BATCH_ACTIONS, + compileSelection, + COMPUTER_BATCH_ACTIONS, + CUA_TOOL_NAMES, + expandSelection, + parseSelection, +} from "../src/catalog"; + +describe("CUA pi selectors", () => { + it("has stable exact browser and computer preset membership", () => { + expect(expandSelection(parseSelection("browser", "pixels")).map((tool) => tool.name)).toEqual([ + "browser_snapshot", + "browser_text", + "browser_find", + "browser_click", + "browser_hover", + "browser_drag", + "browser_fill", + "browser_scroll_to", + "browser_scroll", + "browser_type", + "browser_key", + "browser_navigate", + "browser_list_tabs", + "browser_new_tab", + "browser_screenshot", + "browser_evaluate", + "browser_wait_for", + ]); + expect(expandSelection(parseSelection("computer", "normalized-1000")).map((tool) => tool.name)).toEqual([ + "computer_click", + "computer_double_click", + "computer_mouse_down", + "computer_mouse_up", + "computer_type", + "computer_keypress", + "computer_scroll", + "computer_move", + "computer_drag", + "computer_wait", + "computer_screenshot", + "computer_goto", + "computer_back", + "computer_forward", + "computer_url", + "computer_cursor_position", + ]); + }); + it("expands special selectors without native provider tools", () => { + expect( + expandSelection(parseSelection("browser-act,browser-batch,computer-batch,playwright", "pixels")).map((tool) => tool.name), + ).toEqual(["browser_act", "browser_batch", "computer_batch", "playwright_execute"]); + expect(BROWSER_BATCH_ACTIONS).toHaveLength(17); + expect(COMPUTER_BATCH_ACTIONS).toHaveLength(17); + expect(CUA_TOOL_NAMES).not.toContain("computer"); + }); + it("compiles Anthropic native computer use only for supported Anthropic models", () => { + const selection = parseSelection("anthropic-computer", "pixels"); + const viewport = { width: 1440, height: 900 }; + const { specs, catalog } = compileSelection(getCuaModel("anthropic:claude-fable-5"), selection, viewport); + expect(specs.map((tool) => tool.name)).toEqual(["computer"]); + expect(catalog.entries[0]?.declaration).toMatchObject({ display_width_px: 1440, display_height_px: 900 }); + expect(catalog.entries.map((entry) => entry.transport)).toEqual(["native"]); + expect(catalog.headers.requirements).toContainEqual(expect.objectContaining({ value: "computer-use-2025-11-24" })); + expect(() => compileSelection(getCuaModel("openai:gpt-5.6-sol"), selection)).toThrow("requires a anthropic model"); + }); + + it("accepts an empty selection and rejects ambiguity", () => { + expect(expandSelection(parseSelection(undefined, undefined))).toEqual([]); + expect(() => parseSelection("browser,browser", "pixels")).toThrow("duplicate"); + expect(() => parseSelection("native-openai", "pixels")).toThrow("unknown"); + expect(() => parseSelection("browser", "screen")).toThrow("coordinates"); + expect(() => expandSelection(parseSelection("browser,browser_snapshot", "pixels"))).toThrow("duplicate tool identity"); + }); +}); diff --git a/packages/pi-extension/test/extension.test.ts b/packages/pi-extension/test/extension.test.ts new file mode 100644 index 0000000..6cb2b19 --- /dev/null +++ b/packages/pi-extension/test/extension.test.ts @@ -0,0 +1,281 @@ +import { fileURLToPath } from "node:url"; +import type { Api, Model, Provider } from "@earendil-works/pi-ai"; +import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; +import { getCuaModel } from "@onkernel/cua-ai"; +import { describe, expect, it, vi } from "vitest"; +import { CuaBrowserRuntime } from "../src/browser-runtime"; +import { allSelectableSpecs } from "../src/catalog"; +import extension from "../src/index"; + +type Handler = (event: unknown, ctx: ExtensionContext) => unknown; + +interface FakeTool { + name: string; + description: string; + sourceInfo: { source: string; path?: string }; +} + +interface FakeCommand { + handler(args: string, ctx: ExtensionContext): Promise | void; +} + +interface FakePi { + api: ExtensionAPI; + handlers: Map; + commands: Map; + tools: FakeTool[]; + entries: unknown[]; + providers: Provider[]; + readonly active: string[]; +} + +const extensionPath = fileURLToPath(new URL("../src/index.ts", import.meta.url)); + +function makePi(flags: Record): FakePi { + const handlers = new Map(); + const commands = new Map(); + const tools: FakeTool[] = []; + const entries: unknown[] = []; + const providers: Provider[] = []; + let active = ["bash"]; + const implementation = { + registerFlag() {}, + getFlag: (name: string) => flags[name], + registerTool: (tool: Omit) => { + const registered = { ...tool, sourceInfo: { source: "extension", path: extensionPath } }; + const existing = tools.findIndex((candidate) => candidate.name === tool.name); + if (existing >= 0) tools[existing] = registered; + else tools.push(registered); + }, + registerCommand: (name: string, command: FakeCommand) => commands.set(name, command), + registerProvider: (provider: Provider) => providers.push(provider), + on: (name: string, handler: Handler) => handlers.set(name, handler), + getAllTools: () => tools, + getActiveTools: () => active, + setActiveTools: (names: string[]) => { + active = names; + }, + appendEntry: (customType: string, data: unknown) => entries.push({ type: "custom", customType, data }), + }; + const result: FakePi = { + api: implementation as unknown as ExtensionAPI, + handlers, + commands, + tools, + entries, + providers, + get active() { + return active; + }, + }; + return result; +} + +function getHandler(pi: FakePi, name: string): Handler { + const handler = pi.handlers.get(name); + if (!handler) throw new Error(`missing ${name} handler`); + return handler; +} + +function getCommand(pi: FakePi, name: string): FakeCommand { + const command = pi.commands.get(name); + if (!command) throw new Error(`missing ${name} command`); + return command; +} + +const model = { provider: "openai", id: "gpt-5.6-sol", api: "openai-responses" } as unknown as Model; +const ctx = { + model, + mode: "rpc", + sessionManager: { getBranch: () => [] }, + ui: { setStatus() {}, notify() {} }, +} as unknown as ExtensionContext; +const anthropicCtx = { ...ctx, model: getCuaModel("anthropic:claude-fable-5") } as ExtensionContext; + +describe("pi extension activation", () => { + it("reads parsed flags at session_start, installs selectable batch tools, and preserves unrelated tools", async () => { + const pi = makePi({ + "cua-tools": "browser-batch", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + await getHandler(pi, "session_start")({}, ctx); + expect(pi.tools.map((tool) => tool.name)).toEqual(expect.arrayContaining(allSelectableSpecs("pixels").map((tool) => tool.name))); + expect(pi.active).toEqual(["bash", "browser_batch"]); + }); + + it("rejects invalid parsed flags instead of silently activating no tools", () => { + const pi = makePi({ + "cua-tools": "nope", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + expect(() => getHandler(pi, "session_start")({}, ctx)).toThrow('unknown CUA tool selector "nope"'); + }); + + it("registers the CUA Anthropic provider and serializes native computer use", async () => { + const get = vi.spyOn(CuaBrowserRuntime.prototype, "get").mockResolvedValue({ + viewport: { width: 1024, height: 768 }, + } as unknown as Awaited>); + try { + const pi = makePi({ + "cua-tools": "anthropic-computer", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + expect(pi.providers.map((provider) => provider.id)).toContain("anthropic"); + await getHandler(pi, "session_start")({}, anthropicCtx); + expect(pi.active).toContain("computer"); + + const headers: Record = {}; + await getHandler(pi, "before_provider_headers")({ headers }, anthropicCtx); + expect(headers["anthropic-beta"]).toContain("computer-use-2025-11-24"); + + const payload = { tools: [{ name: "computer", input_schema: { type: "object" } }] }; + const transformed = await getHandler(pi, "before_provider_request")({ payload }, anthropicCtx); + expect(transformed).toEqual({ + tools: [ + expect.objectContaining({ + name: "computer", + type: "computer_20251124", + display_width_px: 1024, + display_height_px: 768, + }), + ], + }); + + await getHandler(pi, "session_shutdown")({}, anthropicCtx); + await expect(getHandler(pi, "before_provider_request")({ payload }, anthropicCtx)).rejects.toThrow("outside an active pi session"); + } finally { + get.mockRestore(); + } + }); + + it("applies provider transforms only for the active CUA subset", async () => { + const pi = makePi({ + "cua-tools": "browser_snapshot", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + await getHandler(pi, "session_start")({}, ctx); + const headers: Record = {}; + await getHandler(pi, "before_provider_headers")({ headers }, ctx); + const transformed = await getHandler(pi, "before_provider_request")({ payload: { tools: [] } }, ctx); + expect(transformed).toEqual({ tools: [] }); + + const inactive = makePi({ "cua-coordinates": "pixels", "cua-browser-timeout": "300", "cua-profile-save-changes": false }); + extension(inactive.api); + await getHandler(inactive, "session_start")({}, ctx); + expect(await getHandler(inactive, "before_provider_request")({ payload: { tools: [] } }, ctx)).toBeUndefined(); + }); + + it("does not persist a flag baseline and restores only command-origin selections", async () => { + const pi = makePi({ + "cua-tools": "browser_snapshot", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + await getHandler(pi, "session_start")({}, ctx); + await getHandler(pi, "session_shutdown")({}, ctx); + expect(pi.entries).toEqual([]); + + await getCommand(pi, "cua-tools").handler("computer", ctx); + expect(pi.entries).toEqual([ + { + type: "custom", + customType: "cua-pi-config-v1", + data: expect.objectContaining({ origin: "command", selectors: ["computer"] }), + }, + ]); + + const resumed = makePi({ + "cua-tools": "browser_snapshot", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + const resumedCtx = { ...ctx, sessionManager: { getBranch: () => pi.entries } } as unknown as ExtensionContext; + extension(resumed.api); + await getHandler(resumed, "session_start")({}, resumedCtx); + expect(resumed.active).toContain("computer_click"); + expect(resumed.active).not.toContain("browser_snapshot"); + + const legacy = makePi({ + "cua-tools": "browser_snapshot", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + const legacyCtx = { + ...ctx, + sessionManager: { + getBranch: () => [ + { + type: "custom", + customType: "cua-pi-config-v1", + data: { version: 1, selectors: ["computer"], coordinates: "normalized-1000" }, + }, + ], + }, + } as unknown as ExtensionContext; + extension(legacy.api); + await getHandler(legacy, "session_start")({}, legacyCtx); + expect(legacy.active).toContain("browser_snapshot"); + expect(legacy.active).not.toContain("computer_click"); + }); + + it("removes stale incompatible CUA schemas from the provider payload", async () => { + const pi = makePi({ + "cua-tools": "browser_snapshot", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }); + extension(pi.api); + await getHandler(pi, "session_start")({}, ctx); + const incompatible = { + ...ctx, + model: { provider: "unsupported", id: "not-a-cua-model", api: "openai-completions" }, + } as unknown as ExtensionContext; + const payload = { + tools: [ + { type: "function", function: { name: "browser_snapshot" } }, + { type: "function", function: { name: "bash" } }, + { functionDeclarations: [{ name: "browser_snapshot" }, { name: "write" }] }, + { functionDeclarations: [{ name: "browser_snapshot" }] }, + ], + }; + const transformed = await getHandler(pi, "before_provider_request")({ payload }, incompatible); + expect(transformed).toEqual({ + tools: [{ type: "function", function: { name: "bash" } }, { functionDeclarations: [{ name: "write" }] }], + }); + expect(pi.active).toEqual(["bash"]); + }); + + it("re-registers declarations when a new session changes coordinate mode", async () => { + const flags: Record = { + "cua-tools": "computer", + "cua-coordinates": "pixels", + "cua-browser-timeout": "300", + "cua-profile-save-changes": false, + }; + const pi = makePi(flags); + extension(pi.api); + await getHandler(pi, "session_start")({}, ctx); + expect(pi.tools.find((tool) => tool.name === "computer_click")?.description).not.toContain("[0, 1000]"); + + flags["cua-coordinates"] = "normalized-1000"; + await getHandler(pi, "session_start")({}, ctx); + expect(pi.tools.find((tool) => tool.name === "computer_click")?.description).toContain("[0, 1000]"); + }); +}); diff --git a/packages/pi-extension/test/pi-modes.test.ts b/packages/pi-extension/test/pi-modes.test.ts new file mode 100644 index 0000000..6ebfccf --- /dev/null +++ b/packages/pi-extension/test/pi-modes.test.ts @@ -0,0 +1,230 @@ +import { once } from "node:events"; +import { createServer, type IncomingMessage, type Server } from "node:http"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawn } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; + +interface FakeServer { + url: string; + payloads: unknown[]; + deletedBrowsers: string[]; + close(): Promise; +} + +async function startFakeServer(): Promise { + const payloads: unknown[] = []; + const deletedBrowsers: string[] = []; + const server = createServer(async (request, response) => { + const body = await readBody(request); + const send = (value: unknown) => { + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify(value)); + }; + if (request.method === "POST" && request.url === "/chat/completions") { + payloads.push(JSON.parse(body || "{}")); + response.writeHead(200, { "content-type": "text/event-stream" }); + const isFollowup = + Array.isArray((payloads.at(-1) as { messages?: Array<{ role?: string }> }).messages) && + (payloads.at(-1) as { messages: Array<{ role?: string }> }).messages.some((message) => message.role === "tool"); + if (!isFollowup) { + response.write( + `data: ${JSON.stringify(completion({ role: "assistant", tool_calls: [{ index: 0, id: "call_1", type: "function", function: { name: "playwright_execute", arguments: '{"code":"return 7"}' } }] }, null))}\n\n`, + ); + response.write(`data: ${JSON.stringify(completion({}, "tool_calls"))}\n\n`); + } else { + response.write(`data: ${JSON.stringify(completion({ role: "assistant", content: "browser tool completed" }, null))}\n\n`); + response.write(`data: ${JSON.stringify(completion({}, "stop"))}\n\n`); + } + response.end("data: [DONE]\n\n"); + return; + } + if (request.method === "POST" && request.url === "/browsers") { + return send({ session_id: "browser_test", created_at: "2026-01-01T00:00:00Z", browser_live_view_url: "https://live.test" }); + } + if (request.method === "POST" && request.url === "/browsers/browser_test/playwright/execute") { + return send({ success: true, result: 7, stdout: "fake browser tool" }); + } + if (request.method === "DELETE" && request.url === "/browsers/browser_test") { + deletedBrowsers.push("browser_test"); + return send({}); + } + response.writeHead(404); + response.end(`${request.method} ${request.url}`); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("fake server did not bind a port"); + return { url: `http://127.0.0.1:${address.port}`, payloads, deletedBrowsers, close: () => close(server) }; +} + +function completion(message: unknown, finish_reason: string | null) { + return { + id: "fake", + object: "chat.completion.chunk", + created: 0, + model: "gpt-5.6-sol", + choices: [{ index: 0, delta: message, finish_reason }], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }; +} +function readBody(request: IncomingMessage): Promise { + return new Promise((resolve, reject) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk) => { + body += chunk; + }); + request.on("end", () => resolve(body)); + request.on("error", reject); + }); +} +function close(server: Server): Promise { + return new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); +} + +async function fakeProviderConfig(dir: string, baseUrl: string): Promise { + const agentDir = join(dir, "agent"); + await mkdir(agentDir, { recursive: true }); + await writeFile( + join(agentDir, "models.json"), + JSON.stringify({ + providers: { + openai: { + baseUrl, + apiKey: "test-key", + api: "openai-completions", + authHeader: true, + models: [ + { + id: "gpt-5.6-sol", + api: "openai-completions", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 4096, + }, + ], + }, + }, + }), + ); + return agentDir; +} + +function parseJsonLines(output: string): Array> { + return output + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as Record); +} + +async function runPrint( + args: string[], + env: NodeJS.ProcessEnv, + cwd: string, +): Promise<{ code: number | null; stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + const child = spawn("pi", args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + const timer = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error(`pi print timed out: ${stderr}; output: ${stdout}`)); + }, 15_000); + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("close", (code) => { + clearTimeout(timer); + resolve({ code, stdout, stderr }); + }); + }); +} + +async function runRpc( + args: string[], + input: string, + env: NodeJS.ProcessEnv, +): Promise<{ code: number | null; stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + const child = spawn("pi", args, { env, stdio: ["pipe", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + const timer = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error(`pi RPC timed out: ${stderr}`)); + }, 15_000); + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + if (stdout.includes('"type":"agent_settled"')) child.stdin.end(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("close", (code) => { + clearTimeout(timer); + resolve({ code, stdout, stderr }); + }); + child.stdin.write(input); + }); +} + +describe("pi modes", () => { + it("runs a deterministic CUA browser tool in print and RPC modes", async () => { + const server = await startFakeServer(); + const directory = await mkdtemp(join(tmpdir(), "cua-pi-mode-")); + try { + const agentDir = await fakeProviderConfig(directory, server.url); + const extension = fileURLToPath(new URL("../src/index.ts", import.meta.url)); + const env = { + ...process.env, + OPENAI_API_KEY: "test-key", + PI_CODING_AGENT_DIR: agentDir, + KERNEL_BASE_URL: server.url, + KERNEL_API_KEY: "test-key", + }; + const args = ["--extension", extension, "--provider", "openai", "--model", "gpt-5.6-sol", "--cua-tools", "playwright"]; + + const print = await runPrint([...args, "-p", "run the browser tool"], env, directory); + expect(print.code, `${print.stdout}\n${print.stderr}`).toBe(0); + expect(server.payloads).toHaveLength(2); + + const rpc = await runRpc(["--mode", "rpc", ...args], '{"id":"prompt-1","type":"prompt","message":"run the browser tool"}\n', env); + expect(rpc.code).toBe(0); + const events = parseJsonLines(rpc.stdout); + expect( + events.some((event) => event.type === "tool_execution_start" && event.toolName === "playwright_execute"), + rpc.stdout, + ).toBe(true); + expect( + events.some((event) => event.type === "tool_execution_end" && event.toolName === "playwright_execute" && event.isError === false), + ).toBe(true); + expect(events.some((event) => event.type === "agent_settled")).toBe(true); + expect(server.payloads).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + tools: expect.arrayContaining([expect.objectContaining({ function: expect.objectContaining({ name: "playwright_execute" }) })]), + }), + ]), + ); + expect(server.deletedBrowsers).toEqual(["browser_test", "browser_test"]); + } finally { + await server.close(); + await rm(directory, { recursive: true, force: true }); + } + }, 30_000); +}); diff --git a/packages/pi-extension/test/published-package.test.ts b/packages/pi-extension/test/published-package.test.ts new file mode 100644 index 0000000..cad7a96 --- /dev/null +++ b/packages/pi-extension/test/published-package.test.ts @@ -0,0 +1,17 @@ +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { describe, expect, it } from "vitest"; + +describe("published pi package", () => { + it("ships a discoverable TypeScript extension manifest and runtime dependencies", async () => { + const pkg = JSON.parse(await readFile(resolve(import.meta.dirname, "../package.json"), "utf8")); + expect(pkg.pi.extensions).toEqual(["./src/index.ts"]); + expect(pkg.files).toContain("src"); + expect(pkg.dependencies).toMatchObject({ + "@onkernel/cua-ai": pkg.version, + "@onkernel/cua-agent": pkg.version, + "@onkernel/sdk": expect.any(String), + }); + expect(pkg.peerDependencies.typebox).toBeUndefined(); + }); +}); diff --git a/packages/pi-extension/tsconfig.build.json b/packages/pi-extension/tsconfig.build.json new file mode 100644 index 0000000..adc2ca9 --- /dev/null +++ b/packages/pi-extension/tsconfig.build.json @@ -0,0 +1,27 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist-tsc", + "rootDir": "./src", + "emitDeclarationOnly": true, + "sourceMap": false, + "declarationMap": false + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "node_modules", + "dist", + "**/*.d.ts", + "src/**/*.d.ts" + ], + "references": [ + { + "path": "../ai" + }, + { + "path": "../agent" + } + ] +} diff --git a/packages/pi-extension/tsconfig.json b/packages/pi-extension/tsconfig.json new file mode 100644 index 0000000..d8faaf5 --- /dev/null +++ b/packages/pi-extension/tsconfig.json @@ -0,0 +1,3 @@ +{ + "extends": "./tsconfig.build.json" +} diff --git a/packages/pi-extension/vitest.config.ts b/packages/pi-extension/vitest.config.ts new file mode 100644 index 0000000..039b588 --- /dev/null +++ b/packages/pi-extension/vitest.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + server: { + host: "127.0.0.1", + }, + test: { + globals: true, + environment: "node", + testTimeout: 30000, + }, +}); diff --git a/tsconfig.json b/tsconfig.json index aad9af1..6b096fe 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,6 +4,7 @@ { "path": "./packages/ai" }, { "path": "./packages/agent" }, { "path": "./packages/ptywright" }, - { "path": "./packages/cli" } + { "path": "./packages/cli" }, + { "path": "./packages/pi-extension" } ] }