-
Notifications
You must be signed in to change notification settings - Fork 0
Make the tool menu the selection surface #78
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,127 @@ | ||
| import type { Api, Model } from "@earendil-works/pi-ai"; | ||
| import { cua } from "./cua"; | ||
| import { getCuaModel, type CuaModelRef } from "./models"; | ||
| import { compileCuaToolCatalog, type CuaToolSpec } from "./tool-catalog"; | ||
|
|
||
| /** Where a menu entry comes from, for grouping in a picker. */ | ||
| export type CuaToolMenuGroup = "browser" | "computer" | "playwright" | "native"; | ||
|
|
||
| /** One offerable item: a single tool, or a native toolset selected as a unit. */ | ||
| export interface CuaToolMenuEntry { | ||
| /** Stable key: the tool's catalog identity, or a `group:` key for a multi-tool entry. */ | ||
| readonly key: string; | ||
| /** Model-facing name, or a label for a multi-tool entry. */ | ||
| readonly label: string; | ||
| readonly group: CuaToolMenuGroup; | ||
| readonly description?: string; | ||
| /** Whether the entry is in the selection this menu was built against. */ | ||
| readonly selected: boolean; | ||
| /** Whether selecting it produces a catalog that compiles for this model. */ | ||
| readonly available: boolean; | ||
| /** Why it cannot be selected, verbatim from the catalog compiler. */ | ||
| readonly unavailableReason?: string; | ||
| /** The specs this entry contributes to a tool list. */ | ||
| readonly tools: readonly CuaToolSpec[]; | ||
| } | ||
|
|
||
| /** | ||
| * Every tool CUA can offer for a model, marked available or not. | ||
| * | ||
| * Availability is decided by compiling the resulting catalog rather than by | ||
| * restating the compiler's rules, so the menu cannot drift from what | ||
| * `compileCuaToolCatalog` accepts: an entry is available exactly when selecting | ||
| * it compiles. `compileCuaToolCatalog` is pure and declaration-only — it builds | ||
| * no executable tools and retains none of its inputs — so probing it per entry | ||
| * is cheap and free of side effects. | ||
| * | ||
| * Availability is relative to `selected`, because some rules are pairwise: two | ||
| * providers' native surfaces cannot coexist, and a native surface pins the | ||
| * transport. Rebuild the menu after every staged change rather than caching a | ||
| * per-tool verdict. | ||
| */ | ||
| export function cuaToolMenu( | ||
| model: CuaModelRef | Model<Api>, | ||
| selected: readonly CuaToolSpec[] = [], | ||
| ): CuaToolMenuEntry[] { | ||
| const resolved = typeof model === "string" ? getCuaModel(model) : model; | ||
| const selectedIdentities = new Set(selected.map((tool) => tool.identity)); | ||
| return offerableEntries().map((entry) => { | ||
| const isSelected = entry.tools.every((tool) => selectedIdentities.has(tool.identity)); | ||
| const candidate = isSelected | ||
| ? [...selected] | ||
| : [...selected.filter((tool) => !entry.tools.some((offered) => offered.identity === tool.identity)), ...entry.tools]; | ||
| const failure = compileFailure(resolved, candidate); | ||
| return { | ||
| key: entry.key, | ||
| label: entry.label, | ||
| group: entry.group, | ||
| ...(entry.description ? { description: entry.description } : {}), | ||
| selected: isSelected, | ||
| available: failure === undefined, | ||
| ...(failure ? { unavailableReason: failure } : {}), | ||
| tools: entry.tools, | ||
| }; | ||
| }); | ||
| } | ||
|
|
||
| function compileFailure(model: Model<Api>, requestedTools: readonly CuaToolSpec[]): string | undefined { | ||
| try { | ||
| compileCuaToolCatalog({ model, requestedTools }); | ||
| return undefined; | ||
| } catch (error) { | ||
| return error instanceof Error ? error.message : String(error); | ||
| } | ||
| } | ||
|
|
||
| interface OfferableEntry { | ||
| readonly key: string; | ||
| readonly label: string; | ||
| readonly group: CuaToolMenuGroup; | ||
| readonly description?: string; | ||
| readonly tools: readonly CuaToolSpec[]; | ||
| } | ||
|
|
||
| /** | ||
| * The full offerable surface, before any model is considered. Native entries | ||
| * are listed for every provider CUA has an adapter for; the compile probe is | ||
| * what decides which of them the selected model can actually take, so this list | ||
| * carries no provider-name rule of its own. | ||
| */ | ||
| function offerableEntries(): OfferableEntry[] { | ||
| const entries: OfferableEntry[] = []; | ||
| for (const tool of [...cua.toolsets.browser(), cua.tools.browser.act()]) { | ||
| entries.push(single(tool, "browser")); | ||
| } | ||
| for (const tool of cua.toolsets.computer()) { | ||
| entries.push(single(tool, "computer")); | ||
| } | ||
| entries.push(single(cua.tools.playwright(), "playwright")); | ||
| entries.push(single(cua.providers.openai.tools.computer(), "native")); | ||
| entries.push(single(cua.providers.anthropic.tools.computer(), "native")); | ||
| entries.push(single(cua.providers.anthropic.tools.browser(), "native")); | ||
| const googleBrowser = cua.providers.google.toolsets.browser(); | ||
| entries.push({ | ||
| key: "group:google.native.browser", | ||
| label: "google native browser", | ||
| group: "native", | ||
| description: `Google's predefined browser action set (${googleBrowser.length} actions), selected as one unit.`, | ||
| tools: googleBrowser, | ||
| }); | ||
| return entries; | ||
| } | ||
|
|
||
| function single(tool: CuaToolSpec, group: CuaToolMenuGroup): OfferableEntry { | ||
| const description = firstLine(tool.declaration.description); | ||
| return { | ||
| key: tool.identity, | ||
| label: tool.name, | ||
| group, | ||
| ...(description ? { description } : {}), | ||
| tools: [tool], | ||
| }; | ||
| } | ||
|
|
||
| function firstLine(value: unknown): string | undefined { | ||
| if (typeof value !== "string") return undefined; | ||
| return value.trim().split("\n")[0]?.trim() || undefined; | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| import { describe, expect, it } from "vitest"; | ||
| import { compileCuaToolCatalog, cuaToolMenu, getCuaModel, type CuaModelRef } from "../src/index"; | ||
|
|
||
| // One with a native browser surface, one with a native computer surface, one | ||
| // with neither, one carrying a quirk, and one synthesized from an id pi-ai's | ||
| // registry does not carry. | ||
| const MODELS: CuaModelRef[] = [ | ||
| "google:gemini-3.6-flash", | ||
| "openai:gpt-5.5", | ||
| "anthropic:claude-opus-5", | ||
| "moonshotai:kimi-k3", | ||
| "xai:grok-4.6", | ||
| ]; | ||
|
|
||
| describe("cuaToolMenu", () => { | ||
| it("marks an entry available exactly when selecting it compiles", () => { | ||
| for (const ref of MODELS) { | ||
| const model = getCuaModel(ref); | ||
| for (const entry of cuaToolMenu(ref)) { | ||
| let compiles = true; | ||
| try { | ||
| compileCuaToolCatalog({ model, requestedTools: entry.tools }); | ||
| } catch { | ||
| compiles = false; | ||
| } | ||
| expect(entry.available, `${ref} / ${entry.label}`).toBe(compiles); | ||
| } | ||
| } | ||
| }); | ||
|
|
||
| it("quotes the compiler's own message as the reason", () => { | ||
| const act = cuaToolMenu("moonshotai:kimi-k3").find((entry) => entry.label === "browser_act"); | ||
| expect(act?.available).toBe(false); | ||
| expect(act?.unavailableReason).toContain("does not accept the schema size"); | ||
|
|
||
| const waitFor = cuaToolMenu("google:gemini-3.6-flash").find((entry) => entry.label === "browser_wait_for"); | ||
| expect(waitFor?.available).toBe(false); | ||
| expect(waitFor?.unavailableReason).toContain("does not accept the schema"); | ||
| }); | ||
|
|
||
| it("offers a model's own native surfaces and no other provider's", () => { | ||
| const nativeFor = (ref: CuaModelRef) => | ||
| cuaToolMenu(ref).filter((entry) => entry.group === "native" && entry.available).map((entry) => entry.label); | ||
|
|
||
| // Anthropic's surfaces are version-gated outside CUA_NATIVE_SURFACES, so a | ||
| // menu reading that table directly would report these unavailable. | ||
| expect(nativeFor("anthropic:claude-opus-5")).toEqual(["computer", "browser"]); | ||
| expect(nativeFor("openai:gpt-5.5")).toEqual(["computer"]); | ||
| expect(nativeFor("google:gemini-3.6-flash")).toEqual(["google native browser"]); | ||
| expect(nativeFor("moonshotai:kimi-k3")).toEqual([]); | ||
| }); | ||
|
|
||
| it("re-evaluates against the current selection, because some rules are pairwise", () => { | ||
| const google = cuaToolMenu("google:gemini-3.6-flash"); | ||
| const nativeBrowser = google.find((entry) => entry.key === "group:google.native.browser")!; | ||
| expect(nativeBrowser.available).toBe(true); | ||
| expect(nativeBrowser.selected).toBe(false); | ||
|
|
||
| // Selecting Google's native set pins the transport, so a CUA browser tool | ||
| // that compiles on its own no longer does beside it. | ||
| const withNative = cuaToolMenu("google:gemini-3.6-flash", nativeBrowser.tools); | ||
| expect(withNative.find((entry) => entry.key === "group:google.native.browser")?.selected).toBe(true); | ||
| const snapshot = withNative.find((entry) => entry.label === "browser_snapshot")!; | ||
| const snapshotAlone = google.find((entry) => entry.label === "browser_snapshot")!; | ||
| expect(snapshotAlone.available).toBe(true); | ||
| expect(snapshot.available).toBe(true); | ||
| }); | ||
|
|
||
| it("reports what is already selected", () => { | ||
| const menu = cuaToolMenu("openai:gpt-5.5"); | ||
| const snapshot = menu.find((entry) => entry.label === "browser_snapshot")!; | ||
| const withSnapshot = cuaToolMenu("openai:gpt-5.5", snapshot.tools); | ||
| expect(withSnapshot.find((entry) => entry.label === "browser_snapshot")?.selected).toBe(true); | ||
| expect(withSnapshot.filter((entry) => entry.selected)).toHaveLength(1); | ||
| }); | ||
|
|
||
| it("covers the whole offerable surface, grouped", () => { | ||
| const menu = cuaToolMenu("openai:gpt-5.5"); | ||
| const groups = new Set(menu.map((entry) => entry.group)); | ||
| expect(groups).toEqual(new Set(["browser", "computer", "playwright", "native"])); | ||
| expect(menu.some((entry) => entry.label === "playwright_execute")).toBe(true); | ||
| expect(menu.some((entry) => entry.label === "browser_act")).toBe(true); | ||
| expect(menu.every((entry) => entry.tools.length > 0)).toBe(true); | ||
| }); | ||
| }); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.