diff --git a/.changeset/ai-sandbox-harness.md b/.changeset/ai-sandbox-harness.md new file mode 100644 index 0000000000..10f2443916 --- /dev/null +++ b/.changeset/ai-sandbox-harness.md @@ -0,0 +1,10 @@ +--- +'@tanstack/ai-sandbox': minor +--- + +Add `@tanstack/ai-sandbox/harness` for `@tanstack/ai-harness`: + +- `codingAgents({ sandbox, agents, workspace? })` is a harness plugin. The lead model gets one tool per coding agent (Claude Code, Codex, Grok Build, or any ACP agent). Each agent works in the sandbox, keeps its own session per thread (also after a restart), and starts read-only in the `plan` mode of `permissions()`. `/fresh [agent]` starts a new session. `workspace: 'shared'` (default) runs the agents of a thread one at a time in one sandbox. `'per-agent'` gives each agent its own sandbox. +- `sandboxWorkspaceBackend(sandbox)` runs the coding tools of `@tanstack/ai-harness/plugins/coding` (`workspaceTools()`, `snapshots()`, `formatter()`) in a sandbox. + +`@tanstack/ai-harness` is an optional peer dependency. diff --git a/docs/config.json b/docs/config.json index db49a35cd5..0baf38d22e 100644 --- a/docs/config.json +++ b/docs/config.json @@ -1007,14 +1007,16 @@ "updatedAt": "2026-10-14" }, { - "label": "Extend the coding tools", + "label": "Sandbox and extend the coding tools", "to": "harness/coding-tools-backends", - "addedAt": "2026-10-13" + "addedAt": "2026-10-13", + "updatedAt": "2026-10-13" }, { "label": "Undo a turn", "to": "harness/snapshots", - "addedAt": "2026-10-13" + "addedAt": "2026-10-13", + "updatedAt": "2026-10-13" }, { "label": "Build a coding agent", @@ -1022,6 +1024,11 @@ "addedAt": "2026-10-13", "updatedAt": "2026-10-14" }, + { + "label": "Delegate to coding agents", + "to": "harness/coding-agents", + "addedAt": "2026-10-13" + }, { "label": "Switch between agent profiles", "to": "harness/agents", diff --git a/docs/harness/coding-agent.md b/docs/harness/coding-agent.md index 8a98aa9c00..0effd1f088 100644 --- a/docs/harness/coding-agent.md +++ b/docs/harness/coding-agent.md @@ -184,7 +184,11 @@ export const noDeploys = definePlugin({ A trailing `*` matches every tool that starts with the text. The last matching rule wins. [Ask before risky tool calls](./permissions) explains the rules, the modes, and the saved answers. -The workspace tools run on your machine with your permissions. To run them on another machine, [write your own backend](./coding-tools-backends#write-your-own-backend). +The workspace tools run on your machine with your permissions. For code that you do not trust, [run the tools in a sandbox](./coding-tools-backends#run-the-tools-in-a-sandbox). + +## Hand work to Claude Code or Codex + +Your agent can also give tasks to coding agents you already use. [Delegate to coding agents](./coding-agents) shows how. ## What you have now diff --git a/docs/harness/coding-agents.md b/docs/harness/coding-agents.md new file mode 100644 index 0000000000..994c0c52ca --- /dev/null +++ b/docs/harness/coding-agents.md @@ -0,0 +1,129 @@ +--- +title: Delegate to coding agents +id: harness-coding-agents +order: 11 +description: "Let a harness hand coding work to Claude Code, Codex, Grok Build, or any ACP agent. Each one works in a sandbox and keeps its own session." +keywords: + - tanstack ai + - harness + - claude code + - codex + - grok build + - sandbox + - subagents +--- + +Your lead agent plans the work, but you want Claude Code or Codex to do the edits, because they are good at code and you already use them. `codingAgents` gives the lead model one tool per coding agent. Each agent works in a sandbox and keeps its own session between calls. + +## 1. Add the plugin + + + +react: @tanstack/ai-harness @tanstack/ai-sandbox @tanstack/ai-sandbox-local-process @tanstack/ai-claude-code @tanstack/ai-codex +vue: @tanstack/ai-harness @tanstack/ai-sandbox @tanstack/ai-sandbox-local-process @tanstack/ai-claude-code @tanstack/ai-codex +solid: @tanstack/ai-harness @tanstack/ai-sandbox @tanstack/ai-sandbox-local-process @tanstack/ai-claude-code @tanstack/ai-codex +svelte: @tanstack/ai-harness @tanstack/ai-sandbox @tanstack/ai-sandbox-local-process @tanstack/ai-claude-code @tanstack/ai-codex +preact: @tanstack/ai-harness @tanstack/ai-sandbox @tanstack/ai-sandbox-local-process @tanstack/ai-claude-code @tanstack/ai-codex +angular: @tanstack/ai-harness @tanstack/ai-sandbox @tanstack/ai-sandbox-local-process @tanstack/ai-claude-code @tanstack/ai-codex +octane: @tanstack/ai-harness @tanstack/ai-sandbox @tanstack/ai-sandbox-local-process @tanstack/ai-claude-code @tanstack/ai-codex +vanilla: @tanstack/ai-harness @tanstack/ai-sandbox @tanstack/ai-sandbox-local-process @tanstack/ai-claude-code @tanstack/ai-codex + + + +```ts group=harness-coding-agents +import { defineHarness } from '@tanstack/ai-harness' +import { permissions } from '@tanstack/ai-harness/plugins' +import { claudeCodeText } from '@tanstack/ai-claude-code' +import { codexText } from '@tanstack/ai-codex' +import { openaiText } from '@tanstack/ai-openai' +import { defineSandbox, defineWorkspace, localSource } from '@tanstack/ai-sandbox' +import { codingAgents } from '@tanstack/ai-sandbox/harness' +import { localProcessSandbox } from '@tanstack/ai-sandbox-local-process' + +const repo = '/path/to/your/repo' + +export const lead = defineHarness({ + name: 'acme/lead', + adapter: openaiText('gpt-6.1-sol'), + plugins: () => [ + permissions(), + codingAgents({ + sandbox: defineSandbox({ + id: 'repo', + provider: localProcessSandbox({ dir: repo }), + workspace: defineWorkspace({ source: localSource(repo) }), + }), + agents: { + claude_code: { + adapter: claudeCodeText('claude-opus-4-8', { permissionMode: 'acceptEdits' }), + description: 'Larger changes, refactors, and reviews', + }, + codex: { + adapter: codexText('gpt-5.3-codex', { sandboxMode: 'workspace-write' }), + description: 'Quick fixes and tests', + }, + }, + }), + ], +}) +``` + +`localProcessSandbox({ dir: repo })` works in your repo folder itself, so every thread and every agent shares that folder. Leave out `dir` to give each sandbox its own copy of the repo. + +The lead model now has a `claude_code` tool and a `codex` tool. Each tool takes one `task`: the whole job, in words. The agent sees only that text and the files in its sandbox. + +## 2. Ask for work + +Open a session, and give the lead a task for both agents: + +```ts group=harness-coding-agents +import { createHarnessHost } from '@tanstack/ai-harness' +import { memoryPersistence } from '@tanstack/ai-persistence' + +const host = createHarnessHost({ persistence: memoryPersistence() }) +const session = await host.open(lead, { threadId: 'date-parser' }) + +const turn = await session.prompt( + 'Have claude_code add a test for the date parser, then have codex fix what fails.', +) +console.log(turn.text) +``` + +The lead calls `claude_code` with a task, and gets back its answer. Then it calls `codex`. Each agent works in the same sandbox, one after the other. + +## Sessions + +Each agent keeps its own session per harness thread. The next task for `claude_code` resumes the same Claude Code session, so it still knows the files it read. The session ids live in the plugin state, so they also survive a restart of the host. + +To start over, run `/fresh claude_code`, or `/fresh` for every agent. + +## Workspaces + +`workspace` chooses where the agents work: + +| Value | Where each agent works | At the same time | +| --- | --- | --- | +| `'shared'` (default) | One sandbox per harness thread | One agent at a time | +| `'per-agent'` | A sandbox per agent | Yes | + +Use `'shared'` when the agents build on each other's changes. Use `'per-agent'` when they work on separate copies and you merge the results. + +## Plan mode + +When the `permissions()` plugin is in [`plan` mode](./permissions#pick-a-mode), the agents start read-only: + +- Claude Code gets `permissionMode: 'plan'`. +- Codex gets `sandboxMode: 'read-only'`. +- Any other agent gets its `planModelOptions`, for example `{ permissionMode: 'default' }` for an ACP agent that asks before each edit. +- An agent with no `planModelOptions` does not run in `plan` mode. Its tool call gets an error. + +## Other agents and sandboxes + +- `adapter` takes any coding-agent adapter: `grokBuildText` from `@tanstack/ai-grok-build`, or `acpCompatibleText` from `@tanstack/ai-acp` for any ACP agent. +- `sandbox` takes any sandbox provider. [Sandbox providers](../sandbox/providers) lists them. +- `modelOptions` on an agent is added to every call, for example a fixed `permissionMode`. + +## What you have now + +- A lead model that hands tasks to Claude Code and Codex. +- One sandbox per thread, and one saved session per agent. diff --git a/docs/harness/coding-tools-backends.md b/docs/harness/coding-tools-backends.md index da9e702484..86718fdda2 100644 --- a/docs/harness/coding-tools-backends.md +++ b/docs/harness/coding-tools-backends.md @@ -1,18 +1,76 @@ --- -title: Extend the coding tools +title: Sandbox and extend the coding tools id: harness-coding-tools-backends order: 7 -description: "Run the coding tools on your own backend, format each written file, and keep tool output small." +description: "Run the coding tools in a sandbox or on your own backend, format each written file, and keep tool output small." keywords: - tanstack ai - harness - coding agent - workspace tools + - sandbox - formatter - workspace hooks --- -Your agent must work on files that are not on this machine. Or you want code to run after each write, like a formatter. The [coding tools](./coding-tools) send every read, write, and command through a backend, and they run hooks after each read and write. This page shows how to change both. +Your agent must work on files that are not on this machine, or on code that you do not trust. Or you want code to run after each write, like a formatter. The [coding tools](./coding-tools) send every read, write, and command through a backend, and they run hooks after each read and write. This page shows how to change both. + +## Run the tools in a sandbox + +By default, the tools run on your machine, with the rights of your user. For code that you do not trust, run them in a sandbox. `sandboxWorkspaceBackend()` sends every read, write, and command to the sandbox. + +1. Install the sandbox packages: + + + +react: @tanstack/ai-sandbox @tanstack/ai-sandbox-docker +vue: @tanstack/ai-sandbox @tanstack/ai-sandbox-docker +solid: @tanstack/ai-sandbox @tanstack/ai-sandbox-docker +svelte: @tanstack/ai-sandbox @tanstack/ai-sandbox-docker +preact: @tanstack/ai-sandbox @tanstack/ai-sandbox-docker +angular: @tanstack/ai-sandbox @tanstack/ai-sandbox-docker +octane: @tanstack/ai-sandbox @tanstack/ai-sandbox-docker +vanilla: @tanstack/ai-sandbox @tanstack/ai-sandbox-docker + + + +2. Start a sandbox, and give its backend to `workspaceTools()`: + +```ts group=harness-coding-tools-sandbox +import { defineHarness } from '@tanstack/ai-harness' +import { permissions } from '@tanstack/ai-harness/plugins' +import { workspaceTools } from '@tanstack/ai-harness/plugins/coding' +import { openaiText } from '@tanstack/ai-openai' +import { sandboxWorkspaceBackend } from '@tanstack/ai-sandbox/harness' +import { dockerSandbox } from '@tanstack/ai-sandbox-docker' + +const sandbox = await dockerSandbox({ image: 'node:22' }).create({}) +// Without `dir`, the clone goes to the workspace root, /workspace. +await sandbox.git.clone({ url: 'https://github.com/acme/web-app.git' }) + +const root = '/workspace' + +export const sandboxed = defineHarness({ + name: 'acme/sandboxed-coder', + adapter: openaiText('gpt-6.1-sol'), + plugins: () => [ + permissions({ root }), + workspaceTools({ root, backend: sandboxWorkspaceBackend(sandbox) }), + ], +}) +``` + +3. When you are done, stop the sandbox with `await sandbox.destroy()`. + +Every provider works. Get the handle from `create()`, or from `resume()` for a sandbox that runs already. See [Providers](../sandbox/providers). + +In a sandbox, some things are different: + +- Paths are POSIX paths in the sandbox, like `/workspace/src/a.ts`, also on a Windows host. +- The tools do not check where links lead. The sandbox backend has no `realpath`. +- `background: true` needs a sandbox with `capabilities.backgroundProcesses`. +- A sandbox with `capabilities.killableProcesses` off cannot stop a command after the time limit. The call returns, and the command continues in the sandbox. +- `list_files` and `grep` use `rg` in the sandbox. The `@vscode/ripgrep` binary is a host file, so the sandbox does not use it. ## Write your own backend @@ -152,7 +210,7 @@ export const withToml = formatter({ - `when(project)` limits a formatter to some projects. Without it, the formatter is used in every project. - `builtins: false` turns the built-in formatters off. - `timeoutMs` limits each run. The default is 20 seconds. -- On your own backend, give `formatter()` the same `backend` as `workspaceTools()`. +- In a sandbox or on your own backend, give `formatter()` the same `backend` as `workspaceTools()`. A formatter that fails, stops at the time limit, or is not installed does not fail the tool call. The file stays as the tool wrote it, and the plugin sends `FormatFailed`: @@ -208,6 +266,7 @@ export const bounded = defineHarness({ ## What you have now +- The coding tools in a sandbox, with one option. - A backend of your own for another place. - Code that runs after each read and each write. - Formatted files after each write, and tool results that fit the context. diff --git a/docs/harness/coding-tools.md b/docs/harness/coding-tools.md index 84d5a711e9..ba3646f952 100644 --- a/docs/harness/coding-tools.md +++ b/docs/harness/coding-tools.md @@ -112,7 +112,7 @@ terminal.close() await host.close() ``` -`@tanstack/ai-harness/plugins/coding` runs only in Node. The tools use the file system and the shell of the machine. +`@tanstack/ai-harness/plugins/coding` runs only in Node. The tools use the file system and the shell of the machine. For code that you do not trust, [run the tools in a sandbox](./coding-tools-backends#run-the-tools-in-a-sandbox). ## 3. Run it @@ -301,4 +301,4 @@ await session.configure({ cwd: 'web-app' }) - A question before each edit, command, and web fetch. - A working folder for each thread, and a question before a path outside `root`. -Next: run the same tools on your own backend, or format each file after a write. See [Extend the coding tools](./coding-tools-backends). +Next: run the same tools in a sandbox, or format each file after a write. See [Sandbox and extend the coding tools](./coding-tools-backends). diff --git a/docs/harness/permissions.md b/docs/harness/permissions.md index fb57124d58..251232eaf7 100644 --- a/docs/harness/permissions.md +++ b/docs/harness/permissions.md @@ -294,6 +294,7 @@ The same check runs in every agent run of the session: - A question from a child waits in the same session. Answer it the same way. - In `plan` mode, a child cannot change a file. Its call gets the tool error `This tool is not allowed in plan mode.` - The `permissions` of an [agent profile](./agents) count only while that profile is the primary agent. They do not apply when the profile runs as a subagent. +- [Coding agents](./coding-agents#plan-mode) such as Claude Code start read-only in `plan` mode. ## Rules for your own tool diff --git a/docs/harness/snapshots.md b/docs/harness/snapshots.md index 2c5fc79dd4..001f0b6883 100644 --- a/docs/harness/snapshots.md +++ b/docs/harness/snapshots.md @@ -111,6 +111,26 @@ await session.unrevert() In your own UI, call `client.revert(messageId)` and `client.unrevert()`. The client needs `'undo'` in `expose.commands`. More about reverts: [Go back to an earlier message](./sessions#go-back-to-an-earlier-message). +## In a sandbox + +When the [coding tools run in a sandbox](./coding-tools-backends#run-the-tools-in-a-sandbox), give `snapshots()` the same `backend`: + +```ts group=harness-snapshots-sandbox +import { snapshots } from '@tanstack/ai-harness/plugins/coding' +import { sandboxWorkspaceBackend } from '@tanstack/ai-sandbox/harness' +import { dockerSandbox } from '@tanstack/ai-sandbox-docker' + +const sandbox = await dockerSandbox({ image: 'node:22' }).create({}) + +export const sandboxHistory = snapshots({ + root: '/workspace', + dataDir: '/tmp/snapshots', + backend: sandboxWorkspaceBackend(sandbox), +}) +``` + +Git then runs in the sandbox, and `dataDir` is a path in the sandbox. The sandbox image needs `git`. + ## Known limits - One level of undo. A second `/undo` answers `Nothing to undo.` After a `/redo`, `/undo` works again. diff --git a/packages/ai-sandbox/package.json b/packages/ai-sandbox/package.json index 2e113c4ab6..0dea764a2c 100644 --- a/packages/ai-sandbox/package.json +++ b/packages/ai-sandbox/package.json @@ -30,6 +30,10 @@ "types": "./dist/esm/index.d.ts", "import": "./dist/esm/index.js" }, + "./harness": { + "types": "./dist/esm/harness.d.ts", + "import": "./dist/esm/harness.js" + }, "./ngrok": { "types": "./dist/esm/ngrok.d.ts", "import": "./dist/esm/ngrok.js" @@ -84,10 +88,14 @@ "peerDependencies": { "@ngrok/ngrok": "^1.0.0", "@tanstack/ai": "workspace:^", + "@tanstack/ai-harness": "workspace:^", "@tanstack/ai-persistence": "workspace:^", "vitest": "^4.1.10" }, "peerDependenciesMeta": { + "@tanstack/ai-harness": { + "optional": true + }, "@ngrok/ngrok": { "optional": true }, @@ -101,6 +109,7 @@ "devDependencies": { "@ngrok/ngrok": "^1.7.0", "@tanstack/ai": "workspace:*", + "@tanstack/ai-harness": "workspace:*", "@tanstack/ai-persistence": "workspace:*", "@vitest/coverage-v8": "4.1.10", "vitest": "^4.1.10" diff --git a/packages/ai-sandbox/src/harness.ts b/packages/ai-sandbox/src/harness.ts new file mode 100644 index 0000000000..b1b6a14265 --- /dev/null +++ b/packages/ai-sandbox/src/harness.ts @@ -0,0 +1,218 @@ +import { EventType, defineAgent } from '@tanstack/ai' +import { defineCommand, definePlugin } from '@tanstack/ai-harness' +import { withSandbox } from './middleware' +import type { + AnyChatMiddleware, + AnyTextAdapter, + JSONSchema, + StreamChunk, +} from '@tanstack/ai' +import type { SandboxMiddlewareOptions } from './middleware' +import type { SandboxDefinition } from './sandbox' + +export { sandboxWorkspaceBackend } from './workspace-backend' + +export interface CodingAgentConfig { + /** A coding-agent adapter: `claudeCodeText`, `codexText`, `grokBuildText`, `acpCompatibleText`, and more. */ + adapter: AnyTextAdapter + /** When the lead model should pick this agent. */ + description: string + /** Options for every call, for example Claude Code's `permissionMode`. */ + modelOptions?: Record + /** + * Options added when the harness `mode` setting is `plan`. Default: Claude + * Code `{ permissionMode: 'plan' }` and Codex `{ sandboxMode: 'read-only' }`. + */ + planModelOptions?: Record +} + +export interface CodingAgentsOptions { + /** Where the agents work. One sandbox per harness thread (or per agent). */ + sandbox: SandboxDefinition + /** The agents, keyed by the tool name the lead model sees. */ + agents: Record + /** + * `shared` (default): every agent of a thread works in one sandbox, one + * agent at a time. `per-agent`: each agent gets its own sandbox. + */ + workspace?: 'shared' | 'per-agent' + /** Passed to `withSandbox`. */ + sandboxOptions?: SandboxMiddlewareOptions +} + +const PLAN_DEFAULTS: Record> = { + 'claude-code': { permissionMode: 'plan' }, + codex: { sandboxMode: 'read-only' }, +} + +const TASK_SCHEMA: JSONSchema = { + type: 'object', + properties: { + task: { + type: 'string', + description: + 'The whole task: what to change, where, and how to check it. The agent sees only this text and the workspace.', + }, + }, + required: ['task'], +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null +} + +/** + * Run `open` while holding the lock for `key`. The lock is taken when the + * stream is first read, so a stream nobody reads never holds it. + */ +async function* serialized( + tails: Map>, + key: string, + open: () => AsyncIterable, +): AsyncGenerator { + const previous = tails.get(key) ?? Promise.resolve() + let release = () => {} + const current = new Promise((resolve) => (release = resolve)) + const tail = previous.then(() => current) + tails.set(key, tail) + await previous + try { + yield* open() + } finally { + release() + if (tails.get(key) === tail) tails.delete(key) + } +} + +/** + * A harness plugin that lets the lead model hand coding work to Claude Code, + * Codex, Grok Build, or any ACP agent: + * + * - each agent runs in `sandbox`, so nobody wires `withSandbox` by hand; + * - each agent keeps its own coding session per thread, also after a restart + * (`/fresh [agent]` starts over); + * - the harness `plan` mode starts the agents read-only. + * + * @example + * ```ts + * codingAgents({ + * sandbox: defineSandbox({ id: 'code', provider: localProcessSandbox() }), + * agents: { + * claude_code: { adapter: claudeCodeText('claude-opus-4-8'), description: 'Large refactors' }, + * codex: { adapter: codexText('gpt-5.5'), description: 'Quick fixes and tests' }, + * }, + * }) + * ``` + */ +export function codingAgents(options: CodingAgentsOptions) { + const shared = (options.workspace ?? 'shared') === 'shared' + const sandboxOptions = options.sandboxOptions ?? {} + // ponytail: an in-process lock per sandbox. Several hosts on one sandbox need a shared lock store. + const tails = new Map>() + const names = Object.keys(options.agents) + + return definePlugin({ + name: 'tanstack/coding-agents', + setup: (ctx) => { + const state = ctx.state<{ sessions: Record }>({ + sessions: {}, + }) + // Saves the session id the adapter reports (`claude-code.session-id`, ...). + const capture = (agent: string): AnyChatMiddleware => ({ + name: 'tanstack/coding-agents:session', + onChunk: async (_run, chunk) => { + if (chunk.type !== EventType.CUSTOM) return + if (!chunk.name.endsWith('.session-id')) return + const value: unknown = chunk.value + if (!isRecord(value) || typeof value.sessionId !== 'string') return + const sessionId = value.sessionId + await state.update((current) => ({ + sessions: { ...current.sessions, [agent]: sessionId }, + })) + }, + }) + + const subagents = Object.entries(options.agents).map(([name, config]) => + defineAgent({ + name, + description: config.description, + inputSchema: TASK_SCHEMA, + run: async (run) => { + const input: unknown = run.input + const task = + isRecord(input) && typeof input.task === 'string' + ? input.task + : '' + const plan = + ctx.config.get('mode') === 'plan' + ? (config.planModelOptions ?? + PLAN_DEFAULTS[config.adapter.name]) + : {} + // Fail closed: an agent with no read-only options must not edit. + if (plan === undefined) { + throw new Error( + `${name} cannot run in plan mode: it has no planModelOptions.`, + ) + } + // The harness thread, not `run.threadId`: a child's own thread is + // `:` by default. + const thread = ctx.session.threadId + const threadId = shared ? thread : `${thread}:${name}` + // Inside the lock, so a queued call resumes the session of the + // call before it. + async function* open(): AsyncGenerator { + const sessionId = (await state.get()).sessions[name] + const modelOptions = { + ...config.modelOptions, + ...plan, + ...(sessionId ? { sessionId } : {}), + } + yield* run.chat({ + adapter: config.adapter, + messages: [{ role: 'user', content: task }], + threadId, + ...(Object.keys(modelOptions).length > 0 + ? { modelOptions } + : {}), + middleware: [ + withSandbox(options.sandbox, sandboxOptions), + capture(name), + ], + }) as AsyncIterable + } + // One call at a time per sandbox, also for the same agent in + // `per-agent` mode. + return serialized(tails, threadId, open) + }, + }), + ) + + return { + subagents, + commands: { + fresh: defineCommand({ + description: `Start new coding-agent sessions (all, or one of: ${names.join(', ')})`, + run: async (input: unknown) => { + const name = typeof input === 'string' ? input.trim() : '' + if (name && !names.includes(name)) { + return `Unknown coding agent "${name}". Agents: ${names.join(', ')}.` + } + await state.update((current) => ({ + sessions: name + ? Object.fromEntries( + Object.entries(current.sessions).filter( + ([agent]) => agent !== name, + ), + ) + : {}, + })) + return name + ? `${name} starts a new session next time.` + : 'Every coding agent starts a new session next time.' + }, + }), + }, + } + }, + }) +} diff --git a/packages/ai-sandbox/src/workspace-backend.ts b/packages/ai-sandbox/src/workspace-backend.ts new file mode 100644 index 0000000000..b52ef87841 --- /dev/null +++ b/packages/ai-sandbox/src/workspace-backend.ts @@ -0,0 +1,154 @@ +import type { WorkspaceBackend } from '@tanstack/ai-harness/plugins/coding' +import type { ExecResult, SandboxHandle } from './contracts' + +/** + * A `WorkspaceBackend` over a live sandbox. With it, `workspaceTools()` + * reads and writes files and runs commands in the sandbox, not on this + * machine. You start the sandbox before, and you stop it after. + * + * - Paths are POSIX paths in the sandbox, like `/workspace/src/a.ts`. + * - `exec` gives exit code 124 when `timeoutMs` stops a command, and 1 when + * `signal` stops it. A sandbox with `capabilities.killableProcesses` off + * cannot stop the command, so it continues in the sandbox. + * - `spawn` is there only when the sandbox has + * `capabilities.backgroundProcesses`. + * - `stat` gives `mtimeMs: 0`. The sandbox file system does not give the + * time of the last change. + * - There is no `realpath`, so the tools do not check where links go. + * + * @param handle - The sandbox, from `provider.create()` or `provider.resume()`. + * + * @example + * ```ts + * const handle = await localProcessSandbox().create({}) + * workspaceTools({ + * root: '/workspace', + * backend: sandboxWorkspaceBackend(handle), + * }) + * ``` + */ +export function sandboxWorkspaceBackend(handle: SandboxHandle) { + const { fs } = handle + + // The read of the sandbox follows links, so this follows links too. A + // path that `readBytes` cannot read is a folder. + // ponytail: reads the whole file for its size. Only links, and a sandbox + // without `lstat`, come here. + const follow: WorkspaceBackend['stat'] = async (path) => { + if (!(await fs.exists(path))) return undefined + const bytes = await fs.readBytes(path).catch(() => undefined) + if (!bytes) return { type: 'dir', size: 0, mtimeMs: 0 } + return { type: 'file', size: bytes.byteLength, mtimeMs: 0 } + } + + const stat: WorkspaceBackend['stat'] = async (path) => { + if (!fs.lstat) return follow(path) + const info = await fs.lstat(path) + if (!info) return undefined + switch (info.type) { + case 'file': + return { type: 'file', size: info.size, mtimeMs: 0 } + case 'dir': + return { type: 'dir', size: 0, mtimeMs: 0 } + // Like `fs.stat` on the host, a fifo or a socket is an empty file. + case 'other': + return { type: 'file', size: 0, mtimeMs: 0 } + case 'symlink': + return follow(path) + } + } + + const spawn: WorkspaceBackend['spawn'] = (command, options) => { + // ponytail: keeps all output in memory, like `hostBackend`. + let output = '' + const collect = async (stream: AsyncIterable) => { + for await (const chunk of stream) output += chunk + } + const started = handle.process.spawn(command, options) + // Wait for the end of the output too, so `output()` is complete. + const exited = started + .then(async (job) => { + const [exitCode] = await Promise.all([ + job.wait(), + collect(job.stdout), + collect(job.stderr), + ]) + return { exitCode } + }) + .catch((error: unknown) => { + output += String(error) + return { exitCode: 1 } + }) + return { + wait: () => exited, + // Before the process starts, this kills it when it starts. + kill: () => { + started.then((job) => job.kill()).catch(() => undefined) + }, + output: () => output, + } + } + + return { + shell: 'sh', + readFile: (path) => fs.readBytes(path), + writeFile: (path, data) => fs.write(path, data), + // The sandbox `remove` also removes folders, and it does not fail for a + // missing path. This backend removes one file only. + remove: async (path) => { + const isFile = (await stat(path))?.type === 'file' + if (!isFile) throw new Error(`Cannot remove ${path}: no such file.`) + await fs.remove(path) + }, + stat, + readdir: async (path) => { + const entries = await fs.list(path) + const { lstat } = fs + // ponytail: one `lstat` for each entry, so a big folder is slow in a + // remote sandbox. Without `lstat`, a link has the type of its target. + return Promise.all( + entries.map(async (entry) => { + const isLink = + lstat !== undefined && (await lstat(entry.path))?.type === 'symlink' + const type = isLink ? ('link' as const) : entry.type + return { name: entry.name, type } + }), + ) + }, + exec: (command, options = {}) => { + const { cwd, env, timeoutMs, signal } = options + const controller = new AbortController() + // Stop with an exit code. The first reason wins. The result does not + // wait for the sandbox, because some sandboxes cannot stop a command. + let stop = (_exitCode: number) => {} + const stopped = new Promise((resolve) => { + stop = (exitCode) => { + resolve({ exitCode, stdout: '', stderr: '' }) + controller.abort() + } + }) + // A timeout of 0 means no timeout. + const timer = timeoutMs + ? setTimeout(() => stop(124), timeoutMs) + : undefined + const onAbort = () => stop(1) + signal?.addEventListener('abort', onAbort) + if (signal?.aborted) onAbort() + // A command that does not start gives exit code 1, like `hostBackend`. + const run = handle.process + .exec(command, { cwd, env, signal: controller.signal }) + .catch( + (error: unknown): ExecResult => ({ + exitCode: 1, + stdout: '', + stderr: String(error), + }), + ) + return Promise.race([stopped, run]).finally(() => { + clearTimeout(timer) + signal?.removeEventListener('abort', onAbort) + }) + }, + ...(handle.capabilities.backgroundProcesses ? { spawn } : {}), + } satisfies WorkspaceBackend +} diff --git a/packages/ai-sandbox/tests/coding-agents.test.ts b/packages/ai-sandbox/tests/coding-agents.test.ts new file mode 100644 index 0000000000..da467217fb --- /dev/null +++ b/packages/ai-sandbox/tests/coding-agents.test.ts @@ -0,0 +1,383 @@ +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { EventType } from '@tanstack/ai' +import { memoryPersistence } from '@tanstack/ai-persistence' +import { createHarnessHost, defineHarness } from '@tanstack/ai-harness' +import { permissions } from '@tanstack/ai-harness/plugins' +import { codingAgents } from '../src/harness' +import { defineSandbox } from '../src/sandbox' +import { InMemorySandboxInstanceStore } from '../src/instance-store' +import { defineWorkspace } from '../src/workspace' +import { makeFakeProvider } from './fakes' +import type { AnyTextAdapter, StreamChunk } from '@tanstack/ai' + +const now = () => Date.now() +// A small local project the sandboxes start from. +const project = mkdtempSync(join(tmpdir(), 'coding-agents-')) +writeFileSync(join(project, 'README.md'), '# demo\n') +const TYPES = { + providerOptions: {} as Record, + inputModalities: ['text'] as readonly ['text'], + messageMetadataByModality: { + text: undefined as unknown, + image: undefined as unknown, + audio: undefined as unknown, + video: undefined as unknown, + document: undefined as unknown, + }, + toolCapabilities: [] as ReadonlyArray, + toolCallMetadata: undefined as unknown, + systemPromptMetadata: undefined as never, +} + +function textChunks(content: string): Array { + return [ + { + type: EventType.TEXT_MESSAGE_START, + messageId: `m-${content}`, + role: 'assistant', + timestamp: now(), + }, + { + type: EventType.TEXT_MESSAGE_CONTENT, + messageId: `m-${content}`, + delta: content, + timestamp: now(), + }, + { + type: EventType.TEXT_MESSAGE_END, + messageId: `m-${content}`, + timestamp: now(), + }, + { + type: EventType.RUN_FINISHED, + runId: 'r', + threadId: 't', + timestamp: now(), + metadata: { tanstack: { finishReason: 'stop' } }, + }, + ] +} + +/** A lead model that calls the given tools in one step, then answers. */ +function lead(steps: Array>) { + let call = 0 + const adapter: AnyTextAdapter = { + kind: 'text', + name: 'lead', + model: 'lead', + '~types': TYPES, + structuredOutput: async () => ({ data: {}, rawText: '{}' }), + chatStream: () => { + const step = steps[call] + call += 1 + return (async function* (): AsyncGenerator { + yield { + type: EventType.RUN_STARTED, + runId: 'r', + threadId: 't', + timestamp: now(), + } + if (!step) { + yield* textChunks('lead done') + return + } + for (const [index, [tool, task]] of step.entries()) { + const id = `call-${call}-${index}` + yield { + type: EventType.TOOL_CALL_START, + toolCallId: id, + toolCallName: tool, + timestamp: now(), + } + yield { + type: EventType.TOOL_CALL_ARGS, + toolCallId: id, + delta: JSON.stringify({ task }), + timestamp: now(), + } + yield { + type: EventType.TOOL_CALL_END, + toolCallId: id, + timestamp: now(), + } + } + yield { + type: EventType.RUN_FINISHED, + runId: 'r', + threadId: 't', + timestamp: now(), + metadata: { tanstack: { finishReason: 'tool_calls' } }, + } + })() + }, + } + return adapter +} + +/** + * A stand-in coding agent. It records each call, reports a new session id the + * way Claude Code does (`.session-id`), and answers with the task. + */ +function codingAgent(name: string, options: { wait?: Promise } = {}) { + const calls: Array = [] + const log: Array = [] + const adapter: AnyTextAdapter = { + kind: 'text', + name, + model: `${name}-model`, + '~types': TYPES, + structuredOutput: async () => ({ data: {}, rawText: '{}' }), + chatStream: (chatOptions: any) => { + calls.push(chatOptions) + const index = calls.length + return (async function* (): AsyncGenerator { + log.push(`${name} start ${index}`) + yield { + type: EventType.RUN_STARTED, + runId: 'c', + threadId: 't', + timestamp: now(), + } + yield { + type: EventType.CUSTOM, + name: `${name}.session-id`, + value: { sessionId: `${name}-session-${index}` }, + timestamp: now(), + } + if (options.wait) await options.wait + const task = String(chatOptions.messages.at(-1)?.content ?? '') + yield* textChunks(`${name} did: ${task}`) + log.push(`${name} end ${index}`) + })() + }, + } + return { adapter, calls, log } +} + +async function open(input: { + leadSteps: Array> + agents: Record< + string, + { adapter: AnyTextAdapter; planModelOptions?: Record } + > + workspace?: 'shared' | 'per-agent' + persistence?: ReturnType +}) { + const provider = makeFakeProvider() + const persistence = input.persistence ?? memoryPersistence() + const host = createHarnessHost({ persistence }) + const harness = defineHarness({ + name: 'test/coding-lead', + adapter: lead(input.leadSteps), + plugins: () => [ + permissions(), + codingAgents({ + sandbox: defineSandbox({ + id: 'code', + provider, + workspace: defineWorkspace({ + source: { type: 'local', path: project }, + }), + }), + agents: Object.fromEntries( + Object.entries(input.agents).map(([name, agent]) => [ + name, + { description: `The ${name} agent`, ...agent }, + ]), + ), + ...(input.workspace ? { workspace: input.workspace } : {}), + // A store per test, so no test resumes a sandbox of another test. + sandboxOptions: { instances: new InMemorySandboxInstanceStore() }, + }), + ], + }) + const session = await host.open(harness, { threadId: 't' }) + return { host, session, provider, persistence, harness } +} + +describe('codingAgents', () => { + it('runs a delegate in the sandbox and resumes its session on the next call', async () => { + const claude = codingAgent('claude-code') + const { host, session, provider } = await open({ + leadSteps: [ + [['claude_code', 'add a test']], + [], + [['claude_code', 'fix the test']], + ], + agents: { claude_code: { adapter: claude.adapter } }, + }) + await session.prompt('first') + await session.prompt('second') + + expect(claude.calls).toHaveLength(2) + expect(claude.calls[0].messages).toEqual([ + { role: 'user', content: 'add a test' }, + ]) + expect(claude.calls[0].modelOptions?.sessionId).toBeUndefined() + // The second call resumes the session the first call reported. + expect(claude.calls[1].modelOptions).toEqual({ + sessionId: 'claude-code-session-1', + }) + expect(claude.calls[0].threadId).toBe('t') + // One sandbox for the thread, created once. + expect(provider.calls.create).toBe(1) + await host.close() + }) + + it('lists the delegates as session agents and forgets sessions with /fresh', async () => { + const claude = codingAgent('claude-code') + const codex = codingAgent('codex') + const { host, session } = await open({ + leadSteps: [ + [['claude_code', 'one']], + [], + [['codex', 'two']], + [], + [['claude_code', 'three']], + ], + agents: { + claude_code: { adapter: claude.adapter }, + codex: { adapter: codex.adapter }, + }, + }) + expect(session.registry.list().map((agent) => agent.name)).toEqual([ + 'claude_code', + 'codex', + ]) + await session.prompt('a') + await session.prompt('b') + expect(await session.command('fresh', 'nope')).toBe( + 'Unknown coding agent "nope". Agents: claude_code, codex.', + ) + expect(await session.command('fresh', 'claude_code')).toBe( + 'claude_code starts a new session next time.', + ) + await session.prompt('c') + expect(claude.calls[1].modelOptions?.sessionId).toBeUndefined() + expect(await session.command('fresh')).toBe( + 'Every coding agent starts a new session next time.', + ) + await host.close() + }) + + it('keeps sessions across a host restart', async () => { + const persistence = memoryPersistence() + const first = codingAgent('claude-code') + const before = await open({ + leadSteps: [[['claude_code', 'start']]], + agents: { claude_code: { adapter: first.adapter } }, + persistence, + }) + await before.session.prompt('go') + await before.host.close() + + const second = codingAgent('claude-code') + const after = await open({ + leadSteps: [[['claude_code', 'continue']]], + agents: { claude_code: { adapter: second.adapter } }, + persistence, + }) + await after.session.prompt('go on') + expect(second.calls[0].modelOptions).toEqual({ + sessionId: 'claude-code-session-1', + }) + await after.host.close() + }) + + it('runs shared-workspace delegates one at a time, and per-agent ones side by side', async () => { + let release = () => {} + const gate = new Promise((resolve) => (release = resolve)) + const claude = codingAgent('claude-code', { wait: gate }) + const codex = codingAgent('codex') + const shared = await open({ + leadSteps: [], + agents: { + claude_code: { adapter: claude.adapter }, + codex: { adapter: codex.adapter }, + }, + }) + // Two background runs at once, like two /agent commands. + const first = shared.session.agent('claude_code')?.start({ task: 'edit a' }) + const second = shared.session.agent('codex')?.start({ task: 'edit b' }) + await new Promise((resolve) => setTimeout(resolve, 30)) + // Codex waits for the lock while Claude Code holds the shared workspace. + expect(claude.calls).toHaveLength(1) + expect(codex.calls).toHaveLength(0) + release() + await first + await second + expect(codex.calls).toHaveLength(1) + expect(codex.calls[0].threadId).toBe('t') + expect(shared.provider.calls.create).toBe(1) + await shared.host.close() + + let releaseSlow = () => {} + const slowGate = new Promise((resolve) => (releaseSlow = resolve)) + const slow = codingAgent('claude-code', { wait: slowGate }) + const fast = codingAgent('codex') + const apart = await open({ + leadSteps: [], + agents: { + claude_code: { adapter: slow.adapter }, + codex: { adapter: fast.adapter }, + }, + workspace: 'per-agent', + }) + const slowRun = apart.session + .agent('claude_code') + ?.start({ task: 'edit a' }) + const fastRun = apart.session.agent('codex')?.start({ task: 'edit b' }) + await fastRun + // Codex finished while Claude Code still works in its own sandbox. + expect(fast.calls[0].threadId).toBe('t:codex') + expect(slow.log).toEqual(['claude-code start 1']) + releaseSlow() + await slowRun + expect(slow.calls[0].threadId).toBe('t:claude_code') + expect(apart.provider.calls.create).toBe(2) + await apart.host.close() + }) + + it('starts delegates read-only in plan mode, and refuses one without plan options', async () => { + const claude = codingAgent('claude-code') + const codex = codingAgent('codex') + const other = codingAgent('grok-build') + const custom = codingAgent('acp') + const { host, session } = await open({ + // Two steps: tool calls in one step run at the same time, and the + // harness runs at most 3 subagents at once. + leadSteps: [ + [ + ['claude_code', 'look'], + ['codex', 'look'], + ['grok', 'look'], + ], + [['custom', 'look']], + ], + agents: { + claude_code: { adapter: claude.adapter }, + codex: { adapter: codex.adapter }, + grok: { adapter: other.adapter }, + custom: { + adapter: custom.adapter, + planModelOptions: { readOnly: true }, + }, + }, + }) + await session.setConfig({ key: 'mode', value: 'plan' }) + await session.prompt('plan it') + expect(claude.calls[0].modelOptions).toMatchObject({ + permissionMode: 'plan', + }) + expect(codex.calls[0].modelOptions).toMatchObject({ + sandboxMode: 'read-only', + }) + // No plan options for grok-build: it does not run in plan mode. + expect(other.calls).toHaveLength(0) + expect(custom.calls[0].modelOptions).toMatchObject({ readOnly: true }) + await host.close() + }) +}) diff --git a/packages/ai-sandbox/tests/workspace-backend.test.ts b/packages/ai-sandbox/tests/workspace-backend.test.ts new file mode 100644 index 0000000000..78958d92dd --- /dev/null +++ b/packages/ai-sandbox/tests/workspace-backend.test.ts @@ -0,0 +1,456 @@ +import { describe, expect, it } from 'vitest' +import { EventType } from '@tanstack/ai' +import { memoryPersistence } from '@tanstack/ai-persistence' +import { createHarnessHost, defineHarness } from '@tanstack/ai-harness' +import { workspaceTools } from '@tanstack/ai-harness/plugins/coding' +import { sandboxWorkspaceBackend } from '../src/harness' +import { FULL_CAPS, makeFakeHandle } from './fakes' +import type { AnyTextAdapter, StreamChunk, TextOptions } from '@tanstack/ai' +import type { + ExecResult, + SandboxCapabilities, + SandboxFs, + SandboxHandle, +} from '../src/contracts' + +const encoder = new TextEncoder() +const decoder = new TextDecoder() + +interface FsSeed { + files?: Record + dirs?: Array + /** A link and its target. */ + links?: Record + /** Paths that are not a file, a folder, or a link, like a fifo. */ + others?: Array +} + +/** + * A sandbox file system in memory, by POSIX path. Like a real sandbox: + * `readBytes`, `exists`, and `list` follow links, `lstat` does not, `write` + * makes the parent folders, and `remove` is `rm -rf`. With + * `withLstat: false`, it has no `lstat`. + */ +function memoryFs(seed: FsSeed, withLstat = true) { + const files = new Map( + Object.entries(seed.files ?? {}).map(([path, text]) => [ + path, + encoder.encode(text), + ]), + ) + const dirs = new Set(['/', ...(seed.dirs ?? [])]) + const links = new Map(Object.entries(seed.links ?? {})) + const others = new Set(seed.others ?? []) + const target = (path: string) => links.get(path) ?? path + const parent = (path: string) => path.slice(0, path.lastIndexOf('/')) || '/' + const baseName = (path: string) => path.slice(path.lastIndexOf('/') + 1) + const paths = () => [...files.keys(), ...dirs, ...links.keys(), ...others] + + const lstat: SandboxFs['lstat'] = async (path) => { + const data = files.get(path) + if (links.has(path)) return { type: 'symlink', mode: 0o120777 } + if (data) return { type: 'file', mode: 0o100644, size: data.byteLength } + if (dirs.has(path)) return { type: 'dir', mode: 0o40755 } + if (others.has(path)) return { type: 'other', mode: 0o10644 } + return undefined + } + const readBytes: SandboxFs['readBytes'] = async (path) => { + const data = files.get(target(path)) + if (data) return data + throw new Error(`${dirs.has(target(path)) ? 'EISDIR' : 'ENOENT'}: ${path}`) + } + const fs: SandboxFs = { + read: async (path) => decoder.decode(await readBytes(path)), + readBytes, + write: async (path, data) => { + for (let dir = parent(path); !dirs.has(dir); dir = parent(dir)) { + dirs.add(dir) + } + files.set(path, typeof data === 'string' ? encoder.encode(data) : data) + }, + list: async (path) => { + if (!dirs.has(target(path))) throw new Error(`ENOENT: ${path}`) + const names = paths() + .filter((each) => each !== '/' && parent(each) === target(path)) + .map(baseName) + .sort() + return names.map((name) => { + const type: 'file' | 'dir' = dirs.has(target(`${path}/${name}`)) + ? 'dir' + : 'file' + return { name, path: `${path}/${name}`, type } + }) + }, + mkdir: async (path) => { + dirs.add(path) + }, + remove: async (path) => { + const gone = paths().filter( + (each) => each === path || each.startsWith(`${path}/`), + ) + for (const each of gone) { + files.delete(each) + dirs.delete(each) + links.delete(each) + } + }, + rename: () => Promise.reject(new Error('rename is not used')), + exists: async (path) => files.has(target(path)) || dirs.has(target(path)), + ...(withLstat ? { lstat } : {}), + } + return { fs, files, dirs } +} + +async function* fromChunks(chunks: Array) { + for (const chunk of chunks) yield chunk +} + +/** + * Sandbox processes. `exec` answers like a shell that prints the command, + * the folder, and `$AGENT`. A `sleep` command never ends, like in a sandbox + * that cannot stop it, and only notes the abort in `aborted`. + * + * `spawn` prints ` out` and ` err`. `serve` runs until it + * is killed, then ends with 143. `nope` does not start. + */ +function fakeProcesses() { + const aborted: Array = [] + const killed: Array = [] + const process: SandboxHandle['process'] = { + exec: (command, options = {}) => { + if (command.startsWith('sleep')) { + options.signal?.addEventListener('abort', () => aborted.push(command)) + return new Promise(() => {}) + } + return Promise.resolve({ + exitCode: 3, + stdout: `ran ${command} in ${options.cwd} with AGENT=${options.env?.AGENT}`, + stderr: 'warn', + }) + }, + spawn: async (command) => { + if (command === 'nope') throw new Error('nope: not found') + let end = () => {} + const ended = + command === 'serve' + ? new Promise((resolve) => (end = resolve)) + : Promise.resolve() + async function* stdout() { + yield `${command} out\n` + await ended + } + return { + pid: 7, + stdout: stdout(), + stderr: fromChunks([`${command} err\n`]), + stdin: { write: async () => {}, end: async () => {} }, + wait: async () => { + await ended + return command === 'serve' ? 143 : 0 + }, + kill: async () => { + killed.push(command) + end() + }, + } + }, + } + return { process, aborted, killed } +} + +/** A sandbox handle with `memoryFs` and `fakeProcesses`. */ +function fakeSandbox( + seed: FsSeed = {}, + options: { withLstat?: boolean; caps?: SandboxCapabilities } = {}, +) { + const { fs, files, dirs } = memoryFs(seed, options.withLstat) + const { process, aborted, killed } = fakeProcesses() + const handle: SandboxHandle = { + ...makeFakeHandle('sandbox-1', 'fake', options.caps), + fs, + process, + } + const backend = sandboxWorkspaceBackend(handle) + return { backend, files, dirs, aborted, killed } +} + +const SEED: FsSeed = { + files: { '/workspace/a.txt': 'hello' }, + dirs: ['/workspace', '/workspace/src'], + links: { + '/workspace/to-file': '/workspace/a.txt', + '/workspace/to-src': '/workspace/src', + '/workspace/broken': '/workspace/gone', + }, + others: ['/workspace/pipe'], +} + +describe('sandboxWorkspaceBackend files', () => { + it('uses POSIX paths and quoting, and checks no links', () => { + const { backend } = fakeSandbox() + expect(backend.shell).toBe('sh') + expect(backend).not.toHaveProperty('realpath') + }) + + it('reads and writes the bytes of a file in the sandbox', async () => { + const { backend, files } = fakeSandbox(SEED) + expect(decoder.decode(await backend.readFile('/workspace/a.txt'))).toBe( + 'hello', + ) + await backend.writeFile('/workspace/new/b.txt', 'made') + await backend.writeFile('/workspace/c.bin', new Uint8Array([1, 2])) + expect(decoder.decode(files.get('/workspace/new/b.txt'))).toBe('made') + expect(files.get('/workspace/c.bin')).toEqual(new Uint8Array([1, 2])) + }) + + it.each([ + { withLstat: true, path: '/workspace/a.txt', type: 'file', size: 5 }, + { withLstat: true, path: '/workspace/src', type: 'dir', size: 0 }, + { withLstat: true, path: '/workspace/to-file', type: 'file', size: 5 }, + { withLstat: true, path: '/workspace/to-src', type: 'dir', size: 0 }, + { withLstat: true, path: '/workspace/pipe', type: 'file', size: 0 }, + { withLstat: false, path: '/workspace/a.txt', type: 'file', size: 5 }, + { withLstat: false, path: '/workspace/src', type: 'dir', size: 0 }, + { withLstat: false, path: '/workspace/to-file', type: 'file', size: 5 }, + ])( + 'stat gives a $type of size $size for $path (lstat: $withLstat)', + async ({ withLstat, path, type, size }) => { + const { backend } = fakeSandbox(SEED, { withLstat }) + expect(await backend.stat(path)).toEqual({ type, size, mtimeMs: 0 }) + }, + ) + + it.each([true, false])( + 'stat gives undefined for a missing path and a broken link (lstat: %s)', + async (withLstat) => { + const { backend } = fakeSandbox(SEED, { withLstat }) + expect(await backend.stat('/workspace/gone')).toBeUndefined() + expect(await backend.stat('/workspace/broken')).toBeUndefined() + }, + ) + + it('lists a folder with links as links when the sandbox has lstat', async () => { + const { backend } = fakeSandbox(SEED) + expect(await backend.readdir('/workspace')).toEqual([ + { name: 'a.txt', type: 'file' }, + { name: 'broken', type: 'link' }, + { name: 'pipe', type: 'file' }, + { name: 'src', type: 'dir' }, + { name: 'to-file', type: 'link' }, + { name: 'to-src', type: 'link' }, + ]) + }) + + it('lists a link with the type of its target when the sandbox has no lstat', async () => { + const { backend } = fakeSandbox(SEED, { withLstat: false }) + expect(await backend.readdir('/workspace')).toEqual([ + { name: 'a.txt', type: 'file' }, + { name: 'broken', type: 'file' }, + { name: 'pipe', type: 'file' }, + { name: 'src', type: 'dir' }, + { name: 'to-file', type: 'file' }, + { name: 'to-src', type: 'dir' }, + ]) + }) + + it('removes one file, and refuses a folder or a missing file', async () => { + const { backend, files, dirs } = fakeSandbox({ + files: { '/workspace/a.txt': 'a', '/workspace/src/b.ts': 'b' }, + dirs: ['/workspace', '/workspace/src'], + }) + await backend.remove('/workspace/a.txt') + expect(files.has('/workspace/a.txt')).toBe(false) + await expect(backend.remove('/workspace/src')).rejects.toThrow( + 'Cannot remove /workspace/src: no such file.', + ) + await expect(backend.remove('/workspace/gone')).rejects.toThrow( + 'Cannot remove /workspace/gone: no such file.', + ) + expect(dirs.has('/workspace/src')).toBe(true) + expect(files.has('/workspace/src/b.ts')).toBe(true) + }) +}) + +describe('sandboxWorkspaceBackend exec', () => { + it('runs a command in the sandbox with the folder and the env', async () => { + const { backend } = fakeSandbox() + expect( + await backend.exec('ls', { cwd: '/workspace', env: { AGENT: '1' } }), + ).toEqual({ + exitCode: 3, + stdout: 'ran ls in /workspace with AGENT=1', + stderr: 'warn', + }) + }) + + it('stops a command at the timeout with exit code 124', async () => { + const { backend, aborted } = fakeSandbox() + expect(await backend.exec('sleep 60', { timeoutMs: 20 })).toEqual({ + exitCode: 124, + stdout: '', + stderr: '', + }) + expect(aborted).toEqual(['sleep 60']) + }) + + it('stops a command with exit code 1 when the signal aborts', async () => { + const { backend, aborted } = fakeSandbox() + const controller = new AbortController() + const result = backend.exec('sleep 60', { + timeoutMs: 60_000, + signal: controller.signal, + }) + controller.abort() + expect(await result).toEqual({ exitCode: 1, stdout: '', stderr: '' }) + expect(aborted).toEqual(['sleep 60']) + }) +}) + +describe('sandboxWorkspaceBackend spawn', () => { + it('gives the exit code and all output of a background command', async () => { + const { backend } = fakeSandbox() + const job = backend.spawn?.('build', { cwd: '/workspace' }) + expect(await job?.wait()).toEqual({ exitCode: 0 }) + expect(job?.output()).toBe('build out\nbuild err\n') + }) + + it('kills a background command, also before it started', async () => { + const { backend, killed } = fakeSandbox() + const job = backend.spawn?.('serve') + job?.kill() + expect(await job?.wait()).toEqual({ exitCode: 143 }) + expect(killed).toEqual(['serve']) + expect(job?.output()).toBe('serve out\nserve err\n') + }) + + it('gives exit code 1 and the error when the command does not start', async () => { + const { backend } = fakeSandbox() + const job = backend.spawn?.('nope') + expect(await job?.wait()).toEqual({ exitCode: 1 }) + expect(job?.output()).toBe('Error: nope: not found') + }) + + it('has no spawn when the sandbox cannot run background processes', () => { + const { backend } = fakeSandbox( + {}, + { caps: { ...FULL_CAPS, backgroundProcesses: false } }, + ) + expect(backend.spawn).toBeUndefined() + }) +}) + +const now = () => Date.now() + +/** The chunks of one model reply: `chunks` in one run. */ +function reply( + finishReason: 'tool_calls' | 'stop', + chunks: Array = [], +) { + const all: Array = [ + { + type: EventType.RUN_STARTED, + runId: 'r', + threadId: 't', + timestamp: now(), + }, + ...chunks, + { + type: EventType.RUN_FINISHED, + runId: 'r', + threadId: 't', + timestamp: now(), + metadata: { tanstack: { finishReason } }, + }, + ] + return all +} + +/** The chunks of one model reply that calls `tool` with `args`. */ +function toolCall(id: string, tool: string, args: Record) { + return reply('tool_calls', [ + { + type: EventType.TOOL_CALL_START, + toolCallId: id, + toolCallName: tool, + timestamp: now(), + }, + { + type: EventType.TOOL_CALL_ARGS, + toolCallId: id, + delta: JSON.stringify(args), + timestamp: now(), + }, + { type: EventType.TOOL_CALL_END, toolCallId: id, timestamp: now() }, + ]) +} + +/** + * A model that gives `replies` in order, then ends the turn. `calls` has the + * options of each call. + */ +function scriptedModel(replies: Array>) { + const calls: Array = [] + const adapter: AnyTextAdapter = { + kind: 'text', + name: 'scripted', + model: 'scripted', + '~types': { + providerOptions: {}, + inputModalities: ['text'], + messageMetadataByModality: {}, + toolCapabilities: [], + toolCallMetadata: undefined, + systemPromptMetadata: undefined, + }, + structuredOutput: async () => ({ data: {}, rawText: '{}' }), + chatStream: (options) => { + calls.push(options) + const chunks = replies[calls.length - 1] ?? reply('stop') + return (async function* () { + yield* chunks + })() + }, + } + return { adapter, calls } +} + +describe('sandboxWorkspaceBackend with workspaceTools', () => { + it('writes and reads a file under /workspace, with POSIX paths for the model', async () => { + const sandbox = fakeSandbox({ dirs: ['/workspace'] }) + const { adapter, calls } = scriptedModel([ + toolCall('c1', 'write_file', { + path: 'src/a.ts', + content: 'export const a = 1\n', + }), + toolCall('c2', 'read_file', { path: 'src/a.ts' }), + ]) + const host = createHarnessHost({ persistence: memoryPersistence() }) + const session = await host.open( + defineHarness({ + name: 'test/sandbox-workspace', + adapter, + plugins: () => [ + workspaceTools({ + root: '/workspace', + backend: sandbox.backend, + web: false, + }), + ], + }), + { threadId: 't' }, + ) + await session.prompt('work') + await host.close() + + expect(decoder.decode(sandbox.files.get('/workspace/src/a.ts'))).toBe( + 'export const a = 1\n', + ) + const results = calls[2]?.messages + .filter((message) => message.role === 'tool') + .map((message) => message.content) + expect(results).toEqual(['Wrote src/a.ts.', '1\texport const a = 1']) + expect(calls[0]?.systemPrompts).toContainEqual( + expect.stringContaining('Your workspace is /workspace.'), + ) + }) +}) diff --git a/packages/ai-sandbox/vite.config.ts b/packages/ai-sandbox/vite.config.ts index 5eaa0e507a..e7d52c2e2a 100644 --- a/packages/ai-sandbox/vite.config.ts +++ b/packages/ai-sandbox/vite.config.ts @@ -34,7 +34,12 @@ export default mergeConfig( // (`@tanstack/ai-sandbox/ngrok`), which lazy-loads the optional `@ngrok/ngrok` // peer dep so the core never pulls in its native binary + the SandboxInstanceStore // conformance testkit (`@tanstack/ai-sandbox/testkit`). - entry: ['./src/index.ts', './src/ngrok.ts', './src/testkit/conformance.ts'], + entry: [ + './src/index.ts', + './src/harness.ts', + './src/ngrok.ts', + './src/testkit/conformance.ts', + ], srcDir: './src', // The conformance testkit imports Vitest; keep it external so the built // artifact references the consumer's Vitest at test time. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 00201dc041..eba60e1814 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3138,6 +3138,9 @@ importers: '@tanstack/ai': specifier: workspace:* version: link:../ai + '@tanstack/ai-harness': + specifier: workspace:* + version: link:../ai-harness '@tanstack/ai-persistence': specifier: workspace:* version: link:../ai-persistence diff --git a/testing/e2e/fixtures/tools-test/coding-tools.json b/testing/e2e/fixtures/tools-test/coding-tools.json index f323e1537f..12d6c61760 100644 --- a/testing/e2e/fixtures/tools-test/coding-tools.json +++ b/testing/e2e/fixtures/tools-test/coding-tools.json @@ -7,7 +7,10 @@ }, "response": { "toolCalls": [ - { "name": "read_file", "arguments": "{\"path\":\"notes.txt\"}" } + { + "name": "read_file", + "arguments": "{\"path\":\"notes.txt\"}" + } ] } }, @@ -72,7 +75,12 @@ "sequenceIndex": 0 }, "response": { - "toolCalls": [{ "name": "grep", "arguments": "{\"pattern\":\"beta\"}" }] + "toolCalls": [ + { + "name": "grep", + "arguments": "{\"pattern\":\"beta\"}" + } + ] } }, { @@ -172,7 +180,32 @@ "userMessage": "[coding-format-ask] run test", "sequenceIndex": 1 }, - "response": { "content": "Wrote a.ts." } + "response": { + "content": "Wrote a.ts." + } + }, + { + "match": { + "userMessage": "[coding-sandbox] run test", + "sequenceIndex": 0 + }, + "response": { + "toolCalls": [ + { + "name": "edit_file", + "arguments": "{\"path\":\"notes.txt\",\"old\":\"alpha\",\"new\":\"sandboxed\"}" + } + ] + } + }, + { + "match": { + "userMessage": "[coding-sandbox] run test", + "sequenceIndex": 1 + }, + "response": { + "content": "Edited notes.txt in the sandbox." + } } ] } diff --git a/testing/e2e/src/routes/api.coding-tools.ts b/testing/e2e/src/routes/api.coding-tools.ts index 7d147b00d3..bb75d5c5c9 100644 --- a/testing/e2e/src/routes/api.coding-tools.ts +++ b/testing/e2e/src/routes/api.coding-tools.ts @@ -14,6 +14,8 @@ import { snapshots, workspaceTools, } from '@tanstack/ai-harness/plugins/coding' +import { sandboxWorkspaceBackend } from '@tanstack/ai-sandbox/harness' +import { localProcessSandbox } from '@tanstack/ai-sandbox-local-process' import type { AnyTextAdapter, StreamChunk } from '@tanstack/ai' import { createTextAdapter } from '@/lib/providers' @@ -103,6 +105,40 @@ async function* revertTurns( } } +/** + * The `coding-sandbox` scenario: one `edit_file` turn with the tools in a + * local-process sandbox on `root`. The tools see `/workspace`, and the + * sandbox maps it to `root`. + */ +async function* sandboxTurn( + root: string, + adapter: AnyTextAdapter, +): AsyncGenerator { + const sandbox = await localProcessSandbox({ dir: root }).create({}) + const host = createHarnessHost() + try { + const session = await host.open( + defineHarness({ + name: 'e2e/coding-tools-sandbox', + adapter, + plugins: () => [ + workspaceTools({ + root: '/workspace', + editStyle: 'edit', + backend: sandboxWorkspaceBackend(sandbox), + }), + ], + }), + { threadId: 'coding-sandbox' }, + ) + yield* session.prompt('[coding-sandbox] run test').stream() + } finally { + await host.close() + // With `dir`, destroy keeps the folder. + await sandbox.destroy() + } +} + /** * The coding tools of `@tanstack/ai-harness/plugins/coding` on real files. * Body: `{ scenario, testId, root }`. `root` must be a folder in the temp @@ -139,7 +175,9 @@ export const Route = createFileRoute('/api/coding-tools')({ return toServerSentEventsResponse( scenario === 'coding-revert' ? revertTurns(root, adapter) - : codingTurn(scenario, root, adapter), + : scenario === 'coding-sandbox' + ? sandboxTurn(root, adapter) + : codingTurn(scenario, root, adapter), ) }, }, diff --git a/testing/e2e/tests/tools-test/coding-tools.spec.ts b/testing/e2e/tests/tools-test/coding-tools.spec.ts index c7016fa0d4..da0ccbcdb4 100644 --- a/testing/e2e/tests/tools-test/coding-tools.spec.ts +++ b/testing/e2e/tests/tools-test/coding-tools.spec.ts @@ -149,6 +149,20 @@ test.describe('Coding tools', () => { expect(await read('bash.txt')).toBe('AGENT=1') }) + test('edit_file works in a sandbox with sandboxWorkspaceBackend', async ({ + request, + testId, + }) => { + await writeFile(file('notes.txt'), 'alpha\nbeta\n') + + // The sandbox maps its /workspace to the test folder. + expect(await runScenario(request, 'coding-sandbox', testId)).toEqual({ + results: ['Edited notes.txt (1 change).'], + text: 'Edited notes.txt in the sandbox.', + }) + expect(await read('notes.txt')).toBe('sandboxed\nbeta\n') + }) + test('revert puts back the files of the turns and keeps the edit of the user', async ({ request, testId,