diff --git a/CLAUDE.md b/CLAUDE.md index ec3f96b725..fc252dd1f6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -151,11 +151,35 @@ gawk, real curl/sqlite/duckdb/vim, …) compiled to `wasm32-wasip1` against a custom host-syscall imports. Treat that target as **native POSIX**; `wasm32-wasip1` is an implementation detail, not a feature ceiling. +### `wasm32-wasip1` capability rule + +- **Never use `target_os = "wasi"` as evidence that a POSIX capability is + absent.** In agentOS it does not imply missing symlinks, executable modes, + TCP/UDP/Unix sockets, processes, signals, polling, or other Linux behavior. +- When an upstream Rust crate hides a supported API behind `cfg(unix)`, expose + the same behavior through the corresponding `std::os::wasi` or libc API. A + crate or Codex patch may adapt target-specific API names and cfg routing; it + must not replace working behavior with `Unsupported`, a no-op, a dummy + capability report, or a Windows fallback. +- Fix a genuinely missing POSIX operation in this order: Rust std, agentOS + libc/sysroot, then the host import and kernel implementation. Do not make an + application compile by disabling the operation one layer above the gap. +- A WASI stub is allowed only for a genuinely host-native architectural + boundary, not for a missing syscall. Its source must state which trusted + agentOS component owns the behavior and why the guest implementation must not + run. For example, guest HTTP uses the sidecar-brokered HTTP transport because + the sidecar owns network policy; that does not mean guest TCP sockets are + unsupported. +- Before accepting a new WASI patch or stub, audit every error path and cfg for + capability loss. Compilation alone is not validation: add or run a guest + behavior test for each POSIX surface the patch exposes. + - **We do not depend on stock WASI / wasi-libc.** The sysroot is ours. A missing libc/POSIX API (`getrlimit`/`RLIMIT_NOFILE`, `getgroups`, spawn, fd dup, …) is - never a blocker — implement it (real, or a sane stub) in the patched + never a blocker — implement the real operation in the patched std/libc/host-import layer. "WASI doesn't have X" is not a reason to stop; X is - ours to add. + ours to add. Do not substitute a success-returning stub, hard-coded value, or + target-level omission for a syscall agentOS can implement. - **Fix portability one layer down, in the sysroot** — a new std/libc patch or a new host import — not with `cfg(target_*)` branches or shims in the tool's own source. A WASM-specific branch in application code usually means the fix diff --git a/crates/execution/assets/runners/wasm-runner.mjs b/crates/execution/assets/runners/wasm-runner.mjs index 919e3ba3c7..404605c450 100644 --- a/crates/execution/assets/runners/wasm-runner.mjs +++ b/crates/execution/assets/runners/wasm-runner.mjs @@ -8647,7 +8647,26 @@ const hostProcessImport = { } }, pty_open(retMasterFdPtr, retSlaveFdPtr) { - return WASI_ERRNO_FAULT; + let masterFd = null; + let slaveFd = null; + try { + if (!hasRunnerOpenFdCapacity(2)) return WASI_ERRNO_MFILE; + const result = callSyncRpc('process.pty_open'); + masterFd = registerKernelDelegateFd(result?.masterFd); + slaveFd = registerKernelDelegateFd(result?.slaveFd); + if (masterFd == null || slaveFd == null) return WASI_ERRNO_FAULT; + if (writeGuestUint32(retMasterFdPtr, masterFd) !== WASI_ERRNO_SUCCESS + || writeGuestUint32(retSlaveFdPtr, slaveFd) !== WASI_ERRNO_SUCCESS) { + wasiImport.fd_close(masterFd); + wasiImport.fd_close(slaveFd); + return WASI_ERRNO_FAULT; + } + return WASI_ERRNO_SUCCESS; + } catch (error) { + if (masterFd != null) wasiImport.fd_close(masterFd); + if (slaveFd != null) wasiImport.fd_close(slaveFd); + return mapHostProcessError(error); + } }, proc_sigaction(signal, action, maskLo, maskHi, flags) { if (permissionTier !== 'full') { @@ -12372,6 +12391,16 @@ const hostTtyImport = { view.setUint16(rowsPtr >>> 0, size.rows & 0xffff, true); return 0; }, + // Resize any kernel PTY descriptor and let the kernel deliver SIGWINCH to + // its foreground process group. + set_size(fd, cols, rows) { + try { + callSyncRpc('process.pty_resize', [fd >>> 0, cols >>> 0, rows >>> 0]); + return 0; + } catch (error) { + return mapHostProcessError(error); + } + }, // Toggle terminal raw mode on the guest's PTY. crossterm/pty_probe/vim call this // instead of tcsetattr; route it to the kernel so the guest gets raw keystrokes. set_raw_mode(enabled) { diff --git a/crates/execution/src/wasm.rs b/crates/execution/src/wasm.rs index e295592c9b..daa9a282e2 100644 --- a/crates/execution/src/wasm.rs +++ b/crates/execution/src/wasm.rs @@ -3475,6 +3475,8 @@ if (typeof globalThis !== "undefined") {{ case "process.waitpid_transition": case "process.itimer_real": case "process.fd_pipe": + case "process.pty_open": + case "process.pty_resize": case "process.fd_open": case "process.path_open_at": case "process.path_mkdir_at": diff --git a/crates/native-sidecar/src/execution/javascript/rpc.rs b/crates/native-sidecar/src/execution/javascript/rpc.rs index bbe575c27e..06d3459e15 100644 --- a/crates/native-sidecar/src/execution/javascript/rpc.rs +++ b/crates/native-sidecar/src/execution/javascript/rpc.rs @@ -56,6 +56,8 @@ const ALLOWED_WASM_PROCESS_SYNC_RPCS: &[&str] = &[ "process.waitpid_transition", "process.itimer_real", "process.fd_pipe", + "process.pty_open", + "process.pty_resize", "process.fd_open", "process.path_open_at", "process.path_mkdir_at", @@ -968,6 +970,31 @@ where .open_pipe(EXECUTION_DRIVER_NAME, process.kernel_pid) .map(|(read_fd, write_fd)| json!({ "readFd": read_fd, "writeFd": write_fd })) .map_err(kernel_error), + "process.pty_open" => kernel + .open_pty(EXECUTION_DRIVER_NAME, process.kernel_pid) + .map(|(master_fd, slave_fd, path)| { + json!({ "masterFd": master_fd, "slaveFd": slave_fd, "path": path }) + }) + .map_err(kernel_error), + "process.pty_resize" => { + let fd = javascript_sync_rpc_arg_u32(&request.args, 0, "PTY fd")?; + let cols = u16::try_from(javascript_sync_rpc_arg_u32( + &request.args, + 1, + "PTY columns", + )?) + .map_err(|_| SidecarError::InvalidState(String::from("EINVAL: PTY columns exceed u16")))?; + let rows = u16::try_from(javascript_sync_rpc_arg_u32( + &request.args, + 2, + "PTY rows", + )?) + .map_err(|_| SidecarError::InvalidState(String::from("EINVAL: PTY rows exceed u16")))?; + kernel + .pty_resize(EXECUTION_DRIVER_NAME, process.kernel_pid, fd, cols, rows) + .map(|_| Value::Null) + .map_err(kernel_error) + } "process.fd_open" => { let path = javascript_sync_rpc_arg_str(&request.args, 0, "fd_open path")?; let flags = javascript_sync_rpc_arg_u32(&request.args, 1, "fd_open flags")?; diff --git a/examples/codex/client.ts b/examples/codex/client.ts index 1a827c5541..9d3056d9ee 100644 --- a/examples/codex/client.ts +++ b/examples/codex/client.ts @@ -59,7 +59,9 @@ Write commit messages in the imperative mood and keep the subject under 50 chara async function withMcp() { // Pre-install the MCP server so `npx` is silent — first-run install output // would otherwise corrupt the MCP stdio handshake ("Connection closed"). - await agent.process.exec("npm install -g @modelcontextprotocol/server-filesystem"); + await agent.process.exec( + "npm install -g @modelcontextprotocol/server-filesystem", + ); const config = `[mcp_servers.filesystem] command = "npx" @@ -79,6 +81,62 @@ http_headers = { Authorization = "Bearer my-token" } } // docs:end mcp +// docs:start agent-plugins +// ── Agent Plugins ───────────────────────────────────────────────── +// +// Install a portable Agent Plugin into Codex's cache before opening the +// session. Other compatible clients use their own installation locations. +async function withAgentPlugin() { + const pluginRoot = + "/home/agentos/.codex/plugins/cache/local/release-workflow/1.0.0"; + await agent.filesystem.mkdir(`${pluginRoot}/.codex-plugin`); + await agent.filesystem.mkdir(`${pluginRoot}/skills/release-notes`); + + await agent.filesystem.writeFile( + `${pluginRoot}/.codex-plugin/plugin.json`, + JSON.stringify({ + $schema: "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", + name: "release-workflow", + version: "1.0.0", + description: "Project release workflow", + }), + ); + await agent.filesystem.writeFile( + `${pluginRoot}/skills/release-notes/SKILL.md`, + `--- +name: release-notes +description: Write release notes for this project. +--- + +Summarize user-visible changes under Added, Changed, and Fixed. +`, + ); + + await agent.filesystem.writeFile( + "/home/agentos/.codex/config.toml", + `[features] +plugins = true + +[plugins."release-workflow@local"] +enabled = true +`, + ); + + await agent.sessions.open({ + agent: "codex", + env: { OPENAI_API_KEY: process.env.OPENAI_API_KEY! }, + }); + await agent.sessions.prompt({ + content: [ + { + type: "text", + text: "Use $release-workflow:release-notes for the current changes.", + }, + ], + }); +} +// docs:end agent-plugins + // ── Skills + MCP together ───────────────────────────────────────── async function withSkillAndMcp() { const skill = `--- @@ -97,7 +155,9 @@ Write commit messages in the imperative mood and keep the subject under 50 chara // Pre-install the MCP server so `npx` is silent — first-run install output // would otherwise corrupt the MCP stdio handshake ("Connection closed"). - await agent.process.exec("npm install -g @modelcontextprotocol/server-filesystem"); + await agent.process.exec( + "npm install -g @modelcontextprotocol/server-filesystem", + ); const config = `[mcp_servers.filesystem] command = "npx" @@ -122,4 +182,4 @@ args = ["-y", "@modelcontextprotocol/server-filesystem", "/home/agentos"] console.log(result.message?.content ?? []); } -export { quickStart, withSkill, withMcp, withSkillAndMcp }; +export { quickStart, withAgentPlugin, withMcp, withSkill, withSkillAndMcp }; diff --git a/packages/core/tests/codex-fullturn.nightly.test.ts b/packages/core/tests/codex-fullturn.nightly.test.ts index 7685387d11..185b3f8da3 100644 --- a/packages/core/tests/codex-fullturn.nightly.test.ts +++ b/packages/core/tests/codex-fullturn.nightly.test.ts @@ -10,7 +10,11 @@ import { } from "./helpers/openai-responses-mock.js"; import { REGISTRY_SOFTWARE } from "./helpers/registry-commands.js"; -const codexConfig = `[features] +const codexConfig = ( + openAiBaseUrl: string, +) => `openai_base_url = "${openAiBaseUrl}" + +[features] # Shell snapshots spawn a pre-turn shell subprocess. The real turn coverage # below does not need that optional context, and disabling it keeps the WASI VM # focused on the codex-core model/tool path under test. @@ -34,6 +38,7 @@ async function runSessionTurn( fixtures: ResponsesFixture[], start: Record, stdinTail = "", + prepare?: (vm: AgentOs, openAiBaseUrl: string) => Promise, ) { const mock = await startResponsesMock(fixtures); const vm = await AgentOs.create({ @@ -53,8 +58,9 @@ async function runSessionTurn( await vm.execArgv("mkdir", ["-p", "/home/agentos/.codex"]); await vm.writeFile( "/home/agentos/.codex/config.toml", - new TextEncoder().encode(codexConfig), + new TextEncoder().encode(codexConfig(`${mock.url}/v1`)), ); + await prepare?.(vm, `${mock.url}/v1`); const r = await vm.execArgv("codex-exec", ["--session-turn"], { timeout: 45000, stdin, @@ -95,6 +101,62 @@ const finalText = (text: string): ResponsesFixture => ({ describe.skipIf(!hasCodexExecArtifact)( "codex full turn (real codex agent in the VM, mock OpenAI Responses)", () => { + test("loads a portable Agent Plugin skill into the model context", async () => { + const pluginRoot = + "/home/agentos/.codex/plugins/cache/local/release-workflow/1.0.0"; + const { stdout, stderr, exitCode, requests } = await runSessionTurn( + [finalText("used the release notes skill")], + { prompt: "Use $release-workflow:release-notes for these changes." }, + "", + async (vm, openAiBaseUrl) => { + await vm.execArgv("mkdir", ["-p", `${pluginRoot}/.codex-plugin`]); + await vm.execArgv("mkdir", [ + "-p", + `${pluginRoot}/skills/release-notes`, + ]); + await vm.writeFile( + `${pluginRoot}/.codex-plugin/plugin.json`, + new TextEncoder().encode( + JSON.stringify({ + $schema: + "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", + name: "release-workflow", + version: "1.0.0", + description: "Project release workflow", + }), + ), + ); + await vm.writeFile( + `${pluginRoot}/skills/release-notes/SKILL.md`, + new TextEncoder().encode(`--- +name: release-notes +description: Write release notes for this project. +--- + +Summarize user-visible changes under Added, Changed, and Fixed. +`), + ); + await vm.writeFile( + "/home/agentos/.codex/config.toml", + new TextEncoder().encode(`${codexConfig(openAiBaseUrl)} +plugins = true + +[plugins."release-workflow@local"] +enabled = true +`), + ); + }, + ); + expect( + requests.length, + `codex-exec did not call mock Responses; exitCode=${exitCode}; stderr=${stderr}; stdout=${stdout}`, + ).toBeGreaterThan(0); + const requestBody = JSON.stringify(requests[0]); + expect(requestBody).toContain("release-workflow:release-notes"); + expect(requestBody).toContain("Write release notes for this project."); + expect(stdout).toContain('"type":"done"'); + }, 70000); + test("codex-exec --session-turn completes a model turn end-to-end", async () => { const { stdout, stderr, exitCode, requests } = await runSessionTurn( [finalText("hello from codex")], diff --git a/toolchain/codex-ref b/toolchain/codex-ref index c50b0698e8..d6bf40e604 100644 --- a/toolchain/codex-ref +++ b/toolchain/codex-ref @@ -1 +1 @@ -rivet-dev/codex@8f4655a9dbfc7db91768c78ccaea7357ba47127e +rivet-dev/codex@aed385c00e5156596d57edb8e4a5f02ec6b8172f diff --git a/toolchain/scripts/clone-and-build-codex-wasi.sh b/toolchain/scripts/clone-and-build-codex-wasi.sh index dce357e58b..ea36e32cee 100755 --- a/toolchain/scripts/clone-and-build-codex-wasi.sh +++ b/toolchain/scripts/clone-and-build-codex-wasi.sh @@ -9,9 +9,10 @@ # # 1. Read the pin from toolchain/codex-ref ("/@"). # 2. Shallow-clone the fork at that exact SHA into a gitignored scratch dir. -# 3. Rewrite the clone's [patch.crates-io] to point at the toolchain's reproducible -# stubs (toolchain/stubs/{reqwest-shim,portable-pty-wasi,ctrlc}); drop tokio's -# machine-path patch so tokio 1.52.3 comes through the vendored+patched tree. +# 3. Rewrite the clone's existing local development overrides to the +# reproducible agentOS adapters, while keeping real AWS authentication and +# the Codex network-policy crate. Drop tokio's machine-path patch so the +# locked Tokio release comes through the vendored+patched tree. # 4. Strip the hardcoded `-L .../self-contained` from the clone's .cargo/config.toml # (that absolute rustup path is machine-specific; it is recomputed and passed via # RUSTFLAGS at build time instead). @@ -116,24 +117,28 @@ grep -q 'Config::load_default_with_cli_overrides' \ } printf '%s\n' "$EXPECTED_STAMP" > "$STAMP" -# --- 3. rewrite [patch.crates-io] -> reproducible toolchain stubs ------------ -echo "== rewriting [patch.crates-io] -> toolchain/stubs/* ==" +# --- 3. rewrite local fork overrides to reproducible agentOS adapters -------- +echo "== rewriting local Codex overrides -> reproducible agentOS adapters ==" python3 - "$WORKSPACE/Cargo.toml" "$STUBS" <<'PY' import re, sys cargo, stubs = sys.argv[1], sys.argv[2] lines = open(cargo).read().split('\n') -# Crates whose patch we own: reqwest/portable-pty/ctrlc get repointed at the -# reproducible toolchain stubs; tokio's patch is dropped so tokio 1.52.3 resolves -# through the vendored+patched sources (std-patches/crates/tokio/0001). +# Remove the fork developer's machine-local paths. reqwest is the trusted +# sidecar HTTP boundary. portable-pty is a thin adapter over agentOS's real +# POSIX PTY/process host imports. The stale ctrlc path override is removed; +# this build does not depend on ctrlc. Tokio resolves through vendored+patched +# sources. OWNED = ('portable-pty', 'reqwest', 'ctrlc', 'tokio') owned_re = re.compile(r'^\s*#?\s*(?:%s)\s*=' % '|'.join(map(re.escape, OWNED))) inject = [ - '# [patch.crates-io] injected by clone-and-build-codex-wasi.sh (reproducible stubs)', + '# [patch.crates-io] injected by clone-and-build-codex-wasi.sh', 'portable-pty = {{ path = "{}/portable-pty-wasi" }}'.format(stubs), 'reqwest = {{ path = "{}/reqwest-shim" }}'.format(stubs), - 'ctrlc = {{ path = "{}/ctrlc" }}'.format(stubs), - '# tokio: vendored+patched tokio 1.52.3 (std-patches/crates/tokio), not a path patch', + '# tokio: vendored and patched from the pinned lockfile, not a path patch', ] +injected_comment_re = re.compile( + r'^# (?:\[patch\.crates-io\] injected by clone-and-build-codex-wasi\.sh|tokio: vendored)' +) in_patch = False injected = False out = [] @@ -147,11 +152,22 @@ for ln in lines: continue if in_patch and owned_re.match(ln): # drop any prior line for an owned crate continue + if in_patch and injected_comment_re.match(ln): + continue out.append(ln) if not injected: # no [patch.crates-io] in the fork: add one out += ['', '[patch.crates-io]'] + inject open(cargo, 'w').write('\n'.join(out)) PY +grep -Fq 'codex-network-proxy = { path = "network-proxy" }' "$WORKSPACE/Cargo.toml" || { + echo "ERROR: Codex must use its pinned network-proxy policy/config crate" >&2 + exit 1 +} +grep -Fq "proxy execution and network policy are owned by the trusted agentOS sidecar" \ + "$WORKSPACE/network-proxy/src/proxy.rs" || { + echo "ERROR: Codex's agentOS proxy boundary must fail explicitly" >&2 + exit 1 +} echo " --- resulting [patch.crates-io] head ---" sed -n '/^\[patch.crates-io\]/,/^\[/p' "$WORKSPACE/Cargo.toml" | sed 's/^/ /' @@ -202,19 +218,46 @@ echo "== applying toolchain/std-patches/crates/* to vendored sources ==" VENDOR_DIR="$WORKSPACE/vendor" "$SCRIPT_DIR/patch-vendor.sh" || \ echo " (patch-vendor reported failures; verifying codex-critical patches below)" +# `cargo vendor --sync` can retain a second libc release for build-std. The +# generic patcher selects the unversioned application copy, so apply the same +# agentOS ABI declarations to every versioned libc source Cargo may compile. +for libc_dir in "$WORKSPACE/vendor"/libc-*; do + [ -d "$libc_dir" ] || continue + libc_patch="$TOOLCHAIN_DIR/std-patches/crates/libc/0001-wasip1-socket-bindings.patch" + if patch --dry-run -p1 -d "$libc_dir" < "$libc_patch" >/dev/null 2>&1; then + patch -p1 -d "$libc_dir" < "$libc_patch" >/dev/null + elif ! patch --dry-run -R -p1 -d "$libc_dir" < "$libc_patch" >/dev/null 2>&1; then + echo "ERROR: agentOS libc socket bindings do not apply to $libc_dir" >&2 + exit 1 + fi + python3 - "$libc_dir/.cargo-checksum.json" <<'PY' +import json, sys +path = sys.argv[1] +with open(path) as source: + checksum = json.load(source) +checksum["files"] = {} +with open(path, "w") as output: + json.dump(checksum, output) +PY +done + echo "== verifying codex-critical crate patches applied ==" assert_patched() { #