diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 24fa26a..e17785f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - node: [20, 22] + node: [22, 24] steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 diff --git a/.gitignore b/.gitignore index c93ce80..ac35bd9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ dist/ *.log .claude/ .hermes/ +.devkit/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1c6dec2..c1ba641 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ npm run typecheck && npm test && npm run build node dist/cli.js run examples/hello.yaml ``` -Node >= 20 is required. +Node >= 22 is required (`execa` v10 uses `Set.prototype.union`, which lands in Node 22). ## Working on a change diff --git a/README.md b/README.md index 82cc696..ede5e58 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ It does not call a model itself. Your agent CLIs are the runtime. npm i -g loomgraph ``` -Requires Node >= 20. The binary is `lg`. +Requires Node >= 22. The binary is `lg`. ## 60-second quickstart @@ -118,7 +118,7 @@ edges: to: END ``` -Templates resolve against run state: `{{vars.ticket}}` (or the shorthand `{{ticket}}`) and `{{nodes..output}}`. An unresolvable reference is an error, not an empty string. +Templates resolve against run state: `{{vars.ticket}}` (or the shorthand `{{ticket}}`) and `{{nodes..output}}`. An unresolvable reference is an error, not an empty string and never a passthrough — which is why node ids are restricted to `[A-Za-z0-9_-]`, 1 to 64 characters. A dot would collide with the reference syntax itself, so `lg validate` rejects it rather than letting `{{nodes.my.node.output}}` mean nothing at run time. Check a graph before running it — `lg validate` catches unknown node ids, cycles, missing budgets, and bad adapters: @@ -171,9 +171,16 @@ lg resume --answer approve="ship it" Every node accepts `retries` (default 0), `timeoutSec` (default 900), and `cwd`. +A `command` node also accepts two optional assertions, because a shell command that exits 0 +having done nothing is not a passing check: `expectNonEmpty: true` fails the node when the +command wrote no output, and `expect: ""` fails it when that substring is absent +from stdout. `npm run lint --if-present` in a repo with no lint script is the case these +exist for. + ## Budgets -Three ceilings, all enforced *before* each dispatch batch, all recorded in the checkpoint: +Three ceilings, all enforced *before* each dispatch batch **and once more before a run is +allowed to finish successfully**, all recorded in the checkpoint: - `maxUsd` — summed from what the adapters actually report. - `maxWallClockSec` — measured from the run's creation, so it survives a resume. @@ -181,6 +188,10 @@ Three ceilings, all enforced *before* each dispatch batch, all recorded in the c Hitting a ceiling stops the run with status `failed`, a `budget_exceeded` event naming the ceiling, and exit code 3. Nothing further is dispatched. +A ceiling breached by the final batch fails the run too. A node that already finished keeps +its result — the run fails, the work does not unwind — so `lg status` still shows what was +done and exactly how far over the line it went. + Cost numbers are never invented. Claude Code reports `total_cost_usd` and that number is used as-is; adapters that report no price record exactly `0.0000`, and `lg status` says so. ## Audit trail @@ -200,11 +211,26 @@ Event kinds: `run_started`, `node_started`, `node_finished`, `edge_crossed`, `bu | Adapter | Command it runs | Status | | --- | --- | --- | -| `claude` | `claude -p --output-format json --permission-mode acceptEdits --max-turns ` | Tested against Claude Code 2.1.232 | +| `claude` | `claude -p --output-format json --permission-mode acceptEdits --max-turns ` | Tested against Claude Code 2.1.232 and the array-form json output of 3.x | | `codex` | `codex exec --json --skip-git-repo-check --sandbox read-only -C ` | Tested against codex-cli 0.145.0 | -| `opencode` | `opencode run ` | **Experimental — never executed against a real binary.** The parser is unit-tested; the invocation is not. | +| `opencode` | `opencode run --format json [-m ] ` | Tested against opencode 1.18.17 | + +Cost reporting differs by CLI: Claude Code reports `total_cost_usd`, and OpenCode reports a price per step under `--format json` — which is the only reason this adapter uses that format, since the default one prints prose and no price at all. Codex reports nothing, and loomgraph records `0` for it rather than estimating from a price table. Wall-clock and node-run ceilings still apply either way. -Cost reporting differs by CLI: Claude Code reports `total_cost_usd`; Codex and OpenCode report nothing, and loomgraph records `0` rather than estimating from a price table. Wall-clock and node-run ceilings still apply to them. +### Choosing a model + +An `agent` or `verifier` node may name the model it wants, passed straight through to the CLI: + +```yaml +review: + type: verifier + adapter: opencode + model: "opencode-go/deepseek-v4-flash" + prompt: "Review the diff. Reply PASS or FAIL." + pass: "PASS" +``` + +Omit it and the CLI's own resolution decides, which is not always what the config says: with no `-m`, opencode ignored a configured `model` and fell through to a provider with no credentials. `OPENCODE_MODEL` is ignored — the flag is the only way. A `command` or `human` node that declares a model is a validation error rather than a silently ignored key. ### Environment @@ -230,6 +256,7 @@ Both agent adapters close stdin before spawning. Codex otherwise prints `Reading | `lg status ` | Per-node table plus the budget line | | `lg ls` | Every run with status and cost | | `lg validate ` | Exit 0 if valid, else exit 1 with the specific error | +| `lg report [--out path] [--publish] [--title t] [--visibility private\|org]` | Render the run to a self-contained html file; `--publish` hosts it with the `enclave` cli | | `lg events [--kind K]` | The JSONL audit trail, filterable | Exit codes: `0` success, `1` validation or usage error, `2` run failed, `3` budget exceeded, `4` paused awaiting a human. @@ -240,6 +267,10 @@ Exit codes: `0` success, `1` validation or usage error, `2` run failed, `3` budg - **Not a replacement for your agent CLI.** It shells out to the CLI you already installed and authenticated. - **Not a workflow server.** No daemon, no web UI, no cloud, no plugin system in v0.1. +`lg report --publish` does not change that: it writes a static file and shells out to the +`enclave` cli the same way a node shells out to `claude`. If `enclave` is not installed the +report is still written, and nothing is uploaded. + Concurrency caveat: fan-out nodes in v0.1 share one working directory. If two branches edit the same files, they will collide. Per-node git worktrees are phase 2. ## Roadmap diff --git a/examples/parallel-verify.yaml b/examples/parallel-verify.yaml index 9e59d9a..49c8675 100644 --- a/examples/parallel-verify.yaml +++ b/examples/parallel-verify.yaml @@ -20,16 +20,19 @@ nodes: type: command run: "npm run test --if-present" timeoutSec: 600 + expectNonEmpty: true lint: type: command - run: "npm run lint --if-present" + run: "npm run typecheck" timeoutSec: 600 + expectNonEmpty: true typecheck: type: command run: "npm run typecheck --if-present" timeoutSec: 600 + expectNonEmpty: true merge: type: command diff --git a/package.json b/package.json index 6164708..532e882 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ "author": "Dat Nguyen", "license": "MIT", "engines": { - "node": ">=20" + "node": ">=22" }, "scripts": { "build": "tsup", diff --git a/src/adapters/claude.test.ts b/src/adapters/claude.test.ts index fec1264..1804fc6 100644 --- a/src/adapters/claude.test.ts +++ b/src/adapters/claude.test.ts @@ -7,6 +7,14 @@ const MAX_TURNS = `{"type":"result","subtype":"error_max_turns","result":"","tot // Note `subtype` is "success" while `is_error` is true - trusting subtype alone // makes an auth failure look like a completed agent run. const AUTH_FAILURE = `{"type":"result","subtype":"success","is_error":true,"result":"Failed to authenticate: OAuth session expired and could not be refreshed","terminal_reason":"api_error","total_cost_usd":0,"num_turns":1}`; +// Captured from Claude Code 3.x: stdout is now a JSON array of message +// objects whose last element carries the run result. +const SUCCESS_ARRAY = `[ + {"type":"system","subtype":"init","cwd":"/tmp/x","session_id":"SID"}, + {"type":"assistant","message":{"role":"assistant","content":[{"type":"text","text":"BANANA"}]}}, + {"type":"rate_limit_event","session_id":"SID"}, + {"type":"result","subtype":"success","is_error":false,"num_turns":1,"stop_reason":"end_turn","total_cost_usd":0.2642395,"result":"BANANA","usage":{"input_tokens":4,"output_tokens":5}} +]`; describe("buildClaudeArgs", () => { it("builds the verified non-interactive argv", () => { @@ -32,6 +40,23 @@ describe("buildClaudeArgs", () => { "acceptEdits", ]); }); + + it("appends --model when a model is given", () => { + expect(buildClaudeArgs("hi", undefined, "claude-opus-5")).toEqual([ + "-p", + "hi", + "--output-format", + "json", + "--permission-mode", + "acceptEdits", + "--model", + "claude-opus-5", + ]); + }); + + it("omits --model when no model is given", () => { + expect(buildClaudeArgs("hi", 8)).not.toContain("--model"); + }); }); describe("parseClaudeJson", () => { @@ -78,4 +103,48 @@ describe("parseClaudeJson", () => { const out = parseClaudeJson(AUTH_FAILURE); expect(out.text).toMatch(/Failed to authenticate/); }); + + it("extracts text and cost from the array-form result element", () => { + const out = parseClaudeJson(SUCCESS_ARRAY); + expect(out.ok).toBe(true); + expect(out.text).toBe("BANANA"); + expect(out.costUsd).toBe(0.2642395); + expect(out.error).toBeNull(); + }); + + it("records the cost from the array even when the result reports an error", () => { + const out = parseClaudeJson(`[ + {"type":"system","subtype":"init","cwd":"/tmp/x","session_id":"SID"}, + {"type":"result","subtype":"error_max_turns","is_error":false,"total_cost_usd":0.11,"result":""} + ]`); + expect(out.ok).toBe(false); + expect(out.text).toBe(""); + expect(out.costUsd).toBe(0.11); + expect(out.error).toBe("claude run ended with subtype error_max_turns"); + }); + + it("fails with a named error when the array carries no result element", () => { + const out = parseClaudeJson('[{"type":"system","subtype":"init"},{"type":"assistant"}]'); + expect(out.ok).toBe(false); + expect(out.text).toBe(""); + expect(out.costUsd).toBe(0); + expect(out.error).toBe("could not parse claude json output: no result element in array"); + }); + + it("takes the last result element when an array carries more than one", () => { + const out = parseClaudeJson(`[ + {"type":"system","subtype":"init"}, + {"type":"result","subtype":"error_max_turns","is_error":false,"total_cost_usd":0.01,"result":""}, + {"type":"result","subtype":"success","is_error":false,"total_cost_usd":0.99,"result":"second"} + ]`); + expect(out.ok).toBe(true); + expect(out.text).toBe("second"); + expect(out.costUsd).toBe(0.99); + }); + + it("keeps the whole array as raw", () => { + const out = parseClaudeJson(SUCCESS_ARRAY); + expect(Array.isArray(out.raw)).toBe(true); + expect((out.raw as unknown[]).length).toBe(4); + }); }); diff --git a/src/adapters/claude.ts b/src/adapters/claude.ts index b68a65c..c1bf1b2 100644 --- a/src/adapters/claude.ts +++ b/src/adapters/claude.ts @@ -8,10 +8,15 @@ import type { Adapter, AdapterInput, AdapterOutput } from "./types.js"; * * stdout is a single JSON object with `subtype`, `result`, `session_id`, * `num_turns` and `total_cost_usd`. + * + * Claude Code 3.x emits a JSON *array* of message objects whose last + * `type: "result"` element carries the run result, so the parser finds that + * element before reading `subtype` and `total_cost_usd`. */ -export function buildClaudeArgs(prompt: string, maxTurns?: number): string[] { +export function buildClaudeArgs(prompt: string, maxTurns?: number, model?: string): string[] { const args = ["-p", prompt, "--output-format", "json", "--permission-mode", "acceptEdits"]; if (maxTurns !== undefined) args.push("--max-turns", String(maxTurns)); + if (model !== undefined) args.push("--model", model); return args; } @@ -22,9 +27,9 @@ export function parseClaudeJson(stdout: string): AdapterOutput { } catch { return { ok: false, - text: stdout, + text: "", costUsd: 0, - raw: stdout, + raw: null, error: `could not parse claude json output: ${stdout.slice(0, 200)}`, }; } @@ -33,30 +38,50 @@ export function parseClaudeJson(stdout: string): AdapterOutput { return { ok: false, text: stdout, costUsd: 0, raw: parsed, error: "could not parse claude json output: not an object" }; } - const obj = parsed as Record; - const cost = typeof obj.total_cost_usd === "number" ? obj.total_cost_usd : 0; - const text = typeof obj.result === "string" ? obj.result : ""; - const subtype = typeof obj.subtype === "string" ? obj.subtype : "unknown"; + // Claude Code 3.x emits an array of messages; the run result is the last + // element with `type: "result"`. The legacy single-object form is `result` + // itself. + let result: Record; + if (Array.isArray(parsed)) { + const items = parsed as unknown[]; + let found: Record | undefined; + for (const item of items) { + if (typeof item === "object" && item !== null && (item as Record).type === "result") { + found = item as Record; + } + } + if (found === undefined) { + return { ok: false, text: "", costUsd: 0, raw: parsed, error: "could not parse claude json output: no result element in array" }; + } + result = found; + } else { + result = parsed as Record; + } + + // Cost is harvested even on failure - budget accounting depends on it. + const costUsd = typeof result.total_cost_usd === "number" ? result.total_cost_usd : 0; + const text = typeof result.result === "string" ? result.result : ""; // Claude Code can report `subtype: "success"` while `is_error` is true - an // expired OAuth session comes back exactly that way. Trusting subtype alone // makes an auth failure look like a completed agent run, so check both. - if (obj.is_error === true) { - const reason = typeof obj.terminal_reason === "string" ? obj.terminal_reason : "is_error"; + if (result.is_error === true) { + const reason = typeof result.subtype === "string" ? result.subtype : "is_error"; return { ok: false, text, - costUsd: cost, + costUsd, raw: parsed, error: `claude run reported is_error (${reason}): ${text || "no message"}`, }; } + const subtype = typeof result.subtype === "string" ? result.subtype : "unknown"; if (subtype !== "success") { - return { ok: false, text, costUsd: cost, raw: parsed, error: `claude run ended with subtype ${subtype}` }; + return { ok: false, text, costUsd, raw: parsed, error: `claude run ended with subtype ${subtype}` }; } - return { ok: true, text, costUsd: cost, raw: parsed, error: null }; + return { ok: true, text, costUsd, raw: parsed, error: null }; } export class ClaudeAdapter implements Adapter { @@ -65,7 +90,7 @@ export class ClaudeAdapter implements Adapter { constructor(private readonly bin = "claude") {} async run(input: AdapterInput): Promise { - const args = buildClaudeArgs(input.prompt, input.maxTurns); + const args = buildClaudeArgs(input.prompt, input.maxTurns, input.model); const result = await execa(this.bin, args, { cwd: input.cwd, timeout: input.timeoutSec * 1000, diff --git a/src/adapters/codex.test.ts b/src/adapters/codex.test.ts index 180b7db..d5e2726 100644 --- a/src/adapters/codex.test.ts +++ b/src/adapters/codex.test.ts @@ -35,6 +35,16 @@ describe("buildCodexArgs", () => { it("keeps read-only as the default policy", () => { expect(buildCodexArgs("x", "/repo")).toEqual(buildCodexArgs("x", "/repo", "read-only")); }); + + it("appends --model when a model is given", () => { + const args = buildCodexArgs("x", "/repo", "read-only", "gpt-5.6-sol"); + expect(args).toContain("--model"); + expect(args[args.indexOf("--model") + 1]).toBe("gpt-5.6-sol"); + }); + + it("omits --model when no model is given", () => { + expect(buildCodexArgs("x", "/repo")).not.toContain("--model"); + }); }); describe("parseCodexJsonl", () => { diff --git a/src/adapters/codex.ts b/src/adapters/codex.ts index adbe128..19bbaf9 100644 --- a/src/adapters/codex.ts +++ b/src/adapters/codex.ts @@ -29,10 +29,16 @@ import type { Adapter, AdapterInput, AdapterOutput } from "./types.js"; */ export type CodexSandbox = "read-only" | "workspace-write" | "bypass"; -export function buildCodexArgs(prompt: string, cwd: string, sandbox: CodexSandbox = "read-only"): string[] { +export function buildCodexArgs( + prompt: string, + cwd: string, + sandbox: CodexSandbox = "read-only", + model?: string, +): string[] { const policy = sandbox === "bypass" ? ["--dangerously-bypass-approvals-and-sandbox"] : ["--sandbox", sandbox]; - return ["exec", prompt, "--json", "--skip-git-repo-check", ...policy, "-C", cwd]; + const chosen = model === undefined ? [] : ["--model", model]; + return ["exec", prompt, "--json", "--skip-git-repo-check", ...policy, ...chosen, "-C", cwd]; } /** Pull agent message text out of the several event shapes codex has shipped. */ @@ -128,7 +134,7 @@ export class CodexAdapter implements Adapter { ) {} async run(input: AdapterInput): Promise { - const result = await execa(this.bin, buildCodexArgs(input.prompt, input.cwd, this.sandbox), { + const result = await execa(this.bin, buildCodexArgs(input.prompt, input.cwd, this.sandbox, input.model), { cwd: input.cwd, timeout: input.timeoutSec * 1000, reject: false, diff --git a/src/adapters/enclave.test.ts b/src/adapters/enclave.test.ts new file mode 100644 index 0000000..1b8139a --- /dev/null +++ b/src/adapters/enclave.test.ts @@ -0,0 +1,50 @@ +import { describe, it, expect } from "vitest"; +import { buildEnclavePushArgs, parseEnclavePushJson } from "./enclave.js"; + +const PUSH_SUCCESS = + '{"artifactId":"9a7ad03c-fb5d-4a33-9ba0-7ab615aca938","versionId":"90bcc070-ea84-436b-9067-372da0947525","versionNo":1,"viewUrl":"https://9a7ad03c-fb5d-4a33-9ba0-7ab615aca938.dat-nguyen.me/","uploaded":["index.html"],"skipped":[]}'; + +const PUSH_DRY_RUN = '{"uploaded":["index.html"],"skipped":[]}'; + +describe("buildEnclavePushArgs", () => { + it("builds the push argv in a fixed order", () => { + expect(buildEnclavePushArgs("/tmp/r", "loomgraph run x", "private")).toEqual([ + "push", "/tmp/r", "--title", "loomgraph run x", "--visibility", "private", "--json", + ]); + }); + + it("appends the dry-run flag last", () => { + expect(buildEnclavePushArgs("/tmp/r", "t", "org", { dryRun: true })).toEqual([ + "push", "/tmp/r", "--title", "t", "--visibility", "org", "--json", "--dry-run", + ]); + }); +}); + +describe("parseEnclavePushJson", () => { + it("parses a real push response", () => { + const result = parseEnclavePushJson(PUSH_SUCCESS); + expect(result).toEqual({ + ok: true, + artifactId: "9a7ad03c-fb5d-4a33-9ba0-7ab615aca938", + versionId: "90bcc070-ea84-436b-9067-372da0947525", + versionNo: 1, + viewUrl: "https://9a7ad03c-fb5d-4a33-9ba0-7ab615aca938.dat-nguyen.me/", + uploaded: ["index.html"], + skipped: [], + }); + }); + + it("rejects a dry-run response that carries no artifact id", () => { + expect(parseEnclavePushJson(PUSH_DRY_RUN)).toEqual({ + ok: false, + error: "enclave push returned no artifactId - was this a dry run?", + }); + }); + + it("reports a parse failure for garbage stdout", () => { + const result = parseEnclavePushJson("boom"); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected a failure"); + expect(result.error.startsWith("could not parse enclave json output: ")).toBe(true); + }); +}); diff --git a/src/adapters/enclave.ts b/src/adapters/enclave.ts new file mode 100644 index 0000000..8805bd9 --- /dev/null +++ b/src/adapters/enclave.ts @@ -0,0 +1,63 @@ +/** + * Adapter for the external `enclave` CLI, which hosts a static directory and + * returns a public URL. loomgraph never embeds enclave - it spawns the binary, + * exactly as it spawns `claude`. The pure helpers below are unit-tested against + * real captured stdout. + */ + +export type EnclavePushResult = + | { + ok: true; + artifactId: string; + versionId: string; + versionNo: number; + viewUrl: string; + uploaded: string[]; + skipped: string[]; + } + | { ok: false; error: string }; + +/** + * Build the enclave `push` argv, excluding the binary name. The order is fixed + * and the title is passed as a single argv element so a title containing spaces + * or quotes cannot be reinterpreted by a shell. + */ +export function buildEnclavePushArgs( + dir: string, + title: string, + visibility: "private" | "org", + opts?: { dryRun?: boolean }, +): string[] { + const args = ["push", dir, "--title", title, "--visibility", visibility, "--json"]; + if (opts?.dryRun === true) args.push("--dry-run"); + return args; +} + +/** Parse `enclave push` stdout. Never throws; a bad reply becomes `ok: false`. */ +export function parseEnclavePushJson(stdout: string): EnclavePushResult { + let parsed: unknown; + try { + parsed = JSON.parse(stdout); + } catch { + return { ok: false, error: `could not parse enclave json output: ${stdout.slice(0, 200)}` }; + } + + if (parsed === null || typeof parsed !== "object") { + return { ok: false, error: `could not parse enclave json output: ${stdout.slice(0, 200)}` }; + } + + const obj = parsed as Record; + if (obj.artifactId === undefined || obj.viewUrl === undefined) { + return { ok: false, error: "enclave push returned no artifactId - was this a dry run?" }; + } + + return { + ok: true, + artifactId: String(obj.artifactId), + versionId: typeof obj.versionId === "string" ? obj.versionId : "", + versionNo: typeof obj.versionNo === "number" ? obj.versionNo : 0, + viewUrl: String(obj.viewUrl), + uploaded: Array.isArray(obj.uploaded) ? (obj.uploaded as string[]) : [], + skipped: Array.isArray(obj.skipped) ? (obj.skipped as string[]) : [], + }; +} diff --git a/src/adapters/opencode.test.ts b/src/adapters/opencode.test.ts index 152e8fb..badf55b 100644 --- a/src/adapters/opencode.test.ts +++ b/src/adapters/opencode.test.ts @@ -1,30 +1,107 @@ -import { describe, it, expect } from "vitest"; -import { buildOpencodeArgs, parseOpencodeStdout } from "./opencode.js"; +import { describe, expect, it } from "vitest"; +import { buildOpencodeArgs, parseOpencodeJsonl } from "./opencode.js"; + +/** + * Captured by hand from opencode 1.18.17 on 2026-08-15: + * + * opencode run --format json -m opencode-go/deepseek-v4-flash "Reply with exactly the word MANGO and nothing else." + * + * Session and part ids are replaced; every field name, nesting level and the + * price are exactly as the binary emitted them. + */ +const SUCCESS_JSONL = [ + '{"type":"step_start","timestamp":1786788608757,"sessionID":"ses_TEST","part":{"id":"prt_0","messageID":"msg_TEST","sessionID":"ses_TEST","type":"step-start"}}', + '{"type":"text","timestamp":1786788608758,"sessionID":"ses_TEST","part":{"id":"prt_1","messageID":"msg_TEST","sessionID":"ses_TEST","type":"text","text":"MANGO","time":{"start":1786788610731,"end":1786788610785}}}', + '{"type":"step_finish","timestamp":1786788608759,"sessionID":"ses_TEST","part":{"id":"prt_2","reason":"stop","messageID":"msg_TEST","sessionID":"ses_TEST","type":"step-finish","tokens":{"total":58887,"input":56948,"output":3,"reasoning":16,"cache":{"write":0,"read":1920}},"cost":0.003991708}}', +].join("\n"); + +const TWO_STEPS = [ + '{"type":"text","timestamp":1,"sessionID":"s","part":{"type":"text","text":"first "}}', + '{"type":"step_finish","timestamp":2,"sessionID":"s","part":{"type":"step-finish","cost":0.01}}', + '{"type":"text","timestamp":3,"sessionID":"s","part":{"type":"text","text":"second"}}', + '{"type":"step_finish","timestamp":4,"sessionID":"s","part":{"type":"step-finish","cost":0.02}}', +].join("\n"); describe("buildOpencodeArgs", () => { - it("builds the run argv", () => { - expect(buildOpencodeArgs("summarise the repo")).toEqual(["run", "summarise the repo"]); + it("builds the json-format run argv", () => { + expect(buildOpencodeArgs("do a thing")).toEqual(["run", "--format", "json", "do a thing"]); + }); + + it("appends the model flag when a model is given", () => { + expect(buildOpencodeArgs("do a thing", "opencode-go/deepseek-v4-flash")).toEqual([ + "run", + "--format", + "json", + "-m", + "opencode-go/deepseek-v4-flash", + "do a thing", + ]); + }); + + it("omits the model flag when no model is given", () => { + expect(buildOpencodeArgs("x")).not.toContain("-m"); }); }); -describe("parseOpencodeStdout", () => { - it("returns trimmed plain-text stdout with zero cost", () => { - const out = parseOpencodeStdout(" the answer \n", 0); +describe("parseOpencodeJsonl", () => { + it("extracts text and the reported cost from the event stream", () => { + const out = parseOpencodeJsonl(SUCCESS_JSONL, 0); expect(out.ok).toBe(true); - expect(out.text).toBe("the answer"); - expect(out.costUsd).toBe(0); + expect(out.text).toBe("MANGO"); + expect(out.costUsd).toBe(0.003991708); expect(out.error).toBeNull(); }); - it("fails on a non-zero exit code", () => { - const out = parseOpencodeStdout("", 1); + it("concatenates text across every text event", () => { + expect(parseOpencodeJsonl(TWO_STEPS, 0).text).toBe("first second"); + }); + + it("sums cost across every step_finish event", () => { + expect(parseOpencodeJsonl(TWO_STEPS, 0).costUsd).toBeCloseTo(0.03, 10); + }); + + it("records zero cost when no event reports a price", () => { + const line = '{"type":"text","timestamp":1,"sessionID":"s","part":{"type":"text","text":"hi"}}'; + const out = parseOpencodeJsonl(line, 0); + expect(out.ok).toBe(true); + expect(out.costUsd).toBe(0); + }); + + it("fails on a non-zero exit code but still reports the cost already spent", () => { + const out = parseOpencodeJsonl(SUCCESS_JSONL, 1); + expect(out.ok).toBe(false); + expect(out.costUsd).toBe(0.003991708); + expect(out.error).toBe("opencode exited with code 1"); + }); + + it("fails when the stream carries no recognisable event", () => { + const out = parseOpencodeJsonl("not json at all", 0); expect(out.ok).toBe(false); - expect(out.error).toMatch(/exit/i); + expect(out.costUsd).toBe(0); + expect(out.error).toBe("could not parse opencode json output: no events found"); + }); + + it("ignores an unparseable line but keeps the events around it", () => { + const mixed = [ + "warning: something on stdout that is not json", + '{"type":"text","timestamp":1,"sessionID":"s","part":{"type":"text","text":"kept"}}', + '{"type":"step_finish","timestamp":2,"sessionID":"s","part":{"type":"step-finish","cost":0.5}}', + ].join("\n"); + const out = parseOpencodeJsonl(mixed, 0); + expect(out.ok).toBe(true); + expect(out.text).toBe("kept"); + expect(out.costUsd).toBe(0.5); }); - it("fails on empty output even with a zero exit code", () => { - const out = parseOpencodeStdout(" \n", 0); + it("fails when events parse but produce no text", () => { + const line = '{"type":"step_finish","timestamp":1,"sessionID":"s","part":{"type":"step-finish","cost":0.1}}'; + const out = parseOpencodeJsonl(line, 0); expect(out.ok).toBe(false); - expect(out.error).toMatch(/no output/i); + expect(out.costUsd).toBe(0.1); + expect(out.error).toBe("opencode produced no output"); + }); + + it("keeps the raw stdout so a run can be audited", () => { + expect(parseOpencodeJsonl(SUCCESS_JSONL, 0).raw).toBe(SUCCESS_JSONL); }); }); diff --git a/src/adapters/opencode.ts b/src/adapters/opencode.ts index 3a6ef65..041ab61 100644 --- a/src/adapters/opencode.ts +++ b/src/adapters/opencode.ts @@ -1,26 +1,80 @@ -// Untested against a real opencode binary - not installed in the development environment. import { execa } from "execa"; import type { Adapter, AdapterInput, AdapterOutput } from "./types.js"; -/** `opencode run `, executed in the node's cwd. */ -export function buildOpencodeArgs(prompt: string): string[] { - return ["run", prompt]; +/** + * Verified against opencode 1.18.17 on 2026-08-15: + * + * opencode run --format json [-m ] + * + * `--format json` emits JSONL - one event object per line, shaped + * `{type, timestamp, sessionID, part: {...}}`. A one-word reply produces + * `step_start` -> `text` (text in `part.text`) -> `step_finish` + * (price in `part.cost`, tokens in `part.tokens`). + * + * The default format is used by nothing here on purpose: it writes prose to + * stdout, the model banner to stderr, and carries no price at all. Under + * `--format json` opencode reports a real cost, which is what makes this + * adapter budget-enforceable rather than a permanent 0 in every ceiling. + * + * A model can only be chosen with `-m`. `OPENCODE_MODEL` is ignored, and with + * no flag opencode's own resolution order decides - which is neither stable + * nor inspectable from here. + */ +export function buildOpencodeArgs(prompt: string, model?: string): string[] { + const args = ["run", "--format", "json"]; + if (model !== undefined) args.push("-m", model); + args.push(prompt); + return args; +} + +/** One JSONL event. Only the fields this adapter reads are named. */ +interface OpencodeEvent { + type?: unknown; + part?: { text?: unknown; cost?: unknown } | null; } /** - * OpenCode prints plain text and reports no price, so cost is always 0. Do not - * substitute an estimate. + * Walk the JSONL stream, concatenating `text` events and summing the price + * every `step_finish` reports. + * + * Cost is summed even when the run failed, because budget accounting depends + * on it. It is never estimated: an event that reports no price contributes 0. */ -export function parseOpencodeStdout(stdout: string, exitCode: number | null): AdapterOutput { - const text = stdout.trim(); +export function parseOpencodeJsonl(stdout: string, exitCode: number | null): AdapterOutput { + const events: OpencodeEvent[] = []; + for (const line of stdout.split("\n")) { + const trimmed = line.trim(); + if (trimmed.length === 0) continue; + try { + const parsed: unknown = JSON.parse(trimmed); + // A line opencode wrote that is not an event object is noise, not data. + if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) { + events.push(parsed as OpencodeEvent); + } + } catch { + // Non-JSON lines on stdout are ignored rather than fatal - but if every + // line is noise the caller still gets a failure, below. + } + } + + let text = ""; + let costUsd = 0; + for (const event of events) { + const part = event.part; + if (event.type === "text" && part && typeof part.text === "string") text += part.text; + if (part && typeof part.cost === "number") costUsd += part.cost; + } + if (events.length === 0) { + return { ok: false, text: "", costUsd: 0, raw: stdout, error: "could not parse opencode json output: no events found" }; + } if (exitCode !== 0) { - return { ok: false, text, costUsd: 0, raw: stdout, error: `opencode exited with code ${exitCode ?? "unknown"}` }; + return { ok: false, text, costUsd, raw: stdout, error: `opencode exited with code ${exitCode ?? "unknown"}` }; } - if (text.length === 0) { - return { ok: false, text, costUsd: 0, raw: stdout, error: "opencode produced no output" }; + if (text.trim().length === 0) { + return { ok: false, text, costUsd, raw: stdout, error: "opencode produced no output" }; } - return { ok: true, text, costUsd: 0, raw: stdout, error: null }; + return { ok: true, text, costUsd, raw: stdout, error: null }; } export class OpenCodeAdapter implements Adapter { @@ -29,7 +83,7 @@ export class OpenCodeAdapter implements Adapter { constructor(private readonly bin = "opencode") {} async run(input: AdapterInput): Promise { - const result = await execa(this.bin, buildOpencodeArgs(input.prompt), { + const result = await execa(this.bin, buildOpencodeArgs(input.prompt, input.model), { cwd: input.cwd, timeout: input.timeoutSec * 1000, reject: false, @@ -44,7 +98,7 @@ export class OpenCodeAdapter implements Adapter { return { ok: false, text: stdout, costUsd: 0, raw: { stdout, stderr }, error: `timeout after ${input.timeoutSec}s` }; } - const parsed = parseOpencodeStdout(stdout, result.exitCode ?? null); + const parsed = parseOpencodeJsonl(stdout, result.exitCode ?? null); if (!parsed.ok && stderr.trim()) return { ...parsed, error: `${parsed.error}: ${stderr.trim()}` }; return parsed; } diff --git a/src/adapters/types.ts b/src/adapters/types.ts index 0a1d57d..ee4d138 100644 --- a/src/adapters/types.ts +++ b/src/adapters/types.ts @@ -2,6 +2,8 @@ export interface AdapterInput { prompt: string; cwd: string; maxTurns?: number; + /** Model id passed straight to the CLI. Undefined leaves the CLI's own resolution alone. */ + model?: string; timeoutSec: number; } diff --git a/src/cli.ts b/src/cli.ts index 06016f7..9a1757f 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -2,6 +2,7 @@ import { Command } from "commander"; import { VERSION } from "./index.js"; import { eventsCommand } from "./commands/events.js"; +import { reportCommand } from "./commands/report.js"; import { resumeCommand } from "./commands/resume.js"; import { runCommand } from "./commands/run.js"; import { lsCommand, statusCommand } from "./commands/status.js"; @@ -68,4 +69,14 @@ program .description("print the jsonl audit trail for a run") .action((runId: string, options) => finish(eventsCommand(runId, options))); +program + .command("report") + .argument("") + .option("--out ", "write the html here instead of the run directory") + .option("--publish", "publish the report with the enclave cli") + .option("--title ", "title for the published artifact") + .option("--visibility <visibility>", "private or org", "private") + .description("render a run to a self-contained html report") + .action((runId: string, opts) => finish(reportCommand(runId, opts))); + program.parse(); diff --git a/src/commands/render.ts b/src/commands/render.ts index 1ee48bd..52f15f0 100644 --- a/src/commands/render.ts +++ b/src/commands/render.ts @@ -4,12 +4,21 @@ import type { Graph } from "../core/graph.js"; import type { RunState } from "../core/types.js"; const COST_NOTE = - "note: adapters that do not report a price (codex, opencode, command) record 0.0000 usd - the number is not estimated."; + "note: adapters that do not report a price (codex, command) record 0.0000 usd - the number is not estimated."; function pad(value: string, width: number): string { return value.length >= width ? value : value + " ".repeat(width - value.length); } +export function escapeHtml(value: string): string { + return value + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); +} + function durationSec(startedAt: string, endedAt: string | null): string { if (!startedAt || !endedAt) return "-"; const ms = Date.parse(endedAt) - Date.parse(startedAt); @@ -94,3 +103,69 @@ export function parseVars(pairs: string[]): Record<string, string> { } return vars; } + +function reportDuration(startedAt: string, endedAt: string | null): string { + if (!startedAt || !endedAt) return "-"; + const ms = Date.parse(endedAt) - Date.parse(startedAt); + if (Number.isNaN(ms)) return "-"; + return `${(ms / 1000).toFixed(1)}s`; +} + +export function renderReportHtml(state: RunState, events: LgEvent[]): string { + const esc = escapeHtml; + + const nodeRows: string[] = []; + for (const nodeId of Object.keys(state.nodes)) { + const n = state.nodes[nodeId]!; + nodeRows.push( + `<tr><td>${esc(n.nodeId)}</td><td>${esc(n.status)}</td><td>${esc(String(n.attempts))}</td>` + + `<td>${esc(n.costUsd.toFixed(4))}</td><td>${esc(reportDuration(n.startedAt, n.endedAt))}</td></tr>`, + ); + if (n.error !== null) { + nodeRows.push(`<tr><td colspan="5">${esc(n.error)}</td></tr>`); + } + } + + const eventRows: string[] = []; + for (const e of events) { + eventRows.push( + `<tr><td>${esc(String(e.seq))}</td><td>${esc(e.ts)}</td><td>${esc(e.kind)}</td>` + + `<td>${esc(e.nodeId ?? "")}</td><td>${esc(JSON.stringify(e.data))}</td></tr>`, + ); + } + + const budgetLine = + `${state.spent.usd.toFixed(4)}/${state.budget.maxUsd.toFixed(4)} usd · ` + + `${Math.round(state.spent.wallClockSec)}s/${state.budget.maxWallClockSec}s wall clock · ` + + `${state.spent.nodeRuns}/${state.budget.maxNodeRuns} node runs`; + + return ( + "<!doctype html>" + + "<html>" + + "<head>" + + `<title>loomgraph run ${esc(state.runId)}` + + "" + + "" + + "" + + "

loomgraph run

" + + `

runId: ${esc(state.runId)}
graphName: ${esc(state.graphName)}
status: ${esc(state.status)}
` + + `cwd: ${esc(state.cwd)}
createdAt: ${esc(state.createdAt)}
updatedAt: ${esc(state.updatedAt)}

` + + "

nodes

" + + `` + + nodeRows.join("") + + "
nodestatusattemptscost usdduration
" + + `

${esc(budgetLine)}

` + + `

${esc(COST_NOTE)}

` + + "

events

" + + `` + + eventRows.join("") + + "
seqtskindnodeIddata
" + + "" + + "" + ); +} diff --git a/src/commands/report.test.ts b/src/commands/report.test.ts new file mode 100644 index 0000000..5cee137 --- /dev/null +++ b/src/commands/report.test.ts @@ -0,0 +1,110 @@ +import { describe, it, expect } from "vitest"; +import { escapeHtml, renderReportHtml } from "./render.js"; +import type { LgEvent } from "../core/events.js"; +import type { RunState } from "../core/types.js"; + +function makeState(overrides: Partial = {}): RunState { + return { + runId: "demo-20260814-120000-ab12", + graphName: "demo", + status: "succeeded", + createdAt: "2026-08-14T12:00:00.000Z", + updatedAt: "2026-08-14T12:01:00.000Z", + cwd: "/repo", + vars: { ticket: "LG-1" }, + budget: { maxUsd: 2, maxWallClockSec: 1800, maxNodeRuns: 20 }, + spent: { usd: 0.25, wallClockSec: 60, nodeRuns: 2 }, + nodes: { + greet: { + nodeId: "greet", status: "succeeded", + startedAt: "2026-08-14T12:00:00.000Z", endedAt: "2026-08-14T12:00:30.000Z", + attempts: 1, output: "hello", error: null, costUsd: 0, + }, + }, + completed: ["greet"], + seq: 2, + ...overrides, + }; +} + +const noEvents: LgEvent[] = []; + +describe("escapeHtml", () => { + it("escapes ampersands, angle brackets and quotes", () => { + expect(escapeHtml("a < b")).toBe("a < b"); + expect(escapeHtml("")).toBe("</script>"); + expect(escapeHtml("Tom & \"Jerry\"")).toBe("Tom & "Jerry""); + }); + + it("returns an empty string unchanged", () => { + expect(escapeHtml("")).toBe(""); + }); +}); + +describe("renderReportHtml", () => { + it("renders a complete html document for a run", () => { + const out = renderReportHtml(makeState(), noEvents); + expect(out.startsWith("")).toBe(true); + expect(out).toContain("greet"); + expect(out).toContain("0.0000"); + }); + + it("escapes a hostile graph name so no raw tag reaches the document", () => { + const out = renderReportHtml(makeState({ graphName: "" }), noEvents); + expect(out).toContain("<img src=x onerror=alert(1)>"); + expect(out).not.toContain(" { + const state = makeState({ + nodes: { + greet: { + nodeId: "greet", status: "failed", + startedAt: "2026-08-14T12:00:00.000Z", endedAt: "2026-08-14T12:00:30.000Z", + attempts: 1, output: "", error: "boom ", costUsd: 0, + }, + }, + }); + expect(renderReportHtml(state, noEvents)).toContain("boom </td>"); + }); + + it("renders the budget line and the cost note", () => { + const out = renderReportHtml(makeState(), noEvents); + expect(out).toContain("0.2500/2.0000 usd · 60s/1800s wall clock · 2/20 node runs"); + expect(out).toContain( + "note: adapters that do not report a price (codex, command) record 0.0000 usd - the number is not estimated.", + ); + }); + + it("is deterministic for identical inputs", () => { + const state = makeState(); + expect(renderReportHtml(state, noEvents)).toBe(renderReportHtml(state, noEvents)); + }); + + it("renders a run with no nodes and no events", () => { + const out = renderReportHtml(makeState({ nodes: {}, completed: [], status: "pending" }), noEvents); + expect(out.startsWith("")).toBe(true); + expect(out).toContain( + "note: adapters that do not report a price (codex, command) record 0.0000 usd - the number is not estimated.", + ); + }); + + it("embeds no external resource", () => { + const out = renderReportHtml(makeState(), noEvents); + expect(out).not.toContain(" { + const events: LgEvent[] = [ + { ts: "2026-08-14T12:00:00.000Z", runId: "demo-20260814-120000-ab12", seq: 0, kind: "node_started", nodeId: "greet", data: {} }, + { ts: "2026-08-14T12:00:30.000Z", runId: "demo-20260814-120000-ab12", seq: 1, kind: "node_finished", nodeId: "greet", data: { ok: true } }, + { ts: "2026-08-14T12:01:00.000Z", runId: "demo-20260814-120000-ab12", seq: 2, kind: "run_finished", data: {} }, + ]; + const out = renderReportHtml(makeState(), events); + expect(out).toContain("node_started"); + expect(out).toContain("node_finished"); + expect(out).toContain("run_finished"); + }); +}); diff --git a/src/commands/report.ts b/src/commands/report.ts new file mode 100644 index 0000000..7dad981 --- /dev/null +++ b/src/commands/report.ts @@ -0,0 +1,61 @@ +import { execa } from "execa"; +import { dirname, join, resolve } from "node:path"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { buildEnclavePushArgs, parseEnclavePushJson } from "../adapters/enclave.js"; +import { openLog, openStore, runsDir } from "./context.js"; +import { renderReportHtml } from "./render.js"; + +export interface ReportOptions { + out?: string; + publish?: boolean; + title?: string; + visibility?: "private" | "org"; +} + +export async function reportCommand(runId: string, opts: ReportOptions = {}): Promise { + const store = openStore(); + const state = store.load(runId); + if (!state) { + console.error(`run not found: ${runId}`); + return 1; + } + + const events = openLog().read(runId); + const html = renderReportHtml(state, events); + + const htmlPath = resolve(opts.out ?? join(runsDir(), runId, "report", "index.html")); + mkdirSync(dirname(htmlPath), { recursive: true }); + writeFileSync(htmlPath, html, "utf8"); + console.log(htmlPath); + + if (opts.publish !== true) return 0; + + const args = buildEnclavePushArgs( + dirname(htmlPath), + opts.title ?? `loomgraph run ${runId}`, + opts.visibility ?? "private", + ); + + const result = await execa("enclave", args, { reject: false }); + + if (result.failed && result.code === "ENOENT") { + console.error("enclave not found on PATH - the report was written but not published"); + return 0; + } + + if (result.exitCode !== 0) { + const stderr = typeof result.stderr === "string" ? result.stderr.trim() : ""; + if (stderr) console.error(stderr); + return 2; + } + + const stdout = typeof result.stdout === "string" ? result.stdout : ""; + const parsed = parseEnclavePushJson(stdout); + if (!parsed.ok) { + console.error(parsed.error); + return 2; + } + + console.log(parsed.viewUrl); + return 0; +} diff --git a/src/core/engine.test.ts b/src/core/engine.test.ts index 757808b..d735f81 100644 --- a/src/core/engine.test.ts +++ b/src/core/engine.test.ts @@ -6,6 +6,7 @@ import { parseGraph } from "./graph.js"; import { CheckpointStore } from "./store.js"; import { EventLog } from "./events.js"; import { execute, newRunState, interpolate, readySet, EngineError } from "./engine.js"; +import * as engine from "./engine.js"; import type { EngineDeps } from "./engine.js"; import type { Adapter, AdapterInput, AdapterOutput } from "../adapters/types.js"; import type { RunState } from "./types.js"; @@ -54,6 +55,15 @@ edges: - { from: c, to: END } `; +const HYPHEN = ` +name: hyphen +budget: { maxUsd: 10, maxWallClockSec: 600, maxNodeRuns: 20 } +nodes: + my-node: { type: command, run: "echo HI" } +edges: + - { from: my-node, to: END } +`; + const FANOUT = ` name: fanout budget: { maxUsd: 10, maxWallClockSec: 600, maxNodeRuns: 20 } @@ -83,6 +93,39 @@ describe("interpolate", () => { it("throws naming an unresolvable reference", () => { expect(() => interpolate("{{vars.nope}}", start(LINEAR))).toThrow(/nope/); }); + + it("resolves a node output whose id contains a hyphen", () => { + const state = start(HYPHEN); + state.nodes["my-node"] = { + nodeId: "my-node", status: "succeeded", startedAt: "", endedAt: null, + attempts: 1, output: "HI", error: null, costUsd: 0, + }; + expect(interpolate("[{{nodes.my-node.output}}]", state)).toBe("[HI]"); + }); + + it("throws naming an unresolvable hyphenated reference", () => { + expect(() => interpolate("{{nodes.no-such.output}}", start(LINEAR))).toThrow( + new EngineError('unknown template reference "{{nodes.no-such.output}}"'), + ); + }); + + it("honours whitespace padding around a hyphenated reference", () => { + const state = start(HYPHEN); + state.nodes["my-node"] = { + nodeId: "my-node", status: "succeeded", startedAt: "", endedAt: null, + attempts: 1, output: "HI", error: null, costUsd: 0, + }; + expect(interpolate("{{ nodes.my-node.output }}", state)).toBe("HI"); + }); + + it("leaves hyphens in surrounding literal text untouched", () => { + const state = start(HYPHEN); + state.nodes["my-node"] = { + nodeId: "my-node", status: "succeeded", startedAt: "", endedAt: null, + attempts: 1, output: "HI", error: null, costUsd: 0, + }; + expect(interpolate("a-b {{nodes.my-node.output}} c-d", state)).toBe("a-b HI c-d"); + }); }); describe("readySet", () => { @@ -97,6 +140,35 @@ describe("readySet", () => { }); }); +describe("checkCommandExpectations", () => { + const check = (engine as unknown as { + checkCommandExpectations: ( + node: { expect?: string; expectNonEmpty?: boolean }, + text: string, + ) => string | null; + }).checkCommandExpectations; + + it("returns null when no expectations are declared", () => { + expect(check({}, "anything")).toBeNull(); + }); + + it("fails when expectNonEmpty is set and the output is blank", () => { + expect(check({ expectNonEmpty: true }, " \n ")).toBe("command produced no output but expectNonEmpty is set"); + }); + + it("fails when the expect substring is absent", () => { + expect(check({ expect: "PASS" }, "FAILED")).toBe("command output did not contain the expected string: PASS"); + }); + + it("passes when the expect substring is present", () => { + expect(check({ expect: "PASS" }, "the result is PASS today")).toBeNull(); + }); + + it("prefers the blank-output message when both expectations fail", () => { + expect(check({ expect: "PASS", expectNonEmpty: true }, "")).toBe("command produced no output but expectNonEmpty is set"); + }); +}); + describe("execute", () => { it("runs a linear graph in order and succeeds", async () => { const order: string[] = []; @@ -330,6 +402,24 @@ edges: expect(final.nodes.v!.error).toMatch(/PASS/); }); + it("fails a command node whose output does not meet its expectations", async () => { + const src = ` +name: expect +budget: { maxUsd: 10, maxWallClockSec: 600, maxNodeRuns: 20 } +nodes: + a: { type: command, run: "npm run lint --if-present", expectNonEmpty: true } +edges: + - { from: a, to: END } +`; + const registry = { command: stub("command", () => ok("")) }; + + const final = await execute(parseGraph(src), start(src), deps(registry)); + + expect(final.status).toBe("failed"); + expect(final.nodes.a!.status).toBe("failed"); + expect(final.nodes.a!.error).toBe("command produced no output but expectNonEmpty is set"); + }); + it("interpolates vars and upstream node output into an agent prompt", async () => { const src = ` name: templated @@ -377,4 +467,233 @@ edges: expect(final.status).toBe("failed"); expect(final.nodes.c).toBeUndefined(); }); + + it("fails a run whose final spend exceeds the usd ceiling", async () => { + const src = ` +name: overshoot +budget: { maxUsd: 0.30, maxWallClockSec: 600, maxNodeRuns: 20 } +nodes: + a: { type: command, run: "echo a" } + b: { type: command, run: "echo b" } +edges: + - { from: a, to: b } + - { from: b, to: END } +`; + const registry = { + command: stub("command", (i) => (i.prompt === "echo a" ? ok("a", 0.2726) : ok("b", 0.2645))), + }; + + const final = await execute(parseGraph(src), start(src), deps(registry)); + + expect(final.status).toBe("failed"); + }); + + it("emits budget_exceeded naming the ceiling when the last batch overshoots", async () => { + const src = ` +name: overshoot-event +budget: { maxUsd: 0.30, maxWallClockSec: 600, maxNodeRuns: 20 } +nodes: + a: { type: command, run: "echo a" } + b: { type: command, run: "echo b" } +edges: + - { from: a, to: b } + - { from: b, to: END } +`; + const registry = { + command: stub("command", (i) => (i.prompt === "echo a" ? ok("a", 0.2726) : ok("b", 0.2645))), + }; + + await execute(parseGraph(src), start(src), deps(registry)); + + const exceeded = log.read("run1").filter((e) => e.kind === "budget_exceeded"); + expect(exceeded).toHaveLength(1); + expect(String(exceeded[0]!.data.reason)).toMatch(/maxUsd/); + }); + + it("keeps every completed node result when the final budget check fails", async () => { + const src = ` +name: overshoot-keep +budget: { maxUsd: 0.30, maxWallClockSec: 600, maxNodeRuns: 20 } +nodes: + a: { type: command, run: "echo a" } + b: { type: command, run: "echo b" } +edges: + - { from: a, to: b } + - { from: b, to: END } +`; + const registry = { + command: stub("command", (i) => (i.prompt === "echo a" ? ok("a", 0.2726) : ok("b", 0.2645))), + }; + + const final = await execute(parseGraph(src), start(src), deps(registry)); + + expect(final.status).toBe("failed"); + expect(final.nodes.a!.status).toBe("succeeded"); + expect(final.nodes.a!.costUsd).toBe(0.2726); + expect(final.nodes.b!.status).toBe("succeeded"); + expect(final.nodes.b!.costUsd).toBe(0.2645); + }); + + it("succeeds when the final spend stays under the ceiling", async () => { + const src = ` +name: under +budget: { maxUsd: 0.30, maxWallClockSec: 600, maxNodeRuns: 20 } +nodes: + a: { type: command, run: "echo a" } + b: { type: command, run: "echo b" } +edges: + - { from: a, to: b } + - { from: b, to: END } +`; + const registry = { + command: stub("command", (i) => (i.prompt === "echo a" ? ok("a", 0.2) : ok("b", 0.0999))), + }; + + const final = await execute(parseGraph(src), start(src), deps(registry)); + + expect(final.status).toBe("succeeded"); + expect(log.read("run1").filter((e) => e.kind === "budget_exceeded")).toHaveLength(0); + }); + + it("treats a final spend exactly at the usd ceiling as a breach", async () => { + const src = ` +name: exactly +budget: { maxUsd: 0.30, maxWallClockSec: 600, maxNodeRuns: 20 } +nodes: + a: { type: command, run: "echo a" } + b: { type: command, run: "echo b" } +edges: + - { from: a, to: b } + - { from: b, to: END } +`; + const registry = { + command: stub("command", (i) => (i.prompt === "echo a" ? ok("a", 0.15) : ok("b", 0.15))), + }; + + const final = await execute(parseGraph(src), start(src), deps(registry)); + + expect(final.status).toBe("failed"); + }); + + it("fails a run whose final node-run count exceeds maxNodeRuns", async () => { + const src = ` +name: runcap +budget: { maxUsd: 10, maxWallClockSec: 600, maxNodeRuns: 2 } +nodes: + a: { type: command, run: "echo a" } + b: { type: command, run: "echo b" } +edges: + - { from: a, to: b } + - { from: b, to: END } +`; + const registry = { command: stub("command", (i) => ok(i.prompt)) }; + + const final = await execute(parseGraph(src), start(src), deps(registry)); + + expect(final.status).toBe("failed"); + const exceeded = log.read("run1").filter((e) => e.kind === "budget_exceeded"); + expect(exceeded).toHaveLength(1); + expect(String(exceeded[0]!.data.reason)).toMatch(/maxNodeRuns/); + }); + + it("does not emit budget_exceeded when a run fails because a node failed", async () => { + const src = ` +name: nodefail +budget: { maxUsd: 10, maxWallClockSec: 600, maxNodeRuns: 20 } +nodes: + a: { type: command, run: "echo a" } + b: { type: command, run: "echo b" } +edges: + - { from: a, to: b } + - { from: b, to: END } +`; + const registry = { + command: stub("command", (i) => (i.prompt === "echo a" ? ok("a") : bad("boom"))), + }; + + const final = await execute(parseGraph(src), start(src), deps(registry)); + + expect(final.status).toBe("failed"); + expect(log.read("run1").filter((e) => e.kind === "budget_exceeded")).toHaveLength(0); + }); + + it("does not emit budget_exceeded when a run pauses on a human node", async () => { + const src = ` +name: pause +budget: { maxUsd: 10, maxWallClockSec: 600, maxNodeRuns: 20 } +nodes: + a: { type: command, run: "echo a" } + h: { type: human, question: "Ship it?" } +edges: + - { from: a, to: h } + - { from: h, to: END } +`; + const registry = { command: stub("command", (i) => ok(i.prompt)) }; + + const final = await execute(parseGraph(src), start(src), deps(registry)); + + expect(final.status).toBe("paused"); + expect(log.read("run1").filter((e) => e.kind === "budget_exceeded")).toHaveLength(0); + }); +}); + +describe("model passthrough", () => { + const WITH_MODEL = ` +name: m +budget: + maxUsd: 1 + maxWallClockSec: 60 + maxNodeRuns: 5 +nodes: + a: + type: agent + adapter: claude + prompt: "hi" + model: "claude-opus-5" +edges: + - from: a + to: END +`; + + const WITHOUT_MODEL = ` +name: m +budget: + maxUsd: 1 + maxWallClockSec: 60 + maxNodeRuns: 5 +nodes: + a: + type: agent + adapter: claude + prompt: "hi" +edges: + - from: a + to: END +`; + + it("passes a node's model through to the adapter", async () => { + let seen: AdapterInput | null = null; + const registry = { + claude: stub("claude", (input) => { + seen = input; + return ok("done"); + }), + }; + await execute(parseGraph(WITH_MODEL), start(WITH_MODEL), deps(registry)); + expect(seen).not.toBeNull(); + expect(seen!.model).toBe("claude-opus-5"); + }); + + it("leaves model undefined on the adapter input when the node declares none", async () => { + let seen: AdapterInput | null = null; + const registry = { + claude: stub("claude", (input) => { + seen = input; + return ok("done"); + }), + }; + await execute(parseGraph(WITHOUT_MODEL), start(WITHOUT_MODEL), deps(registry)); + expect(seen).not.toBeNull(); + expect(seen!.model).toBeUndefined(); + }); }); diff --git a/src/core/engine.ts b/src/core/engine.ts index f715733..93cb262 100644 --- a/src/core/engine.ts +++ b/src/core/engine.ts @@ -54,7 +54,7 @@ export function newRunState( /** Resolve `{{vars.x}}`, `{{x}}` and `{{nodes..output}}` against the run state. */ export function interpolate(template: string, state: RunState): string { - return template.replace(/\{\{\s*([A-Za-z0-9_.]+)\s*\}\}/g, (_match, ref: string) => { + return template.replace(/\{\{\s*([A-Za-z0-9_.\-]+)\s*\}\}/g, (_match, ref: string) => { const parts = ref.split("."); let value: unknown; @@ -106,6 +106,24 @@ export function readySet(graph: Graph, state: RunState): string[] { }); } +/** + * Assert a command node's opt-in expectations against its stdout. Returns null + * when the node satisfies them, otherwise the failure message. A shell command + * that exits 0 having done nothing is not a passing check. + */ +export function checkCommandExpectations( + node: { expect?: string; expectNonEmpty?: boolean }, + text: string, +): string | null { + if (node.expectNonEmpty === true && text.trim() === "") { + return "command produced no output but expectNonEmpty is set"; + } + if (typeof node.expect === "string" && node.expect !== "" && text.includes(node.expect) === false) { + return `command output did not contain the expected string: ${node.expect}`; + } + return null; +} + /** * The batches the scheduler would dispatch, in order. Pure - used by * `lg run --dry-run`, which must not spawn anything. @@ -205,7 +223,14 @@ export async function execute(graph: Graph, initial: RunState, deps: EngineDeps) const cwd = nodeCwd(def, state); if (def.type === "command") { - return adapter.run({ prompt: interpolate(def.run, state), cwd, timeoutSec: def.timeoutSec }); + const out = await adapter.run({ prompt: interpolate(def.run, state), cwd, timeoutSec: def.timeoutSec }); + if (out.ok) { + const expectationError = checkCommandExpectations(def, out.text); + if (expectationError !== null) { + return { ...out, ok: false, error: expectationError }; + } + } + return out; } if (def.type === "human") { throw new EngineError(`human node "${id}" cannot be dispatched to an adapter`); @@ -215,6 +240,7 @@ export async function execute(graph: Graph, initial: RunState, deps: EngineDeps) prompt: interpolate(def.prompt, state), cwd, maxTurns: def.maxTurns, + model: def.model, timeoutSec: def.timeoutSec, }); @@ -243,7 +269,14 @@ export async function execute(graph: Graph, initial: RunState, deps: EngineDeps) }; while (true) { - if (endReached(graph, state)) return finish("succeeded", null); + if (endReached(graph, state)) { + const check = checkBudget(state); + if (!check.ok) { + emit({ kind: "budget_exceeded", data: { reason: check.reason, spent: state.spent, budget: state.budget } }); + return finish("failed", check.reason); + } + return finish("succeeded", null); + } const ready = readySet(graph, state); if (ready.length === 0) { diff --git a/src/core/graph.test.ts b/src/core/graph.test.ts index f75d929..e7841f6 100644 --- a/src/core/graph.test.ts +++ b/src/core/graph.test.ts @@ -182,6 +182,74 @@ edges: expect(() => parseGraph(src)).toThrow(/maxUsd/); }); + it("accepts an agent node with an explicit model", () => { + const src = graph(`nodes: + a: + type: agent + adapter: opencode + prompt: "hi" + model: "opencode-go/deepseek-v4-flash" +edges: + - from: a + to: END +`); + const parsed = parseGraph(src); + const node = parsed.nodes["a"]; + expect(node?.type).toBe("agent"); + if (node?.type === "agent") expect(node.model).toBe("opencode-go/deepseek-v4-flash"); + }); + + it("accepts a verifier node with an explicit model", () => { + const src = graph(`nodes: + a: + type: verifier + adapter: codex + prompt: "hi" + pass: "PASS" + model: "gpt-5.6-sol" +edges: + - from: a + to: END +`); + const node = parseGraph(src).nodes["a"]; + if (node?.type === "verifier") expect(node.model).toBe("gpt-5.6-sol"); + else throw new Error("expected a verifier node"); + }); + + it("leaves model undefined when a node does not declare one", () => { + const node = parseGraph(FULL_EXAMPLE).nodes["reproduce"]; + if (node?.type === "agent") expect(node.model).toBeUndefined(); + else throw new Error("expected an agent node"); + }); + + it("rejects an agent node whose model is an empty string", () => { + const src = graph(`nodes: + a: + type: agent + adapter: claude + prompt: "hi" + model: "" +edges: + - from: a + to: END +`); + expect(() => parseGraph(src)).toThrow(/a/); + }); + + it("rejects a command node that declares a model", () => { + const src = graph(`nodes: + a: + type: command + run: "echo hi" + model: "claude-opus-5" +edges: + - from: a + to: END +`); + expect(() => parseGraph(src)).toThrow(/a/); + expect(() => parseGraph(src)).toThrow(/model/); + }); + it("rejects an agent node with an unknown adapter", () => { const src = graph(`nodes: a: @@ -233,6 +301,38 @@ edges: expect(() => parseGraph(src)).toThrow(/run/); }); + it("accepts a command node with expect and expectNonEmpty", () => { + const src = graph(`nodes: + a: + type: command + run: "npm test" + expect: "0 failed" + expectNonEmpty: true +edges: + - from: a + to: END +`); + expect(parseGraph(src).nodes.a).toMatchObject({ + type: "command", + run: "npm test", + expect: "0 failed", + expectNonEmpty: true, + }); + }); + + it("rejects a command node whose expect is an empty string", () => { + const src = graph(`nodes: + a: + type: command + run: "npm test" + expect: "" +edges: + - from: a + to: END +`); + expect(() => parseGraph(src)).toThrow(/node "a"/); + }); + it("rejects a human node without a question", () => { const src = graph(`nodes: a: @@ -285,4 +385,77 @@ edges: `); expect(() => parseGraph(src)).toThrow(/END/); }); + + it("accepts a node id containing a hyphen", () => { + const src = graph(`nodes: + my-node: + type: command + run: "true" +edges: + - from: my-node + to: END +`); + expect(parseGraph(src).nodes["my-node"]).toMatchObject({ type: "command", run: "true" }); + }); + + it("rejects a node id containing a dot", () => { + const src = graph(`nodes: + "my.node": + type: command + run: "true" +edges: + - from: "my.node" + to: END +`); + expect(() => parseGraph(src)).toThrow(/my\.node/); + }); + + it("rejects a node id containing a space", () => { + const src = graph(`nodes: + "my node": + type: command + run: "true" +edges: + - from: "my node" + to: END +`); + expect(() => parseGraph(src)).toThrow(/must match/); + }); + + it("rejects a node id longer than 64 characters", () => { + const long = "a".repeat(65); + const src = graph(`nodes: + ${long}: + type: command + run: "true" +edges: + - from: ${long} + to: END +`); + expect(() => parseGraph(src)).toThrow(/must match/); + }); + + it("rejects a node id containing a path separator", () => { + const src = graph(`nodes: + "../escape": + type: command + run: "true" +edges: + - from: "../escape" + to: END +`); + expect(() => parseGraph(src)).toThrow(/must match/); + }); + + it("still reports END as reserved rather than as a bad character", () => { + const src = graph(`nodes: + END: + type: command + run: "true" +edges: + - from: END + to: END +`); + expect(() => parseGraph(src)).toThrow(/reserved/); + }); }); diff --git a/src/core/graph.ts b/src/core/graph.ts index bf6852a..32222d0 100644 --- a/src/core/graph.ts +++ b/src/core/graph.ts @@ -35,6 +35,7 @@ const agentNode = z.object({ adapter: z.enum(ADAPTER_NAMES), prompt: z.string().min(1), maxTurns: z.number().int().positive().default(20), + model: z.string().min(1).optional(), ...common, }); @@ -44,12 +45,15 @@ const verifierNode = z.object({ prompt: z.string().min(1), pass: z.string().min(1), maxTurns: z.number().int().positive().default(20), + model: z.string().min(1).optional(), ...common, }); const commandNode = z.object({ type: z.literal("command"), run: z.string().min(1), + expect: z.string().min(1).optional(), + expectNonEmpty: z.boolean().optional(), ...common, }); @@ -139,6 +143,9 @@ export function parseGraph(source: string, sourceName = "graph"): Graph { const nodes: Record = {}; for (const [id, value] of Object.entries(rawNodes as Record)) { if (id === END) fail(`node id "${END}" is reserved`); + if (!/^[A-Za-z0-9_-]{1,64}$/.test(id)) { + fail(`node id "${id}" must match [A-Za-z0-9_-] and be 1-64 characters`); + } if (value === null || typeof value !== "object" || Array.isArray(value)) { fail(`node "${id}" must be a mapping`); } @@ -146,6 +153,12 @@ export function parseGraph(source: string, sourceName = "graph"): Graph { if (typeof type !== "string" || !(NODE_TYPES as readonly string[]).includes(type)) { fail(`node "${id}" has unknown type "${String(type)}" - valid types are ${NODE_TYPES.join(", ")}`); } + // zod strips unknown keys, so a `model` on a command or human node would + // vanish silently rather than fail. Only the two node types that dispatch a + // prompt to an agent CLI can carry one. + if ((value as Record).model !== undefined && type !== "agent" && type !== "verifier") { + fail(`node "${id}" of type "${type}" cannot declare a model - only agent and verifier nodes dispatch to an adapter`); + } const parsed = nodeSchema.safeParse(value); if (!parsed.success) fail(issuesToMessage(`node "${id}"`, parsed.error)); nodes[id] = parsed.data; diff --git a/tsup.config.ts b/tsup.config.ts index 0be806a..990e26d 100644 --- a/tsup.config.ts +++ b/tsup.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from "tsup"; export default defineConfig({ entry: ["src/index.ts", "src/cli.ts"], format: ["esm"], - target: "node20", + target: "node22", platform: "node", clean: true, sourcemap: true,