From 50939d0f9127b4914b1114a6d43a7505a78bdfc6 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 9 Jun 2026 03:08:54 +0000 Subject: [PATCH] feat(headless): add --output=json to opencli run + JSON schema contract eval Implements the headless --output=json sub-item of D2 (#234): - src/cli/json-output.ts: defines JsonOutputEvent (stable public schema) and toJsonLine() which serialises every AgentEvent to a newline-terminated JSON line, stripping the provider-internal thoughtSignature from tool_call. - src/cli/index.ts: adds --output to opencli run; in json mode every AgentEvent is written as an NDJSON line to stdout instead of the human-readable text rendering. - src/eval/headless/json-schema.test.ts: contract eval with 33 tests asserting parseable JSON, newline termination, required fields per event type, thoughtSignature exclusion, and exhaustive schema coverage. Part of #234 Co-Authored-By: Claude Sonnet 4.6 --- src/cli/index.ts | 27 ++++-- src/cli/json-output.ts | 46 ++++++++++ src/eval/headless/json-schema.test.ts | 121 ++++++++++++++++++++++++++ 3 files changed, 189 insertions(+), 5 deletions(-) create mode 100644 src/cli/json-output.ts create mode 100644 src/eval/headless/json-schema.test.ts diff --git a/src/cli/index.ts b/src/cli/index.ts index aeba1de..575affb 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -10,6 +10,7 @@ import { resolveApiKey } from "./keys.js"; import { runRepl } from "./repl.js"; import { createConfirmFn } from "./confirm.js"; import { printError, printInfo } from "./renderer.js"; +import { toJsonLine } from "./json-output.js"; import type { ObservabilityEvent } from "../core/observability.js"; import { createSandboxRunner } from "../tools/exec/sandbox/index.js"; import type { SandboxMode } from "../tools/exec/sandbox/types.js"; @@ -97,7 +98,13 @@ program .option("--provider ", "Override provider detection: gemini | anthropic | openai") .option("--base-url ", "Custom base URL for proxy or local inference (e.g. LiteLLM)") .option("--temperature ", "LLM temperature (use 0 for determinism)", parseTemperature) + .option("--output ", "Output format: text | json (default: text)") .action(async (prompt: string, opts) => { + const output = opts.output as string | undefined; + if (output !== undefined && output !== "text" && output !== "json") { + printError(`Invalid --output value '${output}'. Valid values: text, json`); + process.exit(1); + } await runSingle( prompt, opts.model, @@ -109,6 +116,7 @@ program opts.provider as string | undefined, opts.baseUrl as string | undefined, opts.temperature as number | undefined, + output as "text" | "json" | undefined, ); }); @@ -224,6 +232,7 @@ async function runSingle( providerOverride?: string, baseUrlOverride?: string, temperature?: number, + outputFormat?: "text" | "json", ): Promise { const { agent, mcpManager } = await createAgent( modelOverride, @@ -243,15 +252,23 @@ async function runSingle( } // no confirmFn → executor auto-denies tools that require confirmation + const jsonMode = outputFormat === "json"; + const stream = async (input: string, mode: "react" | "plan") => { let text = ""; for await (const event of agent.run(input, mode)) { - if (event.type === "text") { - process.stdout.write(event.text); - text += event.text; + if (jsonMode) { + const line = toJsonLine(event); + if (line) process.stdout.write(line); + if (event.type === "text") text += event.text; + } else { + if (event.type === "text") { + process.stdout.write(event.text); + text += event.text; + } + if (event.type === "error") process.stderr.write(`Error: ${event.message}\n`); + if (event.type === "done") process.stdout.write("\n"); } - if (event.type === "error") process.stderr.write(`Error: ${event.message}\n`); - if (event.type === "done") process.stdout.write("\n"); } return text; }; diff --git a/src/cli/json-output.ts b/src/cli/json-output.ts new file mode 100644 index 0000000..3d30a00 --- /dev/null +++ b/src/cli/json-output.ts @@ -0,0 +1,46 @@ +import type { AgentEvent } from "../core/agent.js"; + +/** Stable public schema for each NDJSON line emitted by --output=json. */ +export type JsonOutputEvent = + | { type: "text"; text: string } + | { type: "tool_call"; name: string; args: Record } + | { type: "tool_result"; name: string; result: string } + | { type: "skill_activated"; name: string } + | { type: "error"; message: string } + | { type: "notice"; message: string } + | { type: "done" }; + +/** + * Converts an AgentEvent to a newline-terminated JSON line for --output=json. + * Returns null for events with no public representation (forward-compat guard). + * Strips thoughtSignature from tool_call — it is a provider-internal Gemini field. + */ +export function toJsonLine(event: AgentEvent): string | null { + let out: JsonOutputEvent; + switch (event.type) { + case "text": + out = { type: "text", text: event.text }; + break; + case "tool_call": + out = { type: "tool_call", name: event.name, args: event.args }; + break; + case "tool_result": + out = { type: "tool_result", name: event.name, result: event.result }; + break; + case "skill_activated": + out = { type: "skill_activated", name: event.name }; + break; + case "error": + out = { type: "error", message: event.message }; + break; + case "notice": + out = { type: "notice", message: event.message }; + break; + case "done": + out = { type: "done" }; + break; + default: + return null; + } + return JSON.stringify(out) + "\n"; +} diff --git a/src/eval/headless/json-schema.test.ts b/src/eval/headless/json-schema.test.ts new file mode 100644 index 0000000..3c0f894 --- /dev/null +++ b/src/eval/headless/json-schema.test.ts @@ -0,0 +1,121 @@ +import { describe, it, expect } from "vitest"; +import type { AgentEvent } from "../../core/agent.js"; +import { toJsonLine, type JsonOutputEvent } from "../../cli/json-output.js"; + +const ALL_EVENT_CASES: AgentEvent[] = [ + { type: "text", text: "hello world" }, + { type: "tool_call", name: "bash", args: { command: "ls" } }, + { + type: "tool_call", + name: "edit", + args: { file_path: "foo.ts", old_string: "a", new_string: "b" }, + thoughtSignature: "sig-abc", + }, + { type: "tool_result", name: "bash", result: "file.ts\n" }, + { type: "skill_activated", name: "review" }, + { type: "error", message: "Maximum iterations reached." }, + { type: "notice", message: "Context compacted." }, + { type: "done" }, +]; + +describe("json-schema contract — parseable NDJSON", () => { + it.each(ALL_EVENT_CASES)("$type line is valid JSON", (event) => { + const line = toJsonLine(event); + expect(line).not.toBeNull(); + expect(() => JSON.parse(line!.trimEnd())).not.toThrow(); + }); + + it.each(ALL_EVENT_CASES)("$type line is newline-terminated", (event) => { + const line = toJsonLine(event); + expect(line!.endsWith("\n")).toBe(true); + }); +}); + +describe("json-schema contract — type field always present", () => { + it.each(ALL_EVENT_CASES)("$type has type field", (event) => { + const parsed: JsonOutputEvent = JSON.parse(toJsonLine(event)!); + expect(typeof parsed.type).toBe("string"); + expect(parsed.type).toBe(event.type); + }); +}); + +describe("json-schema contract — required fields per event type", () => { + it("text: { type, text: string }", () => { + const parsed = JSON.parse(toJsonLine({ type: "text", text: "hi" })!) as JsonOutputEvent; + expect(parsed).toEqual({ type: "text", text: "hi" }); + }); + + it("tool_call: { type, name: string, args: object }", () => { + const parsed = JSON.parse( + toJsonLine({ type: "tool_call", name: "bash", args: { command: "ls" } })!, + ) as JsonOutputEvent; + expect(parsed).toEqual({ type: "tool_call", name: "bash", args: { command: "ls" } }); + expect(typeof (parsed as { name: string }).name).toBe("string"); + expect(typeof (parsed as { args: unknown }).args).toBe("object"); + }); + + it("tool_call strips thoughtSignature (provider-internal field)", () => { + const parsed = JSON.parse( + toJsonLine({ + type: "tool_call", + name: "read", + args: { file_path: "a.ts" }, + thoughtSignature: "secret-sig", + })!, + ); + expect(parsed).not.toHaveProperty("thoughtSignature"); + }); + + it("tool_result: { type, name: string, result: string }", () => { + const parsed = JSON.parse( + toJsonLine({ type: "tool_result", name: "read", result: "contents\n" })!, + ) as JsonOutputEvent; + expect(parsed).toEqual({ type: "tool_result", name: "read", result: "contents\n" }); + }); + + it("skill_activated: { type, name: string }", () => { + const parsed = JSON.parse( + toJsonLine({ type: "skill_activated", name: "review" })!, + ) as JsonOutputEvent; + expect(parsed).toEqual({ type: "skill_activated", name: "review" }); + }); + + it("error: { type, message: string }", () => { + const parsed = JSON.parse( + toJsonLine({ type: "error", message: "something went wrong" })!, + ) as JsonOutputEvent; + expect(parsed).toEqual({ type: "error", message: "something went wrong" }); + }); + + it("notice: { type, message: string }", () => { + const parsed = JSON.parse( + toJsonLine({ type: "notice", message: "context compacted" })!, + ) as JsonOutputEvent; + expect(parsed).toEqual({ type: "notice", message: "context compacted" }); + }); + + it("done: { type } only — no extra fields", () => { + const parsed = JSON.parse(toJsonLine({ type: "done" })!); + expect(parsed).toEqual({ type: "done" }); + expect(Object.keys(parsed)).toHaveLength(1); + }); +}); + +describe("json-schema contract — schema coverage", () => { + it("covers every AgentEvent type", () => { + const covered = new Set(ALL_EVENT_CASES.map((e) => e.type)); + // Exhaustive list from AgentEvent union — update here when new event types land + const expectedTypes = new Set([ + "text", + "tool_call", + "tool_result", + "skill_activated", + "error", + "notice", + "done", + ]); + for (const t of expectedTypes) { + expect(covered.has(t)).toBe(true); + } + }); +});