feat(events): emit 5-way token breakdown + context-window utilization in message_complete

EVENTS.md

@@ -82,13 +82,36 @@ Emitted when an assistant message finishes (one per LLM turn).
   "providerID": "anthropic",
   "agent": "default",
   "cost": { "input": 0.003, "output": 0.012, "cache": { "read": 0, "write": 0 } },
-  "tokens": { "input": 1024, "output": 512, "cache": { "read": 0, "write": 0 } },
+  "tokens": {
+    "input": 1024,
+    "output": 512,
+    "reasoning": 0,
+    "cache": { "read": 8800, "write": 1024 }
+  },
+  "context": { "used": 10848, "limit": 200000, "ratio": 0.05424 },
   "finish": "tool-calls"
 }
 ```
 
 `finish` values: `"tool-calls"` (model wants to call tools), `"end_turn"` (model is done), `"max_tokens"` (output truncated).
 
+**`tokens`** (5-way breakdown, mirrors upstream `LLM.Usage`):
+
+- `input` (number) — raw input tokens billed at the standard input rate.
+- `output` (number) — output (completion) tokens.
+- `reasoning` (number) — extended-thinking / reasoning tokens (0 when thinking is off).
+- `cache.read` (number) — tokens served from the prompt cache (billed at cache-read rate). Distinguishes cache hits from fresh input.
+- `cache.write` (number) — tokens written to the prompt cache (billed at cache-write rate).
+
+These fields are non-overlapping: a token is counted in exactly one bucket.
+
+**`context`** (context-window utilization):
+
+- `used` (number) — tokens occupying the model's context window this turn: `input + cache.read + cache.write`.
+- `limit` (number) — the model's total context-window size in tokens, sourced from the models.dev registry (`model.limit.context`).
+- `ratio` (number) — `used / limit` (≥0; may exceed 1 if usage exceeds the model's registered limit). A value approaching or exceeding 1 signals context-exhaustion risk.
+- `null` — emitted when the model's context limit is not known (e.g. unregistered custom endpoint).
+
 ### `text`
 
 Emitted when a text block from the assistant is complete.

packages/cli/src/cli/cmd/run.ts

@@ -80,6 +80,31 @@ function fallback(part: ToolPart) {
   })
 }
 
+/**
+ * Build the context-window utilization object for `message_complete` events.
+ *
+ * Returns `null` (meaning "unknown") when:
+ * - `contextLimit` is `null` — Provider.getModel threw (unregistered model)
+ * - `contextLimit` is `0`   — custom model without a registered limit defaults to
+ *   `limit.context = 0` (the provider's default for unregistered custom models).
+ *   A zero limit would yield `Infinity`/`NaN` for ratio, which `JSON.stringify`
+ *   serialises as `null` inside the object — diverging from the documented
+ *   top-level `null` contract (EVENTS.md).
+ *
+ * @internal exported for unit-testing only
+ */
+export function buildContextWindow(
+  contextLimit: number | null,
+  contextUsed: number,
+): { used: number; limit: number; ratio: number } | null {
+  if (contextLimit == null || contextLimit <= 0) return null
+  return {
+    used: contextUsed,
+    limit: contextLimit,
+    ratio: contextUsed / contextLimit,
+  }
+}
+
 function glob(info: ToolProps<typeof GlobTool>) {
   const root = info.input.path ?? ""
   const title = `Glob "${info.input.pattern}"`
@@ -463,12 +488,46 @@ export const RunCommand = cmd({
             const info = event.properties.info
             if (args.format === "json") {
               if (info.finish) {
+                // Build 5-way token breakdown mirroring upstream LLM.Usage shape.
+                // info.tokens already carries the full breakdown from StepFinishPart
+                // accumulation — reasoning and cache split are not dropped upstream.
+                const tokens = {
+                  input: info.tokens.input,
+                  output: info.tokens.output,
+                  reasoning: info.tokens.reasoning,
+                  cache: {
+                    read: info.tokens.cache.read,
+                    write: info.tokens.cache.write,
+                  },
+                }
+
+                // Context-window utilization: used = input + cache.read + cache.write
+                // (all prompt tokens that occupy the model's context window this turn).
+                // cache.write tokens are written to the cache ON this turn — they are
+                // part of the prompt sent to the model and count against the context
+                // window, just billed at the cache-write rate. Excluding them
+                // undercounts utilization on the first turn of a conversation.
+                // limit comes from the model registry (models.dev). On lookup failure
+                // (or limit===0 for custom models) buildContextWindow returns null.
+                const contextLimit = await Provider.getModel(info.providerID, info.modelID)
+                  .then((m) => m.limit.context)
+                  .catch((e) => {
+                    if (e instanceof Provider.ModelNotFoundError) return null
+                    throw e
+                  })
+                const contextUsed = tokens.input + tokens.cache.read + tokens.cache.write
+                const context = buildContextWindow(contextLimit, contextUsed)
+
                 emit("message_complete", {
                   modelID: info.modelID,
                   providerID: info.providerID,
                   agent: info.agent,
+                  // cost is sourced from info.cost which accumulates real per-step costs
+                  // from StepFinishPart. Do NOT use the new step.ended event cost field
+                  // which emits cost:0 and is reconciled later (the cost:0 trap).
                   cost: info.cost,
-                  tokens: info.tokens,
+                  tokens,
+                  context,
                   finish: info.finish,
                 })
               }

packages/cli/test/cli/usage-token-breakdown.test.ts

@@ -0,0 +1,191 @@
+import path from "path"
+import { describe, expect, test } from "bun:test"
+import { buildContextWindow } from "../../src/cli/cmd/run"
+
+const EVENTS_MD = path.resolve(import.meta.dir, "../../../../EVENTS.md")
+
+/**
+ * Tests for issue #86 — 5-way token breakdown + context-window utilization
+ * in message_complete events.
+ */
+describe("buildContextWindow (#86)", () => {
+  // 🟠 regression: limit===0 (custom model default) must return null, not {ratio:Infinity}
+  test("returns null when contextLimit is 0 (custom model without registered limit)", () => {
+    const result = buildContextWindow(0, 9824)
+    expect(result).toBeNull()
+  })
+
+  test("returns null when contextLimit is null (Provider.getModel threw)", () => {
+    const result = buildContextWindow(null, 9824)
+    expect(result).toBeNull()
+  })
+
+  test("returns null when both limit and used are 0", () => {
+    const result = buildContextWindow(0, 0)
+    expect(result).toBeNull()
+  })
+
+  test("computes used as the value passed in (caller sets input + cache.read + cache.write)", () => {
+    // Regression: cache.write must be included in contextUsed at the call site (run.ts).
+    // buildContextWindow receives the pre-summed value; this test verifies the helper
+    // honours it (and that the sum is documented correctly: input + cache.read + cache.write).
+    const input = 1024
+    const cacheRead = 8800
+    const cacheWrite = 1024
+    const contextUsed = input + cacheRead + cacheWrite // = 10848
+    const result = buildContextWindow(200_000, contextUsed)
+    expect(result).not.toBeNull()
+    expect(result!.used).toBe(10848)
+  })
+
+  test("sets limit to the contextLimit value", () => {
+    const result = buildContextWindow(200_000, 9824)
+    expect(result).not.toBeNull()
+    expect(result!.limit).toBe(200_000)
+  })
+
+  test("ratio is used / limit", () => {
+    const result = buildContextWindow(200_000, 9824)
+    expect(result).not.toBeNull()
+    expect(result!.ratio).toBeCloseTo(9824 / 200_000, 10)
+  })
+
+  test("ratio is between 0 and 1 for realistic values", () => {
+    const result = buildContextWindow(128_000, 64_000)
+    expect(result).not.toBeNull()
+    expect(result!.ratio).toBe(0.5)
+  })
+
+  test("ratio is exactly 1 when context is fully used", () => {
+    const result = buildContextWindow(100_000, 100_000)
+    expect(result).not.toBeNull()
+    expect(result!.ratio).toBe(1)
+  })
+
+  test("ratio is 0 when no tokens used (empty prompt start)", () => {
+    const result = buildContextWindow(200_000, 0)
+    expect(result).not.toBeNull()
+    expect(result!.ratio).toBe(0)
+  })
+
+  test("result is JSON-serialisable without Infinity or NaN", () => {
+    const result = buildContextWindow(200_000, 9824)
+    const serialised = JSON.stringify(result)
+    expect(serialised).not.toContain("null")
+    const parsed = JSON.parse(serialised)
+    expect(parsed.ratio).toBeCloseTo(9824 / 200_000, 10)
+  })
+
+  test("top-level null serialises cleanly (not as object with null ratio)", () => {
+    // The documented contract: limit unknown → top-level null, not {ratio:null}
+    const result = buildContextWindow(0, 9824)
+    expect(JSON.stringify(result)).toBe("null")
+  })
+
+  test("ratio may exceed 1 when usage exceeds the registered limit (unclamped)", () => {
+    // EVENTS.md documents ratio as ≥0 (may exceed 1), not clamped to [0,1].
+    // Stale/lowered model.limit.context can produce ratio > 1 in production.
+    const result = buildContextWindow(100_000, 110_000)
+    expect(result).not.toBeNull()
+    expect(result!.ratio).toBeGreaterThan(1)
+    expect(result!.ratio).toBeCloseTo(1.1, 10)
+  })
+})
+
+describe("message_complete emit block shape (source-verified, #86)", () => {
+  // These source-text checks verify structural wiring in the emit call site
+  // that cannot be covered by pure unit-testing buildContextWindow.
+  const RUN_SRC = path.resolve(import.meta.dir, "../../src/cli/cmd/run.ts")
+
+  test("emit block passes tokens with reasoning and cache read/write fields", async () => {
+    const source = await Bun.file(RUN_SRC).text()
+    const emitIdx = source.indexOf('emit("message_complete"')
+    expect(emitIdx).toBeGreaterThan(-1)
+    const blockStart = Math.max(0, emitIdx - 1500)
+    const block = source.slice(blockStart, emitIdx + 200)
+    expect(block).toContain("reasoning")
+    expect(block).toContain("cache")
+    expect(block).toContain("read")
+    expect(block).toContain("write")
+  })
+
+  test("emit block calls buildContextWindow (not inline ternary)", async () => {
+    const source = await Bun.file(RUN_SRC).text()
+    const emitIdx = source.indexOf('emit("message_complete"')
+    expect(emitIdx).toBeGreaterThan(-1)
+    const blockStart = Math.max(0, emitIdx - 1500)
+    const block = source.slice(blockStart, emitIdx + 200)
+    expect(block).toContain("buildContextWindow")
+  })
+
+  test("emit block includes context field", async () => {
+    const source = await Bun.file(RUN_SRC).text()
+    const emitIdx = source.indexOf('emit("message_complete"')
+    expect(emitIdx).toBeGreaterThan(-1)
+    // emit object spans ~400 chars; search up to closing paren
+    const block = source.slice(emitIdx, emitIdx + 500)
+    expect(block).toContain("context")
+  })
+
+  test("cost field is still emitted (not regressed)", async () => {
+    const source = await Bun.file(RUN_SRC).text()
+    const idx = source.indexOf('emit("message_complete"')
+    expect(idx).toBeGreaterThan(-1)
+    const block = source.slice(idx, idx + 800)
+    expect(block).toContain("cost:")
+  })
+
+  test("getModel catch rethrows non-ModelNotFoundError (targeted catch, not swallow-all)", async () => {
+    // Verify the catch block only silences ModelNotFoundError; unexpected errors must propagate.
+    // Source check: catch body must reference ModelNotFoundError (not be an empty arrow).
+    const source = await Bun.file(RUN_SRC).text()
+    const getModelIdx = source.indexOf("Provider.getModel(info.providerID")
+    expect(getModelIdx).toBeGreaterThan(-1)
+    const catchWindow = source.slice(getModelIdx, getModelIdx + 400)
+    expect(catchWindow).toContain("ModelNotFoundError")
+    expect(catchWindow).toContain("throw e")
+  })
+
+  test("contextUsed includes cache.write (regression: must not omit cache.write from context sum)", async () => {
+    // Regression for the bug where `contextUsed = tokens.input + tokens.cache.read`
+    // omitted cache.write, undercounting first-turn utilization. The fix is:
+    //   const contextUsed = tokens.input + tokens.cache.read + tokens.cache.write
+    // This source-text check verifies the three-way sum is present at the call site.
+    const source = await Bun.file(RUN_SRC).text()
+    const contextUsedIdx = source.indexOf("const contextUsed =")
+    expect(contextUsedIdx).toBeGreaterThan(-1)
+    const line = source.slice(contextUsedIdx, contextUsedIdx + 100)
+    expect(line).toContain("cache.write")
+  })
+})
+
+describe("EVENTS.md documents token breakdown and context (#86)", () => {
+  test("EVENTS.md message_complete section includes reasoning token field", async () => {
+    const doc = await Bun.file(EVENTS_MD).text()
+    const idx = doc.indexOf("message_complete")
+    expect(idx).toBeGreaterThan(-1)
+    const section = doc.slice(idx, idx + 1500)
+    expect(section).toContain("reasoning")
+  })
+
+  test("EVENTS.md message_complete section documents context field", async () => {
+    const doc = await Bun.file(EVENTS_MD).text()
+    const idx = doc.indexOf("message_complete")
+    expect(idx).toBeGreaterThan(-1)
+    const section = doc.slice(idx, idx + 1500)
+    expect(section).toContain("context")
+    expect(section).toContain("used")
+    expect(section).toContain("limit")
+    expect(section).toContain("ratio")
+  })
+
+  test("EVENTS.md documents null as the unknown-limit sentinel", async () => {
+    const doc = await Bun.file(EVENTS_MD).text()
+    const idx = doc.indexOf("message_complete")
+    expect(idx).toBeGreaterThan(-1)
+    // null sentinel doc is ~1593 chars after message_complete heading; use 2000 window
+    const section = doc.slice(idx, idx + 2000)
+    // The documented contract: null = context limit not known
+    expect(section).toContain("null")
+  })
+})